You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/11/26 06:16:21 UTC
[3/4] kylin git commit: KYLIN-3044 minor code review
KYLIN-3044 minor code review
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/695971a2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/695971a2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/695971a2
Branch: refs/heads/KYLIN-2966
Commit: 695971a2a27837bbf10f3fb0c4079ce11333951d
Parents: cac1f8b
Author: Li Yang <li...@apache.org>
Authored: Sun Nov 26 13:04:27 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Sun Nov 26 13:04:27 2017 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/KylinConfigBase.java | 2 +-
.../java/org/apache/kylin/job/JoinedFlatTable.java | 3 ++-
.../org/apache/kylin/source/jdbc/JdbcExplorer.java | 2 +-
.../apache/kylin/source/jdbc/JdbcHiveMRInput.java | 15 ++++++++-------
.../java/org/apache/kylin/source/jdbc/SqlUtil.java | 2 +-
.../org/apache/kylin/source/jdbc/SqlUtilTest.java | 6 +++---
6 files changed, 16 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/695971a2/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index b2368ce..1a93dd4 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -752,7 +752,7 @@ abstract public class KylinConfigBase implements Serializable {
return Integer.parseInt(getOptional("kylin.source.jdbc.sqoop-mapper-num", "4"));
}
- public String getFieldDelimiter() {
+ public String getSourceFieldDelimiter() {
return getOptional("kylin.source.jdbc.field-delimiter", "|");
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/695971a2/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
index 0ab3d3d..d136ec6 100644
--- a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
@@ -85,7 +85,8 @@ public class JoinedFlatTable {
public static String generateCreateTableStatement(IJoinedFlatTableDesc flatDesc, String storageDfsDir,
String format) {
- return generateCreateTableStatement(flatDesc, storageDfsDir, format, "|");
+ String fieldDelimiter = flatDesc.getDataModel().getConfig().getSourceFieldDelimiter();
+ return generateCreateTableStatement(flatDesc, storageDfsDir, format, fieldDelimiter);
}
public static String generateDropTableStatement(IJoinedFlatTableDesc flatDesc) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/695971a2/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java b/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java
index 1278128..2827b3d 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java
@@ -112,7 +112,7 @@ public class JdbcExplorer implements ISourceMetadataExplorer, ISampleDataDeploye
ColumnDesc cdesc = new ColumnDesc();
cdesc.setName(cname.toUpperCase());
- String kylinType = SqlUtil.jdbcTypetoKylinDataType(type);
+ String kylinType = SqlUtil.jdbcTypeToKylinDataType(type);
int precision = (SqlUtil.isPrecisionApplicable(kylinType) && csize > 0) ? csize : -1;
int scale = (SqlUtil.isScaleApplicable(kylinType) && digits > 0) ? digits : -1;
http://git-wip-us.apache.org/repos/asf/kylin/blob/695971a2/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.java
index e83518a..15259cc 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.java
@@ -21,7 +21,6 @@ package org.apache.kylin.source.jdbc;
import java.util.List;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.engine.mr.steps.CubingExecutableUtil;
import org.apache.kylin.job.JoinedFlatTable;
import org.apache.kylin.job.constant.ExecutableConstants;
@@ -51,6 +50,10 @@ public class JdbcHiveMRInput extends HiveMRInput {
public BatchCubingInputSide(IJoinedFlatTableDesc flatDesc) {
super(flatDesc);
}
+
+ private KylinConfig getConfig() {
+ return flatDesc.getDataModel().getConfig();
+ }
@Override
protected void addStepPhase1_DoCreateFlatTable(DefaultChainedExecutable jobFlow) {
@@ -64,8 +67,7 @@ public class JdbcHiveMRInput extends HiveMRInput {
private AbstractExecutable createFlatHiveTableFromFiles(String hiveInitStatements, String jobWorkingDir) {
final String dropTableHql = JoinedFlatTable.generateDropTableStatement(flatDesc);
- KylinConfig config = KylinConfig.getInstanceFromEnv();
- String filedDelimiter = config.getFieldDelimiter();
+ String filedDelimiter = getConfig().getSourceFieldDelimiter();
// Sqoop does not support exporting SEQUENSEFILE to Hive now SQOOP-869
final String createTableHql = JoinedFlatTable.generateCreateTableStatement(flatDesc, jobWorkingDir,
"TEXTFILE", filedDelimiter);
@@ -97,7 +99,7 @@ public class JdbcHiveMRInput extends HiveMRInput {
return partitionDesc.getPartitionDateColumnRef();
}
TblColRef splitColumn = null;
- TableMetadataManager tblManager = TableMetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
+ TableMetadataManager tblManager = TableMetadataManager.getInstance(getConfig());
long maxCardinality = 0;
for (TableRef tableRef : flatDesc.getDataModel().getAllTables()) {
TableExtDesc tableExtDesc = tblManager.getTableExt(tableRef.getTableDesc());
@@ -123,8 +125,7 @@ public class JdbcHiveMRInput extends HiveMRInput {
}
private AbstractExecutable createSqoopToFlatHiveStep(String jobWorkingDir, String cubeName) {
- KylinConfig config = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).getCube(cubeName)
- .getConfig();
+ KylinConfig config = getConfig();
PartitionDesc partitionDesc = flatDesc.getDataModel().getPartitionDesc();
String partCol = null;
String partitionString = null;
@@ -151,7 +152,7 @@ public class JdbcHiveMRInput extends HiveMRInput {
String jdbcUser = config.getJdbcSourceUser();
String jdbcPass = config.getJdbcSourcePass();
String sqoopHome = config.getSqoopHome();
- String filedDelimiter = config.getFieldDelimiter();
+ String filedDelimiter = config.getSourceFieldDelimiter();
int mapperNum = config.getSqoopMapperNum();
String bquery = String.format("SELECT min(%s), max(%s) FROM %s.%s", splitColumn, splitColumn, splitDatabase,
http://git-wip-us.apache.org/repos/asf/kylin/blob/695971a2/source-hive/src/main/java/org/apache/kylin/source/jdbc/SqlUtil.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/jdbc/SqlUtil.java b/source-hive/src/main/java/org/apache/kylin/source/jdbc/SqlUtil.java
index 79fab7d..715bb99 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/jdbc/SqlUtil.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/jdbc/SqlUtil.java
@@ -96,7 +96,7 @@ public class SqlUtil {
return con;
}
- public static String jdbcTypetoKylinDataType(int sqlType) {
+ public static String jdbcTypeToKylinDataType(int sqlType) {
String result = "any";
switch (sqlType) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/695971a2/source-hive/src/test/java/org/apache/kylin/source/jdbc/SqlUtilTest.java
----------------------------------------------------------------------
diff --git a/source-hive/src/test/java/org/apache/kylin/source/jdbc/SqlUtilTest.java b/source-hive/src/test/java/org/apache/kylin/source/jdbc/SqlUtilTest.java
index d952675..7a7fd33 100644
--- a/source-hive/src/test/java/org/apache/kylin/source/jdbc/SqlUtilTest.java
+++ b/source-hive/src/test/java/org/apache/kylin/source/jdbc/SqlUtilTest.java
@@ -27,9 +27,9 @@ public class SqlUtilTest {
@Test
public void testJdbcTypetoKylinDataType() {
this.getClass().getClassLoader().toString();
- Assert.assertEquals("double", SqlUtil.jdbcTypetoKylinDataType(Types.FLOAT));
- Assert.assertEquals("varchar", SqlUtil.jdbcTypetoKylinDataType(Types.NVARCHAR));
- Assert.assertEquals("any", SqlUtil.jdbcTypetoKylinDataType(Types.ARRAY));
+ Assert.assertEquals("double", SqlUtil.jdbcTypeToKylinDataType(Types.FLOAT));
+ Assert.assertEquals("varchar", SqlUtil.jdbcTypeToKylinDataType(Types.NVARCHAR));
+ Assert.assertEquals("any", SqlUtil.jdbcTypeToKylinDataType(Types.ARRAY));
}
@Test