You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2017/12/05 10:05:50 UTC
kylin git commit: KYLIN-3052 Support Redshift as data source
Repository: kylin
Updated Branches:
refs/heads/KYLIN-3052 [created] fb2c9a5b6
KYLIN-3052 Support Redshift as data source
Signed-off-by: shaofengshi <sh...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/fb2c9a5b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/fb2c9a5b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/fb2c9a5b
Branch: refs/heads/KYLIN-3052
Commit: fb2c9a5b6e033907e5109d2b0cb0027fdd5e5647
Parents: 6474e22
Author: ZhansShaoxiong <sh...@gmail.com>
Authored: Sat Dec 2 15:28:00 2017 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Tue Dec 5 17:55:11 2017 +0800
----------------------------------------------------------------------
.../org/apache/kylin/job/constant/ExecutableConstants.java | 1 +
.../java/org/apache/kylin/rest/service/TableService.java | 8 ++++----
.../java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.java | 3 ++-
.../kylin/source/jdbc/metadata/MySQLJdbcMetadata.java | 6 ++----
.../kylin/source/jdbc/metadata/MySQLJdbcMetadataTest.java | 9 ++-------
5 files changed, 11 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/fb2c9a5b/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java b/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
index f3caf3b..cf61722 100644
--- a/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
+++ b/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
@@ -35,6 +35,7 @@ public final class ExecutableConstants {
public static final String STEP_NAME_BUILD_DICTIONARY = "Build Dimension Dictionary";
public static final String STEP_NAME_BUILD_UHC_DICTIONARY = "Build UHC Dictionary";
public static final String STEP_NAME_CREATE_FLAT_HIVE_TABLE = "Create Intermediate Flat Hive Table";
+ public static final String STEP_NAME_SQOOP_TO_FLAT_HIVE_TABLE = "Sqoop To Flat Hive Table";
public static final String STEP_NAME_MATERIALIZE_HIVE_VIEW_IN_LOOKUP = "Materialize Hive View in Lookup Tables";
public static final String STEP_NAME_FACT_DISTINCT_COLUMNS = "Extract Fact Table Distinct Columns";
public static final String STEP_NAME_CALCULATE_STATS_FROM_BASE_CUBOID = "Calculate Statistics from Base Cuboid";
http://git-wip-us.apache.org/repos/asf/kylin/blob/fb2c9a5b/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java b/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java
index ffb71a6..901ac46 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java
@@ -119,7 +119,7 @@ public class TableService extends BasicService {
SetMultimap<String, String> db2tables = LinkedHashMultimap.create();
for (String fullTableName : tables) {
String[] parts = HadoopUtil.parseHiveTableName(fullTableName);
- db2tables.put(parts[0].toUpperCase(), parts[1].toUpperCase());
+ db2tables.put(parts[0], parts[1]);
}
// load all tables first
@@ -128,9 +128,9 @@ public class TableService extends BasicService {
for (Map.Entry<String, String> entry : db2tables.entries()) {
Pair<TableDesc, TableExtDesc> pair = explr.loadTableMetadata(entry.getKey(), entry.getValue(), project);
TableDesc tableDesc = pair.getFirst();
- Preconditions.checkState(tableDesc.getDatabase().equals(entry.getKey()));
- Preconditions.checkState(tableDesc.getName().equals(entry.getValue()));
- Preconditions.checkState(tableDesc.getIdentity().equals(entry.getKey() + "." + entry.getValue()));
+ Preconditions.checkState(tableDesc.getDatabase().equals(entry.getKey().toUpperCase()));
+ Preconditions.checkState(tableDesc.getName().equals(entry.getValue().toUpperCase()));
+ Preconditions.checkState(tableDesc.getIdentity().equals(entry.getKey().toUpperCase() + "." + entry.getValue().toUpperCase()));
TableExtDesc extDesc = pair.getSecond();
Preconditions.checkState(tableDesc.getIdentity().equals(extDesc.getIdentity()));
allMeta.add(pair);
http://git-wip-us.apache.org/repos/asf/kylin/blob/fb2c9a5b/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.java
index 15259cc..0bfc145 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.java
@@ -74,6 +74,7 @@ public class JdbcHiveMRInput extends HiveMRInput {
HiveCmdStep step = new HiveCmdStep();
step.setCmd(hiveInitStatements + dropTableHql + createTableHql);
+ step.setName(ExecutableConstants.STEP_NAME_CREATE_FLAT_HIVE_TABLE);
return step;
}
@@ -171,7 +172,7 @@ public class JdbcHiveMRInput extends HiveMRInput {
logger.debug(String.format("sqoop cmd:%s", cmd));
CmdStep step = new CmdStep();
step.setCmd(cmd);
- step.setName(ExecutableConstants.STEP_NAME_CREATE_FLAT_HIVE_TABLE);
+ step.setName(ExecutableConstants.STEP_NAME_SQOOP_TO_FLAT_HIVE_TABLE);
return step;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/fb2c9a5b/source-hive/src/main/java/org/apache/kylin/source/jdbc/metadata/MySQLJdbcMetadata.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/jdbc/metadata/MySQLJdbcMetadata.java b/source-hive/src/main/java/org/apache/kylin/source/jdbc/metadata/MySQLJdbcMetadata.java
index 6404fd6..54c2a03 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/jdbc/metadata/MySQLJdbcMetadata.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/jdbc/metadata/MySQLJdbcMetadata.java
@@ -35,10 +35,8 @@ public class MySQLJdbcMetadata extends DefaultJdbcMetadata {
@Override
public List<String> listDatabases() throws SQLException {
List<String> ret = new ArrayList<>();
- try (Connection con = SqlUtil.getConnection(dbconf); ResultSet res = con.getMetaData().getCatalogs()) {
- while (res.next()) {
- ret.add(res.getString("TABLE_CAT"));
- }
+ try (Connection con = SqlUtil.getConnection(dbconf)) {
+ ret.add(con.getCatalog());
}
return ret;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/fb2c9a5b/source-hive/src/test/java/org/apache/kylin/source/jdbc/metadata/MySQLJdbcMetadataTest.java
----------------------------------------------------------------------
diff --git a/source-hive/src/test/java/org/apache/kylin/source/jdbc/metadata/MySQLJdbcMetadataTest.java b/source-hive/src/test/java/org/apache/kylin/source/jdbc/metadata/MySQLJdbcMetadataTest.java
index d0cb6c4..e461c15 100644
--- a/source-hive/src/test/java/org/apache/kylin/source/jdbc/metadata/MySQLJdbcMetadataTest.java
+++ b/source-hive/src/test/java/org/apache/kylin/source/jdbc/metadata/MySQLJdbcMetadataTest.java
@@ -47,16 +47,11 @@ public class MySQLJdbcMetadataTest extends DefaultJdbcMetadataTest {
@Test
public void testListDatabases() throws SQLException {
- ResultSet rs = mock(ResultSet.class);
- when(rs.next()).thenReturn(true).thenReturn(true).thenReturn(false);
- when(rs.getString("TABLE_CAT")).thenReturn("catalog1").thenReturn("catalog2");
-
- when(connection.getMetaData()).thenReturn(dbmd);
- when(dbmd.getCatalogs()).thenReturn(rs);
+ when(connection.getCatalog()).thenReturn("catalog1");
List<String> dbs = jdbcMetadata.listDatabases();
- Assert.assertEquals(2, dbs.size());
+ Assert.assertEquals(1, dbs.size());
Assert.assertEquals("catalog1", dbs.get(0));
}