You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ka...@apache.org on 2017/07/07 08:38:49 UTC
[47/50] [abbrv] kylin git commit: KYLIN-1351 Add unit tests for JDBC
source
KYLIN-1351 Add unit tests for JDBC source
Signed-off-by: Li Yang <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ad47dc00
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ad47dc00
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ad47dc00
Branch: refs/heads/KYLIN-2606
Commit: ad47dc00e672e8e967d8605d2c8d607ce28225d5
Parents: 926a17a
Author: auphyroc99 <45...@qq.com>
Authored: Wed Jul 5 11:03:00 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Wed Jul 5 17:02:56 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/common/KylinConfigBase.java | 1 +
.../jdbc/ITJdbcSourceTableLoaderTest.java | 121 +++++++++++++++++++
.../source/jdbc/ITJdbcTableReaderTest.java | 116 ++++++++++++++++++
source-hive/pom.xml | 5 +
4 files changed, 243 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/ad47dc00/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 179d61f..f9c3adb 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -532,6 +532,7 @@ abstract public class KylinConfigBase implements Serializable {
// ref constants in ISourceAware
r.put(0, "org.apache.kylin.source.hive.HiveSource");
r.put(1, "org.apache.kylin.source.kafka.KafkaSource");
+ r.put(8, "org.apache.kylin.source.jdbc.JdbcSource");
r.putAll(convertKeyToInteger(getPropertiesByPrefix("kylin.source.provider.")));
return r;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/ad47dc00/kylin-it/src/test/java/org/apache/kylin/source/jdbc/ITJdbcSourceTableLoaderTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/source/jdbc/ITJdbcSourceTableLoaderTest.java b/kylin-it/src/test/java/org/apache/kylin/source/jdbc/ITJdbcSourceTableLoaderTest.java
new file mode 100644
index 0000000..a84dca9
--- /dev/null
+++ b/kylin-it/src/test/java/org/apache/kylin/source/jdbc/ITJdbcSourceTableLoaderTest.java
@@ -0,0 +1,121 @@
+package org.apache.kylin.source.jdbc;
+
+import static org.junit.Assert.assertTrue;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.sql.Types;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.model.ISourceAware;
+import org.apache.kylin.metadata.model.TableDesc;
+import org.apache.kylin.metadata.model.TableExtDesc;
+import org.apache.kylin.query.H2Database;
+import org.apache.kylin.source.ISource;
+import org.apache.kylin.source.ISourceMetadataExplorer;
+import org.apache.kylin.source.SourceFactory;
+import org.apache.kylin.source.datagen.ModelDataGenerator;
+import org.dbunit.DatabaseUnitException;
+import org.dbunit.database.DatabaseConfig;
+import org.dbunit.dataset.datatype.DataType;
+import org.dbunit.dataset.datatype.DataTypeException;
+import org.dbunit.ext.h2.H2Connection;
+import org.dbunit.ext.h2.H2DataTypeFactory;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class ITJdbcSourceTableLoaderTest extends LocalFileMetadataTestCase implements ISourceAware {
+
+ protected KylinConfig config = null;
+ protected static Connection h2Connection = null;
+
+ @Before
+ public void setup() throws Exception {
+
+ super.createTestMetadata();
+
+ System.setProperty("kylin.source.jdbc.connection-url", "jdbc:h2:mem:db");
+ System.setProperty("kylin.source.jdbc.driver", "org.h2.Driver");
+ System.setProperty("kylin.source.jdbc.user", "sa");
+ System.setProperty("kylin.source.jdbc.pass", "");
+
+ config = KylinConfig.getInstanceFromEnv();
+
+ h2Connection = DriverManager.getConnection("jdbc:h2:mem:db", "sa", "");
+
+ H2Database h2DB = new H2Database(h2Connection, config);
+
+ MetadataManager mgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
+ ModelDataGenerator gen = new ModelDataGenerator(mgr.getDataModelDesc("ci_left_join_model"), 10000);
+ gen.generate();
+
+ h2DB.loadAllTables();
+
+ }
+
+ @After
+ public void after() throws Exception {
+
+ super.cleanupTestMetadata();
+
+ if (h2Connection != null) {
+ try {
+ h2Connection.close();
+ } catch (SQLException e) {
+ e.printStackTrace();
+ }
+ }
+
+ System.clearProperty("kylin.source.jdbc.connection-url");
+ System.clearProperty("kylin.source.jdbc.driver");
+ System.clearProperty("kylin.source.jdbc.user");
+ System.clearProperty("kylin.source.jdbc.pass");
+
+ }
+
+ @Test
+ public void test() throws Exception {
+
+ ISource source = SourceFactory.getSource(new ITJdbcSourceTableLoaderTest());
+ ISourceMetadataExplorer explr = source.getSourceMetadataExplorer();
+ Pair<TableDesc, TableExtDesc> pair;
+
+ pair = explr.loadTableMetadata("DEFAULT", "TEST_KYLIN_FACT");
+ assertTrue(pair.getFirst().getIdentity().equals("DEFAULT.TEST_KYLIN_FACT"));
+
+ pair = explr.loadTableMetadata("EDW", "TEST_CAL_DT");
+ assertTrue(pair.getFirst().getIdentity().equals("EDW.TEST_CAL_DT"));
+
+ }
+
+ @Override
+ public int getSourceType() {
+ return ISourceAware.ID_JDBC;
+ }
+
+ @SuppressWarnings("deprecation")
+ protected static H2Connection newH2Connection() throws DatabaseUnitException {
+ H2Connection h2Conn = new H2Connection(h2Connection, null);
+ h2Conn.getConfig().setProperty(DatabaseConfig.PROPERTY_DATATYPE_FACTORY, new TestH2DataTypeFactory());
+ h2Conn.getConfig().setFeature(DatabaseConfig.FEATURE_DATATYPE_WARNING, false);
+ return h2Conn;
+ }
+
+ public static class TestH2DataTypeFactory extends H2DataTypeFactory {
+ @Override
+ public DataType createDataType(int sqlType, String sqlTypeName, String tableName, String columnName)
+ throws DataTypeException {
+
+ if ((columnName.startsWith("COL") || columnName.startsWith("col")) && sqlType == Types.BIGINT) {
+ return DataType.INTEGER;
+ }
+ return super.createDataType(sqlType, sqlTypeName);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/ad47dc00/kylin-it/src/test/java/org/apache/kylin/source/jdbc/ITJdbcTableReaderTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/source/jdbc/ITJdbcTableReaderTest.java b/kylin-it/src/test/java/org/apache/kylin/source/jdbc/ITJdbcTableReaderTest.java
new file mode 100644
index 0000000..7850952
--- /dev/null
+++ b/kylin-it/src/test/java/org/apache/kylin/source/jdbc/ITJdbcTableReaderTest.java
@@ -0,0 +1,116 @@
+package org.apache.kylin.source.jdbc;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.sql.Types;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.model.ISourceAware;
+import org.apache.kylin.query.H2Database;
+import org.apache.kylin.source.datagen.ModelDataGenerator;
+import org.dbunit.DatabaseUnitException;
+import org.dbunit.database.DatabaseConfig;
+import org.dbunit.dataset.datatype.DataType;
+import org.dbunit.dataset.datatype.DataTypeException;
+import org.dbunit.ext.h2.H2Connection;
+import org.dbunit.ext.h2.H2DataTypeFactory;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class ITJdbcTableReaderTest extends LocalFileMetadataTestCase implements ISourceAware {
+
+ protected KylinConfig config = null;
+ protected static Connection h2Connection = null;
+
+ @Before
+ public void setup() throws Exception {
+
+ super.createTestMetadata();
+
+ System.setProperty("kylin.source.jdbc.connection-url", "jdbc:h2:mem:db");
+ System.setProperty("kylin.source.jdbc.driver", "org.h2.Driver");
+ System.setProperty("kylin.source.jdbc.user", "sa");
+ System.setProperty("kylin.source.jdbc.pass", "");
+
+ config = KylinConfig.getInstanceFromEnv();
+
+ h2Connection = DriverManager.getConnection("jdbc:h2:mem:db", "sa", "");
+
+ H2Database h2DB = new H2Database(h2Connection, config);
+
+ MetadataManager mgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
+ ModelDataGenerator gen = new ModelDataGenerator(mgr.getDataModelDesc("ci_left_join_model"), 10000);
+ gen.generate();
+
+ h2DB.loadAllTables();
+
+ }
+
+ @After
+ public void after() throws Exception {
+
+ super.cleanupTestMetadata();
+
+ if (h2Connection != null) {
+ try {
+ h2Connection.close();
+ } catch (SQLException e) {
+ e.printStackTrace();
+ }
+ }
+
+ System.clearProperty("kylin.source.jdbc.connection-url");
+ System.clearProperty("kylin.source.jdbc.driver");
+ System.clearProperty("kylin.source.jdbc.user");
+ System.clearProperty("kylin.source.jdbc.pass");
+
+ }
+
+ @Test
+ public void test() throws Exception {
+
+ JdbcTableReader reader = new JdbcTableReader("default", "test_kylin_fact");
+ int rowNumber = 0;
+ while (reader.next()) {
+ String[] row = reader.getRow();
+ Assert.assertEquals(11, row.length);
+
+ rowNumber++;
+ }
+
+ reader.close();
+ Assert.assertEquals(10000, rowNumber);
+
+ }
+
+ @Override
+ public int getSourceType() {
+ return ISourceAware.ID_JDBC;
+ }
+
+ @SuppressWarnings("deprecation")
+ protected static H2Connection newH2Connection() throws DatabaseUnitException {
+ H2Connection h2Conn = new H2Connection(h2Connection, null);
+ h2Conn.getConfig().setProperty(DatabaseConfig.PROPERTY_DATATYPE_FACTORY, new TestH2DataTypeFactory());
+ h2Conn.getConfig().setFeature(DatabaseConfig.FEATURE_DATATYPE_WARNING, false);
+ return h2Conn;
+ }
+
+ public static class TestH2DataTypeFactory extends H2DataTypeFactory {
+ @Override
+ public DataType createDataType(int sqlType, String sqlTypeName, String tableName, String columnName)
+ throws DataTypeException {
+
+ if ((columnName.startsWith("COL") || columnName.startsWith("col")) && sqlType == Types.BIGINT) {
+ return DataType.INTEGER;
+ }
+ return super.createDataType(sqlType, sqlTypeName);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/ad47dc00/source-hive/pom.xml
----------------------------------------------------------------------
diff --git a/source-hive/pom.xml b/source-hive/pom.xml
index abe8af5..b58f25e 100644
--- a/source-hive/pom.xml
+++ b/source-hive/pom.xml
@@ -41,6 +41,11 @@
<!-- Env & Test -->
<dependency>
+ <groupId>com.h2database</groupId>
+ <artifactId>h2</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin-core-common</artifactId>
<type>test-jar</type>