You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2018/12/24 09:00:15 UTC
[kylin] branch master updated: KYLIN-3737 refactor cache part for
RDBMS
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/master by this push:
new 5982bb7 KYLIN-3737 refactor cache part for RDBMS
5982bb7 is described below
commit 5982bb79bacc5e0728df52a6e602239e1d2c6b26
Author: woyumen4597 <wo...@gmail.com>
AuthorDate: Mon Dec 24 13:51:06 2018 +0800
KYLIN-3737 refactor cache part for RDBMS
---
.../datasource/adaptor/AbstractJdbcAdaptor.java | 10 +-
.../sdk/datasource/adaptor/DefaultAdaptor.java | 122 +++++++++++++++------
.../src/main/resources/datasource/mysql.xml | 9 +-
3 files changed, 97 insertions(+), 44 deletions(-)
diff --git a/datasource-sdk/src/main/java/org/apache/kylin/sdk/datasource/adaptor/AbstractJdbcAdaptor.java b/datasource-sdk/src/main/java/org/apache/kylin/sdk/datasource/adaptor/AbstractJdbcAdaptor.java
index 3e36fae..3a66499 100644
--- a/datasource-sdk/src/main/java/org/apache/kylin/sdk/datasource/adaptor/AbstractJdbcAdaptor.java
+++ b/datasource-sdk/src/main/java/org/apache/kylin/sdk/datasource/adaptor/AbstractJdbcAdaptor.java
@@ -53,11 +53,11 @@ public abstract class AbstractJdbcAdaptor implements Closeable {
protected final DataSourceDef dataSourceDef;
protected SqlConverter.IConfigurer configurer;
protected final Cache<String, List<String>> columnsCache = CacheBuilder.newBuilder()
- .expireAfterWrite(1, TimeUnit.DAYS).maximumSize(30).build();
+ .expireAfterWrite(1, TimeUnit.DAYS).maximumSize(4096).build();
protected final Cache<String, List<String>> databasesCache = CacheBuilder.newBuilder()
- .expireAfterWrite(1, TimeUnit.DAYS).maximumSize(30).build();
+ .expireAfterWrite(1, TimeUnit.DAYS).maximumSize(4096).build();
protected final Cache<String, List<String>> tablesCache = CacheBuilder.newBuilder()
- .expireAfterWrite(1, TimeUnit.DAYS).maximumSize(30).build();
+ .expireAfterWrite(1, TimeUnit.DAYS).maximumSize(4096).build();
private static Joiner joiner = Joiner.on("_");
@@ -308,7 +308,7 @@ public abstract class AbstractJdbcAdaptor implements Closeable {
*/
public List<String> listDatabasesWithCache(boolean init) throws SQLException {
if (configurer.enableCache()) {
- String cacheKey = config.datasourceId + config.url + "_databases";
+ String cacheKey = joiner.join(config.datasourceId, config.url, "databases");
List<String> cachedDatabases;
if (init || (cachedDatabases = databasesCache.getIfPresent(cacheKey)) == null) {
cachedDatabases = listDatabases();
@@ -429,7 +429,7 @@ public abstract class AbstractJdbcAdaptor implements Closeable {
*/
public List<String> listColumnsWithCache(String database, String tableName, boolean init) throws SQLException {
if (configurer.enableCache()) {
- String cacheKey = config.datasourceId + config.url + "_" + tableName + "_columns";
+ String cacheKey = joiner.join(config.datasourceId, config.url, database, tableName, "columns");
List<String> cachedColumns;
if (init || (cachedColumns = columnsCache.getIfPresent(cacheKey)) == null) {
cachedColumns = listColumns(database, tableName);
diff --git a/datasource-sdk/src/main/java/org/apache/kylin/sdk/datasource/adaptor/DefaultAdaptor.java b/datasource-sdk/src/main/java/org/apache/kylin/sdk/datasource/adaptor/DefaultAdaptor.java
index 66c45e1..da24e98 100644
--- a/datasource-sdk/src/main/java/org/apache/kylin/sdk/datasource/adaptor/DefaultAdaptor.java
+++ b/datasource-sdk/src/main/java/org/apache/kylin/sdk/datasource/adaptor/DefaultAdaptor.java
@@ -28,6 +28,7 @@ import java.util.Locale;
import java.util.Map;
import javax.sql.rowset.CachedRowSet;
+import com.google.common.base.Joiner;
import org.apache.commons.lang.StringUtils;
/**
@@ -36,9 +37,7 @@ import org.apache.commons.lang.StringUtils;
*/
public class DefaultAdaptor extends AbstractJdbcAdaptor {
- protected static final String QUOTE_REG_LFT = "[`\"\\[]*";
- protected static final String QUOTE_REG_RHT = "[`\"\\]]*";
- private final static String [] POSSIBLE_TALBE_END= {",", " ", ")", "\r", "\n", "."};
+ private static Joiner joiner = Joiner.on('_');
public DefaultAdaptor(AdaptorConfig config) throws Exception {
super(config);
@@ -140,19 +139,6 @@ public class DefaultAdaptor extends AbstractJdbcAdaptor {
return sql;
}
- private boolean checkSqlContainstable(String orig, String table) {
- // ensure table is single match(e.g match account but not match accountant)
- if (orig.endsWith(table.toUpperCase(Locale.ROOT))) {
- return true;
- }
- for (String end:POSSIBLE_TALBE_END) {
- if (orig.contains(table.toUpperCase(Locale.ROOT) + end)){
- return true;
- }
- }
- return false;
- }
-
/**
* By default, use schema as database of kylin.
* @return
@@ -270,26 +256,100 @@ public class DefaultAdaptor extends AbstractJdbcAdaptor {
public String fixIdentifierCaseSensitve(String identifier) {
try {
List<String> databases = listDatabasesWithCache();
- for (String db : databases) {
- if (db.equalsIgnoreCase(identifier)) {
- return db;
+ for (String database : databases) {
+ if (identifier.equalsIgnoreCase(database)) {
+ return database;
}
- List<String> tables = listTablesWithCache(db);
- for (String table : tables) {
- if (table.equalsIgnoreCase(identifier)) {
- return table;
- }
- List<String> cols = listColumnsWithCache(db, table);
- for (String col : cols) {
- if (col.equalsIgnoreCase(identifier)) {
- return col;
- }
- }
+ }
+ List<String> tables = listTables();
+ for (String table : tables) {
+ if (identifier.equalsIgnoreCase(table)) {
+ return table;
+ }
+ }
+ List<String> columns = listColumns();
+ for (String column : columns) {
+ if (identifier.equalsIgnoreCase(column)) {
+ return column;
}
}
} catch (Exception e) {
- throw new RuntimeException(e);
+ throw new IllegalStateException(e);
}
return identifier;
}
+
+ /**
+ * Get All tables for sql case sensitive
+ * @return
+ * @throws SQLException
+ */
+ public List<String> listTables() throws SQLException {
+ List<String> ret = new ArrayList<>();
+ if (tablesCache == null || tablesCache.size() == 0) {
+ try (Connection conn = getConnection();
+ ResultSet rs = conn.getMetaData().getTables(null, null, null, null)) {
+ while (rs.next()) {
+ String name = rs.getString("TABLE_NAME");
+ String database = rs.getString("TABLE_SCHEM") != null ? rs.getString("TABLE_SCHEM")
+ : rs.getString("TABLE_CAT");
+ String cacheKey = joiner.join(config.datasourceId, config.url, database, "tables");
+ List<String> cachedTables = tablesCache.getIfPresent(cacheKey);
+ if (cachedTables == null) {
+ cachedTables = new ArrayList<>();
+ tablesCache.put(cacheKey, cachedTables);
+ logger.debug("Add table cache for database {}", database);
+ }
+ if (!cachedTables.contains(name)) {
+ cachedTables.add(name);
+ }
+ ret.add(name);
+ }
+ }
+ } else {
+ for (Map.Entry<String, List<String>> entry : tablesCache.asMap().entrySet()) {
+ ret.addAll(entry.getValue());
+ }
+ }
+
+ return ret;
+ }
+
+ /**
+ * Get All columns for sql case sensitive
+ * @return
+ * @throws SQLException
+ */
+ public List<String> listColumns() throws SQLException {
+ List<String> ret = new ArrayList<>();
+ if (columnsCache == null || columnsCache.size() == 0) {
+ CachedRowSet columnsRs = null;
+ try (Connection conn = getConnection();
+ ResultSet rs = conn.getMetaData().getColumns(null, null, null, null)) {
+ columnsRs = cacheResultSet(rs);
+ }
+ while (columnsRs.next()) {
+ String database = columnsRs.getString("TABLE_SCHEM") != null ? columnsRs.getString("TABLE_SCHEM")
+ : columnsRs.getString("TABLE_CAT");
+ String table = columnsRs.getString("TABLE_NAME");
+ String column = columnsRs.getString("COLUMN_NAME");
+ String cacheKey = joiner.join(config.datasourceId, config.url, database, table, "columns");
+ List<String> cachedColumns = columnsCache.getIfPresent(cacheKey);
+ if (cachedColumns == null) {
+ cachedColumns = new ArrayList<>();
+ columnsCache.put(cacheKey, cachedColumns);
+ logger.debug("Add column cache for table {}.{}", database, table);
+ }
+ if (!cachedColumns.contains(column)) {
+ cachedColumns.add(column);
+ }
+ ret.add(column);
+ }
+ } else {
+ for (Map.Entry<String, List<String>> entry : columnsCache.asMap().entrySet()) {
+ ret.addAll(entry.getValue());
+ }
+ }
+ return ret;
+ }
}
\ No newline at end of file
diff --git a/datasource-sdk/src/main/resources/datasource/mysql.xml b/datasource-sdk/src/main/resources/datasource/mysql.xml
index d31c11e..5e23cc3 100644
--- a/datasource-sdk/src/main/resources/datasource/mysql.xml
+++ b/datasource-sdk/src/main/resources/datasource/mysql.xml
@@ -23,15 +23,8 @@
<PROPERTY NAME="sql.allow-no-orderby-with-fetch" VALUE="true"/>
<PROPERTY NAME="sql.keyword-default-escape" VALUE="true"/>
<PROPERTY NAME="sql.keyword-default-uppercase" VALUE="true"/>
- <PROPERTY NAME="table-sampling.template.max-or-min-value-default"
- VALUE="SELECT MAX({0}), MIN({0}) FROM {1} WHERE {0} IS NOT NULL"/>
- <PROPERTY NAME="table-sampling.template.max-or-min-len-value-default"
- VALUE="SELECT {0} FROM {1} WHERE {0} IS NOT NULL ORDER BY CHAR_LENGTH({0}){2} LIMIT 1"/>
- <PROPERTY NAME="table-sampling.template.exceed-precision-count-default"
- VALUE="SELECT COUNT({0}) FROM {1} WHERE CHAR_LENGTH({0}) > {2}"/>
- <PROPERTY NAME="table-sampling.template.exceed-precision-max-length-value-default"
- VALUE="SELECT MAX(CHAR_LENGTH({0})) FROM {1} WHERE CHAR_LENGTH({0}) > {2}"/>
<PROPERTY NAME="sql.case-sensitive" VALUE="true"/>
+ <PROPERTY NAME="metadata.enable-cache" VALUE="true"/>
<PROPERTY NAME="sql.paging-type" VALUE="LIMIT_OFFSET"/>