You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2018/01/30 09:38:51 UTC
[1/2] kylin git commit: KYLIN-3204 fix potentially unclosed resources
Repository: kylin
Updated Branches:
refs/heads/master 4c31b1d4a -> fee573025
KYLIN-3204 fix potentially unclosed resources
Signed-off-by: lidongsjtu <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b07a0566
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b07a0566
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b07a0566
Branch: refs/heads/master
Commit: b07a05666d5474a9089fcd38219ffa204809ba10
Parents: 4c31b1d
Author: etherge <et...@163.com>
Authored: Tue Jan 30 17:07:27 2018 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Tue Jan 30 17:23:32 2018 +0800
----------------------------------------------------------------------
.../apache/kylin/source/jdbc/JdbcExplorer.java | 47 +++++++++++---------
.../org/apache/kylin/source/jdbc/SqlUtil.java | 11 ++---
2 files changed, 28 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/b07a0566/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java b/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java
index c34597a..2dd8f9a 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java
@@ -18,7 +18,6 @@
package org.apache.kylin.source.jdbc;
-import java.io.IOException;
import java.sql.Connection;
import java.sql.DatabaseMetaData;
import java.sql.ResultSet;
@@ -29,7 +28,6 @@ import java.util.List;
import java.util.UUID;
import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.hive.ql.CommandNeedRetryException;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.DBUtils;
import org.apache.kylin.common.util.Pair;
@@ -106,7 +104,6 @@ public class JdbcExplorer implements ISourceMetadataExplorer, ISampleDataDeploye
DBUtils.closeQuietly(con);
}
-
TableExtDesc tableExtDesc = new TableExtDesc();
tableExtDesc.setIdentity(tableDesc.getIdentity());
tableExtDesc.setUuid(UUID.randomUUID().toString());
@@ -208,20 +205,23 @@ public class JdbcExplorer implements ISourceMetadataExplorer, ISampleDataDeploye
return new String[] { dropView, dropTable, createSql };
}
- private void executeSQL(String sql) throws CommandNeedRetryException, IOException, SQLException {
+ private void executeSQL(String sql) throws SQLException {
Connection con = SqlUtil.getConnection(dbconf);
- logger.info(String.format(sql));
- SqlUtil.execUpdateSQL(con, sql);
- DBUtils.closeQuietly(con);
+ logger.info(String.format("Executing sql : %s"), sql);
+ try {
+ SqlUtil.execUpdateSQL(con, sql);
+ } finally {
+ DBUtils.closeQuietly(con);
+ }
}
- private void executeSQL(String[] sqls) throws CommandNeedRetryException, IOException, SQLException {
- Connection con = SqlUtil.getConnection(dbconf);
- for (String sql : sqls) {
- logger.info(String.format(sql));
- SqlUtil.execUpdateSQL(con, sql);
+ private void executeSQL(String[] sqls) throws SQLException {
+ try (Connection con = SqlUtil.getConnection(dbconf)) {
+ for (String sql : sqls) {
+ logger.info(String.format("Executing sql : %s"), sql);
+ SqlUtil.execUpdateSQL(con, sql);
+ }
}
- DBUtils.closeQuietly(con);
}
@Override
@@ -237,32 +237,35 @@ public class JdbcExplorer implements ISourceMetadataExplorer, ISampleDataDeploye
KylinConfig config = KylinConfig.getInstanceFromEnv();
String tmpDatabase = config.getHiveDatabaseForIntermediateTable();
- String tmpView = tmpDatabase + ".kylin_eval_query_" + UUID.nameUUIDFromBytes(query.getBytes()).toString().replaceAll("-", "");
+ String tmpView = tmpDatabase + ".kylin_eval_query_"
+ + UUID.nameUUIDFromBytes(query.getBytes()).toString().replaceAll("-", "");
String dropViewSql = "DROP VIEW IF EXISTS " + tmpView;
String evalViewSql = "CREATE VIEW " + tmpView + " as " + query;
+ Connection con = null;
+ ResultSet rs = null;
try {
logger.debug("Removing duplicate view {}", tmpView);
executeSQL(dropViewSql);
logger.debug("Creating view {} for query: {}", tmpView, query);
executeSQL(evalViewSql);
logger.debug("Evaluating query columns' metadata");
- Connection con = SqlUtil.getConnection(dbconf);
+ con = SqlUtil.getConnection(dbconf);
DatabaseMetaData dbmd = con.getMetaData();
- ResultSet rs = dbmd.getColumns(null, tmpDatabase, tmpView, null);
+ rs = dbmd.getColumns(null, tmpDatabase, tmpView, null);
ColumnDesc[] result = extractColumnFromMeta(rs);
- DBUtils.closeQuietly(rs);
- DBUtils.closeQuietly(con);
return result;
- } catch (Exception e) {
+ } catch (SQLException e) {
throw new RuntimeException("Cannot evaluate metadata of query: " + query, e);
} finally {
+ DBUtils.closeQuietly(con);
+ DBUtils.closeQuietly(rs);
try {
- logger.debug("Cleaning up.");
+ logger.debug("Cleaning up temp view.");
executeSQL(dropViewSql);
- } catch (Exception e) {
- logger.warn("Cannot drop temp view of query: {}", query, e);
+ } catch (SQLException e) {
+ logger.warn("Failed to clean up temp view of query: {}", query, e);
}
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b07a0566/source-hive/src/main/java/org/apache/kylin/source/jdbc/SqlUtil.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/jdbc/SqlUtil.java b/source-hive/src/main/java/org/apache/kylin/source/jdbc/SqlUtil.java
index 715bb99..f86fd7e 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/jdbc/SqlUtil.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/jdbc/SqlUtil.java
@@ -20,6 +20,7 @@ package org.apache.kylin.source.jdbc;
import java.sql.Connection;
import java.sql.DriverManager;
+import java.sql.SQLException;
import java.sql.Statement;
import java.sql.Types;
import java.util.Random;
@@ -50,15 +51,9 @@ public class SqlUtil {
}
}
- public static void execUpdateSQL(Connection db, String sql) {
- Statement statement = null;
- try {
- statement = db.createStatement();
+ public static void execUpdateSQL(Connection db, String sql) throws SQLException {
+ try (Statement statement = db.createStatement()) {
statement.executeUpdate(sql);
- } catch (Exception e) {
- logger.error("", e);
- } finally {
- closeResources(null, statement);
}
}
[2/2] kylin git commit: KYLIN-3204 minor refine on loggings
Posted by li...@apache.org.
KYLIN-3204 minor refine on loggings
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/fee57302
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/fee57302
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/fee57302
Branch: refs/heads/master
Commit: fee5730258a5082999bcf8d3141aa4cb179b693c
Parents: b07a056
Author: lidongsjtu <li...@apache.org>
Authored: Tue Jan 30 17:28:58 2018 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Tue Jan 30 17:28:58 2018 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/source/jdbc/JdbcExplorer.java | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/fee57302/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java b/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java
index 2dd8f9a..d858cd4 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java
@@ -135,7 +135,7 @@ public class JdbcExplorer implements ISourceMetadataExplorer, ISampleDataDeploye
return String.format("IF NOT EXISTS (SELECT name FROM sys.schemas WHERE name = N'%s') EXEC('CREATE SCHEMA"
+ " [%s] AUTHORIZATION [dbo]')", schemaName, schemaName);
} else {
- logger.error(String.format("unsupported dialect %s.", dialect));
+ logger.error("unsupported dialect {}.", dialect);
return null;
}
}
@@ -156,7 +156,7 @@ public class JdbcExplorer implements ISourceMetadataExplorer, ISampleDataDeploye
return String.format("BULK INSERT %s FROM '%s/%s.csv' WITH(FIELDTERMINATOR = ',')", tableName, tableFileDir,
tableName);
} else {
- logger.error(String.format("unsupported dialect %s.", dialect));
+ logger.error("unsupported dialect {}.", dialect);
return null;
}
}
@@ -167,7 +167,7 @@ public class JdbcExplorer implements ISourceMetadataExplorer, ISampleDataDeploye
}
private String[] generateCreateTableSql(TableDesc tableDesc) {
- logger.info(String.format("gen create table sql:%s", tableDesc));
+ logger.info("Generate create table sql: {}", tableDesc);
String tableIdentity = String.format("%s.%s", tableDesc.getDatabase().toUpperCase(), tableDesc.getName())
.toUpperCase();
String dropsql = "DROP TABLE IF EXISTS " + tableIdentity;
@@ -207,7 +207,7 @@ public class JdbcExplorer implements ISourceMetadataExplorer, ISampleDataDeploye
private void executeSQL(String sql) throws SQLException {
Connection con = SqlUtil.getConnection(dbconf);
- logger.info(String.format("Executing sql : %s"), sql);
+ logger.info("Executing sql : {}", sql);
try {
SqlUtil.execUpdateSQL(con, sql);
} finally {
@@ -218,7 +218,7 @@ public class JdbcExplorer implements ISourceMetadataExplorer, ISampleDataDeploye
private void executeSQL(String[] sqls) throws SQLException {
try (Connection con = SqlUtil.getConnection(dbconf)) {
for (String sql : sqls) {
- logger.info(String.format("Executing sql : %s"), sql);
+ logger.info("Executing sql : {}", sql);
SqlUtil.execUpdateSQL(con, sql);
}
}