You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2015/10/26 10:08:08 UTC
[22/45] incubator-kylin git commit: KYLIN-742 Minor refactoring
KYLIN-742 Minor refactoring
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/143aa608
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/143aa608
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/143aa608
Branch: refs/heads/master
Commit: 143aa608b88c0ceadeb11b38bf6b2dbaee105c5e
Parents: 7990438
Author: Li, Yang <ya...@ebay.com>
Authored: Tue Sep 29 17:41:53 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Tue Sep 29 17:42:35 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/KylinConfig.java | 43 +++---
.../test_case_data/sandbox/kylin.properties | 17 ++-
server/pom.xml | 11 +-
.../apache/kylin/rest/service/QueryService.java | 48 +++----
.../org/apache/kylin/rest/util/HiveReroute.java | 142 +++++++++++++++++++
.../apache/kylin/rest/util/HiveRerouteUtil.java | 133 -----------------
6 files changed, 195 insertions(+), 199 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/143aa608/common/src/main/java/org/apache/kylin/common/KylinConfig.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/KylinConfig.java b/common/src/main/java/org/apache/kylin/common/KylinConfig.java
index 72b6a65..14c0fb4 100644
--- a/common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -44,6 +44,13 @@ import com.google.common.collect.Sets;
* @author yangli9
*/
public class KylinConfig {
+
+ /*
+ * NO NEED TO DEFINE PUBLIC CONSTANTS FOR KEY NAMES!
+ *
+ * Kylin configuration is exposed as public methods. A client never need to access key names directly.
+ * Feel free to hard code key names, as long as they are encapsulated in this class. It reads better!
+ */
public static final String KYLIN_STORAGE_URL = "kylin.storage.url";
@@ -96,11 +103,6 @@ public class KylinConfig {
public static final String KYLIN_HBASE_CLUSTER_FS = "kylin.hbase.cluster.fs";
- public static final String HIVE_PASSWORD = "hive.password";
-
- public static final String HIVE_USER = "hive.user";
-
- public static final String HIVE_URL = "hive.url";
/**
* Kylin properties file
*/
@@ -133,15 +135,6 @@ public class KylinConfig {
public static final String HBASE_REGION_COUNT_MIN = "kylin.hbase.region.count.min";
public static final String HBASE_REGION_COUNT_MAX = "kylin.hbase.region.count.max";
- // property for Hive query rerouting enablement
- public static final String KYLIN_ROUTE_HIVE_ENABLED = "kylin.route.hive.enabled";
- public static final boolean KYLIN_ROUTE_HIVE_ENABLED_DEFAULT = false;
- // JDBC Hive connection details for query rerouting
- public static final String KYLIN_ROUTE_HIVE_URL = "kylin.route.hive.url";
- public static final String KYLIN_ROUTE_HIVE_URL_DEFAULT = "jdbc:hive2://sandbox:10000";
- public static final String KYLIN_ROUTE_HIVE_USERNAME = "kylin.route.hive.username";
- public static final String KYLIN_ROUTE_HIVE_PASSWORD = "kylin.route.hive.password";
-
// static cached instances
private static KylinConfig ENV_INSTANCE = null;
@@ -274,38 +267,42 @@ public class KylinConfig {
// ============================================================================
- // start: properties for Hive rerouting
public boolean isHiveReroutingEnabled() {
- return Boolean.parseBoolean(getOptional(KYLIN_ROUTE_HIVE_ENABLED));
+ return Boolean.parseBoolean(getOptional("kylin.route.hive.enabled", "false"));
}
public String getHiveRerouteUrl() {
- return getOptional(KYLIN_ROUTE_HIVE_URL, KYLIN_ROUTE_HIVE_URL_DEFAULT);
+ return getOptional("kylin.route.hive.url", "jdbc:hive2://");
}
public String getHiveRerouteUsername() {
- return getOptional(KYLIN_ROUTE_HIVE_USERNAME, "");
+ return getOptional("kylin.route.hive.username", "");
}
public String getHiveReroutePassword() {
- return getOptional(KYLIN_ROUTE_HIVE_PASSWORD, "");
+ return getOptional("kylin.route.hive.password", "");
}
- // end: properties for JDBC Hive rerouting
public String getStorageUrl() {
return storageUrl;
}
+ /** Use the hive reroute feature instead */
+ @Deprecated
public String getHiveUrl() {
- return getOptional(HIVE_URL, "");
+ return getOptional("hive.url", "");
}
+ /** Use the hive reroute feature instead */
+ @Deprecated
public String getHiveUser() {
- return getOptional(HIVE_USER, "");
+ return getOptional("hive.user", "");
}
+ /** Use the hive reroute feature instead */
+ @Deprecated
public String getHivePassword() {
- return getOptional(HIVE_PASSWORD, "");
+ return getOptional("hive.password", "");
}
public String getHdfsWorkingDirectory() {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/143aa608/examples/test_case_data/sandbox/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/kylin.properties b/examples/test_case_data/sandbox/kylin.properties
index 6fe63a8..fa1e4de 100644
--- a/examples/test_case_data/sandbox/kylin.properties
+++ b/examples/test_case_data/sandbox/kylin.properties
@@ -73,6 +73,7 @@ kylin.hbase.region.count.min=1
kylin.hbase.region.count.min=500
## Config for Restful APP ##
+
# database connection settings:
ldap.server=
ldap.username=
@@ -88,6 +89,13 @@ acl.defaultRole=
ganglia.group=
ganglia.port=8664
+# route to hive settings
+kylin.route.hive.enabled=false
+# default to the embedded server (jdbc:hive2://)
+#kylin.route.hive.url=
+#kylin.route.hive.username=
+#kylin.route.hive.password=
+
## Config for mail service
# If true, will send email notification;
@@ -131,12 +139,3 @@ kylin.monitor.ext.log.base.dir = /tmp/kylin_log1,/tmp/kylin_log2
#will create external hive table to query result csv file
#will set to kylin_query_log by default if not config here
kylin.monitor.query.log.parse.result.table = kylin_query_log
-
-##### Config for enabling query rerouting to Hive. Disabled by default. ######
-kylin.route.hive.enabled=false
-
-# If query rerouting is enabled, provide the hive configurations
-# Default value for kylin.route.hive.url will be pointing to the embedded server (jdbc:hive2://)
-kylin.route.hive.url=
-kylin.route.hive.username=
-kylin.route.hive.password=
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/143aa608/server/pom.xml
----------------------------------------------------------------------
diff --git a/server/pom.xml b/server/pom.xml
index adeb5ba..eed7068 100644
--- a/server/pom.xml
+++ b/server/pom.xml
@@ -94,6 +94,11 @@
</dependency>
<dependency>
+ <groupId>net.sf.opencsv</groupId>
+ <artifactId>opencsv</artifactId>
+ </dependency>
+
+ <dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
<version>0.5.0.M6</version>
@@ -360,10 +365,10 @@
</exclusion>
</exclusions>
</dependency>
-
<dependency>
- <groupId>net.sf.opencsv</groupId>
- <artifactId>opencsv</artifactId>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-jdbc</artifactId>
+ <scope>provided</scope>
</dependency>
<!-- Tomcat Env -->
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/143aa608/server/src/main/java/org/apache/kylin/rest/service/QueryService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 81905c8..8a397cd 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -40,9 +40,6 @@ import java.util.Set;
import javax.sql.DataSource;
-import org.apache.commons.lang.exception.ExceptionUtils;
-import org.apache.calcite.sql.parser.impl.ParseException;
-import org.apache.calcite.sql.validate.SqlValidatorException;
import org.apache.calcite.avatica.ColumnMetaData.Rep;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.hbase.client.Get;
@@ -65,7 +62,7 @@ import org.apache.kylin.rest.model.TableMeta;
import org.apache.kylin.rest.request.PrepareSqlRequest;
import org.apache.kylin.rest.request.SQLRequest;
import org.apache.kylin.rest.response.SQLResponse;
-import org.apache.kylin.rest.util.HiveRerouteUtil;
+import org.apache.kylin.rest.util.HiveReroute;
import org.apache.kylin.rest.util.QueryUtil;
import org.apache.kylin.rest.util.Serializer;
import org.slf4j.Logger;
@@ -102,7 +99,7 @@ public class QueryService extends BasicService {
tableNameBase = cut < 0 ? DEFAULT_TABLE_PREFIX : metadataUrl.substring(0, cut);
hbaseUrl = cut < 0 ? metadataUrl : metadataUrl.substring(cut + 1);
userTableName = tableNameBase + USER_TABLE_NAME;
-
+
badQueryDetector.start();
}
@@ -113,9 +110,9 @@ public class QueryService extends BasicService {
public SQLResponse query(SQLRequest sqlRequest) throws Exception {
try {
badQueryDetector.queryStart(Thread.currentThread(), sqlRequest);
-
+
return queryWithSqlMassage(sqlRequest);
-
+
} finally {
badQueryDetector.queryEnd(Thread.currentThread());
}
@@ -373,33 +370,22 @@ public class QueryService extends BasicService {
oneRow.add((resultSet.getString(i + 1)));
}
- results.add(new LinkedList<String>(oneRow));
+ results.add(new ArrayList<String>(oneRow));
oneRow.clear();
}
} catch (SQLException sqlException) {
- // unsuccessful statement execution, retry with Hive on Spark. Code modification as part of the jira https://issues.apache.org/jira/browse/KYLIN-742
- boolean isExpectedCause = (ExceptionUtils.getRootCause(sqlException).getClass().equals(SqlValidatorException.class)) || (ExceptionUtils.getRootCause(sqlException).getClass().equals(ParseException.class));
- KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
- if (isExpectedCause && kylinConfig.isHiveReroutingEnabled()) {
- logger.debug("query rerouting option enabled for Kylin");
- // running query to hive
- HiveRerouteUtil rerouteUtil = new HiveRerouteUtil();
- try {
- conn = rerouteUtil.createConnection(kylinConfig.getHiveRerouteUrl(), kylinConfig.getHiveRerouteUsername(), kylinConfig.getHiveReroutePassword());
- resultSet = rerouteUtil.executQuery(conn, sql);
- columnMetas = rerouteUtil.extractColumnMetadata(resultSet, columnMetas);
- results = rerouteUtil.extractResults(resultSet, results);
- } catch (Exception exception) {
- logger.error("exception in re-routing the query to hive", exception);
- throw exception;
- } finally {
- rerouteUtil.closeConnection(conn);
- }
- } else {
- logger.error("exception in running the query: " + sql);
- throw sqlException;
- }
- } finally {
+ // hive maybe able to answer where kylin failed
+ HiveReroute reroute = new HiveReroute();
+ if (reroute.shouldReroute(sqlException) == false) {
+ throw sqlException;
+ }
+ try {
+ reroute.query(sql, results, columnMetas);
+ } catch (Throwable ex) {
+ logger.error("Reroute hive failed", ex);
+ throw sqlException;
+ }
+ } finally {
close(resultSet, stat, conn);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/143aa608/server/src/main/java/org/apache/kylin/rest/util/HiveReroute.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/util/HiveReroute.java b/server/src/main/java/org/apache/kylin/rest/util/HiveReroute.java
new file mode 100644
index 0000000..c83afb2
--- /dev/null
+++ b/server/src/main/java/org/apache/kylin/rest/util/HiveReroute.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.rest.util;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.ResultSetMetaData;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.calcite.sql.parser.impl.ParseException;
+import org.apache.calcite.sql.validate.SqlValidatorException;
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.rest.model.SelectedColumnMeta;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class HiveReroute {
+
+ private static final Logger logger = LoggerFactory.getLogger(HiveReroute.class);
+
+ public static final String HIVE_DRIVER_CLASS = "org.apache.hive.jdbc.HiveDriver";
+
+ public boolean shouldReroute(SQLException sqlException) {
+ Throwable rootCause = ExceptionUtils.getRootCause(sqlException);
+ boolean isExpectedCause = (rootCause instanceof SqlValidatorException || rootCause instanceof ParseException);
+ KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+ return isExpectedCause && kylinConfig.isHiveReroutingEnabled();
+ }
+
+ public void query(String sql, List<List<String>> results, List<SelectedColumnMeta> columnMetas) throws Exception {
+ KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+ logger.debug("query rerouting option enabled for Kylin");
+
+ // running query to hive
+ Connection conn = null;
+ ResultSet resultSet = null;
+ try {
+ conn = createConnection(kylinConfig.getHiveRerouteUrl(), kylinConfig.getHiveRerouteUsername(), kylinConfig.getHiveReroutePassword());
+ resultSet = executQuery(conn, sql);
+ extractColumnMetadata(resultSet, columnMetas);
+ extractResults(resultSet, results);
+ } finally {
+ closeConnection(conn);
+ }
+ }
+
+ /**
+ * Create a connection to the Hive server by passing the required connection parameters.
+ * @param connectionURL: JDBC URL to connect to the Hive server
+ * @param username: Username to connect with (optional)
+ * @param password: Password to connect with (optional)
+ * @return: Connection object to the Hive server
+ * @throws Exception
+ */
+ private Connection createConnection(String connectionURL, String username, String password) throws Exception {
+ logger.info("rerouting to : " + connectionURL + " for executing the query");
+
+ Class.forName(HIVE_DRIVER_CLASS);
+ Connection connection = DriverManager.getConnection(connectionURL, username, password);
+ return connection;
+ }
+
+ /**
+ * Close the connection to the Hive server.
+ * @param connection: Connection object to be closed
+ */
+ private void closeConnection(Connection connection) {
+ if (null != connection) {
+ try {
+ connection.close();
+ } catch (SQLException sqlException) {
+ logger.error("failed to close connection", sqlException);
+ }
+ }
+ }
+
+ /**
+ * Execute a query in Hive.
+ * @param connection: Connection object to the Hive server
+ * @param query: Query to be executed
+ * @return: ResultSet object of the query executed
+ * @throws Exception
+ */
+ private ResultSet executQuery(Connection connection, String query) throws Exception {
+ Statement statement = null;
+ ResultSet resultSet = null;
+
+ statement = connection.createStatement();
+ resultSet = statement.executeQuery(query);
+ return resultSet;
+ }
+
+ private void extractColumnMetadata(ResultSet resultSet, List<SelectedColumnMeta> columnMetas) throws SQLException {
+ ResultSetMetaData metaData = null;
+ int columnCount = 0;
+
+ metaData = resultSet.getMetaData();
+ columnCount = metaData.getColumnCount();
+
+ // fill in selected column meta
+ for (int i = 1; i <= columnCount; ++i) {
+ columnMetas.add(new SelectedColumnMeta(metaData.isAutoIncrement(i), metaData.isCaseSensitive(i), false, metaData.isCurrency(i), metaData.isNullable(i), false, metaData.getColumnDisplaySize(i), metaData.getColumnLabel(i), metaData.getColumnName(i), null, null, null, metaData.getPrecision(i), metaData.getScale(i), metaData.getColumnType(i), metaData.getColumnTypeName(i), metaData.isReadOnly(i), false, false));
+ }
+ }
+
+ private void extractResults(ResultSet resultSet, List<List<String>> results) throws SQLException {
+ List<String> oneRow = new LinkedList<String>();
+
+ while (resultSet.next()) {
+ //logger.debug("resultSet value: " + resultSet.getString(1));
+ for (int i = 0; i < resultSet.getMetaData().getColumnCount(); i++) {
+ oneRow.add((resultSet.getString(i + 1)));
+ }
+
+ results.add(new ArrayList<String>(oneRow));
+ oneRow.clear();
+ }
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/143aa608/server/src/main/java/org/apache/kylin/rest/util/HiveRerouteUtil.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/util/HiveRerouteUtil.java b/server/src/main/java/org/apache/kylin/rest/util/HiveRerouteUtil.java
deleted file mode 100644
index ca1e048..0000000
--- a/server/src/main/java/org/apache/kylin/rest/util/HiveRerouteUtil.java
+++ /dev/null
@@ -1,133 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.rest.util;
-
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.ResultSet;
-import java.sql.ResultSetMetaData;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.util.LinkedList;
-import java.util.List;
-
-import org.apache.kylin.rest.model.SelectedColumnMeta;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/** @author murkrishn **/
-public class HiveRerouteUtil {
-
- private static final Logger logger = LoggerFactory.getLogger(HiveRerouteUtil.class);
- public static final String driverName = "org.apache.hive.jdbc.HiveDriver";
-
- /**
- * Create a connection to the Hive server by passing the required connection parameters.
- * @param connectionURL: JDBC URL to connect to the Hive server
- * @param username: Username to connect with (optional)
- * @param password: Password to connect with (optional)
- * @return: Connection object to the Hive server
- * @throws Exception
- */
- public Connection createConnection (String connectionURL, String username, String password) throws Exception {
- logger.info("rerouting to : " + connectionURL + " for executing the query");
-
- try {
- Class.forName(driverName);
- } catch (ClassNotFoundException classNotFoundException) {
- throw classNotFoundException;
- }
-
- Connection connection = DriverManager.getConnection(connectionURL, username, password);
- return connection;
- }
-
- /**
- * Close the connection to the Hive server.
- * @param connection: Connection object to be closed
- */
- public void closeConnection(Connection connection) {
- if (null != connection) {
- try {
- connection.close();
- } catch (SQLException sqlException) {
- logger.error("failed to close connection", sqlException);
- }
- }
- }
-
- /**
- * Execute a query in Hive.
- * @param connection: Connection object to the Hive server
- * @param query: Query to be executed
- * @return: ResultSet object of the query executed
- * @throws Exception
- */
- public ResultSet executQuery (Connection connection, String query) throws Exception {
- Statement statement = null;
- ResultSet resultSet = null;
-
- try {
- statement = connection.createStatement();
- resultSet = statement.executeQuery(query);
- return resultSet;
- } catch (SQLException sqlException) {
- throw sqlException;
- }
- }
-
- public List<SelectedColumnMeta> extractColumnMetadata (ResultSet resultSet, List<SelectedColumnMeta> columnMetas) throws SQLException {
- ResultSetMetaData metaData = null;
- int columnCount = 0;
-
- try {
- metaData = resultSet.getMetaData();
- columnCount = metaData.getColumnCount();
-
- // fill in selected column meta
- for (int i = 1; i <= columnCount; ++i) {
- columnMetas.add(new SelectedColumnMeta(metaData.isAutoIncrement(i), metaData.isCaseSensitive(i), false, metaData.isCurrency(i), metaData.isNullable(i), false, metaData.getColumnDisplaySize(i), metaData.getColumnLabel(i), metaData.getColumnName(i), null, null, null, metaData.getPrecision(i), metaData.getScale(i), metaData.getColumnType(i), metaData.getColumnTypeName(i), metaData.isReadOnly(i), false, false));
- }
-
- return columnMetas;
- } catch (SQLException sqlException) {
- throw sqlException;
- }
- }
-
- public List<List<String>> extractResults (ResultSet resultSet, List<List<String>> results) throws SQLException {
- List<String> oneRow = new LinkedList<String>();
-
- try {
- while (resultSet.next()) {
- //logger.debug("resultSet value: " + resultSet.getString(1));
- for (int i = 0; i < resultSet.getMetaData().getColumnCount(); i++) {
- oneRow.add((resultSet.getString(i + 1)));
- }
-
- results.add(new LinkedList<String>(oneRow));
- oneRow.clear();
- }
-
- return results;
- } catch (SQLException sqlException) {
- throw sqlException;
- }
- }
-}
\ No newline at end of file