You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by lu...@apache.org on 2015/09/06 10:00:07 UTC
[43/50] [abbrv] incubator-kylin git commit: KYLIN-792,
support config mapreduce.queue for hivesql in monitor module
KYLIN-792, support config mapreduce.queue for hivesql in monitor module
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/25cbb2c9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/25cbb2c9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/25cbb2c9
Branch: refs/heads/0.7
Commit: 25cbb2c9dbd7c4d628874b5efe3aebbf5caf3acd
Parents: ebf4629
Author: jiazhong <ji...@ebay.com>
Authored: Fri Aug 28 17:36:03 2015 +0800
Committer: Luke Han <lu...@apache.org>
Committed: Sun Sep 6 14:38:00 2015 +0800
----------------------------------------------------------------------
bin/performance-monitor.sh | 2 +-
.../java/org/apache/kylin/monitor/Client.java | 2 +-
.../org/apache/kylin/monitor/ConfigUtils.java | 8 ++-
.../org/apache/kylin/monitor/DebugClient.java | 2 +-
.../org/apache/kylin/monitor/FileUtils.java | 4 +-
.../apache/kylin/monitor/HiveJdbcClient.java | 52 +++++++++++++++++++-
.../kylin/monitor/MonitorMetaManager.java | 2 +-
7 files changed, 63 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/25cbb2c9/bin/performance-monitor.sh
----------------------------------------------------------------------
diff --git a/bin/performance-monitor.sh b/bin/performance-monitor.sh
index 4d30364..d8421e3 100644
--- a/bin/performance-monitor.sh
+++ b/bin/performance-monitor.sh
@@ -30,7 +30,7 @@
source ${dir}/find-hive-dependency.sh
source ${dir}/find-hbase-dependency.sh
- export HBASE_CLASSPATH=$hive_dependency:${HBASE_CLASSPATH}
+ export HBASE_CLASSPATH=$hive_dependency:${HBASE_CLASSPATH}:$KYLIN_HOME/conf
_monitorjar=`ls ${KYLIN_HOME}/lib |grep kylin-monitor`
hbase \
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/25cbb2c9/monitor/src/main/java/org/apache/kylin/monitor/Client.java
----------------------------------------------------------------------
diff --git a/monitor/src/main/java/org/apache/kylin/monitor/Client.java b/monitor/src/main/java/org/apache/kylin/monitor/Client.java
index 699e0de..9c881b2 100644
--- a/monitor/src/main/java/org/apache/kylin/monitor/Client.java
+++ b/monitor/src/main/java/org/apache/kylin/monitor/Client.java
@@ -24,7 +24,7 @@ import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
/**
- * Created by jiazhong on 2015/5/7.
+ * @author jiazhong
*/
public class Client {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/25cbb2c9/monitor/src/main/java/org/apache/kylin/monitor/ConfigUtils.java
----------------------------------------------------------------------
diff --git a/monitor/src/main/java/org/apache/kylin/monitor/ConfigUtils.java b/monitor/src/main/java/org/apache/kylin/monitor/ConfigUtils.java
index 93cb00f..7cff4d7 100644
--- a/monitor/src/main/java/org/apache/kylin/monitor/ConfigUtils.java
+++ b/monitor/src/main/java/org/apache/kylin/monitor/ConfigUtils.java
@@ -34,7 +34,7 @@ import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
/**
- * Created by jiazhong on 2015/4/28.
+ * @author jiazhong
*/
public class ConfigUtils {
@@ -68,6 +68,8 @@ public class ConfigUtils {
public static final String HIVE_JDBC_CON_USERNAME = "kylin.hive.jdbc.connection.username";
public static final String HIVE_JDBC_CON_PASSWD = "kylin.hive.jdbc.connection.password";
+ public static final String KYLIN_MAP_JOB_QUEUE = "mapred.job.queue.name";
+
public static final String DEPLOY_ENV = "deploy.env";
public static final String HIVE_JDBC_CON_URL = "kylin.hive.jdbc.connection.url";
@@ -169,6 +171,10 @@ public class ConfigUtils {
return this.monitorConfig.getProperty(KYLIN_METADATA_URL);
}
+ public String getKylinMapJobQueue(){
+ return this.monitorConfig.getProperty(KYLIN_MAP_JOB_QUEUE);
+ }
+
public String getMetadataUrlPrefix() {
String hbaseMetadataUrl = getMetadataUrl();
String defaultPrefix = "kylin_metadata";
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/25cbb2c9/monitor/src/main/java/org/apache/kylin/monitor/DebugClient.java
----------------------------------------------------------------------
diff --git a/monitor/src/main/java/org/apache/kylin/monitor/DebugClient.java b/monitor/src/main/java/org/apache/kylin/monitor/DebugClient.java
index 0ce4a47..3a683da 100644
--- a/monitor/src/main/java/org/apache/kylin/monitor/DebugClient.java
+++ b/monitor/src/main/java/org/apache/kylin/monitor/DebugClient.java
@@ -23,7 +23,7 @@ import java.io.File;
import org.apache.log4j.Logger;
/**
- * Created by jiazhong on 2015/5/7
+ * @author jiazhong
*/
public class DebugClient {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/25cbb2c9/monitor/src/main/java/org/apache/kylin/monitor/FileUtils.java
----------------------------------------------------------------------
diff --git a/monitor/src/main/java/org/apache/kylin/monitor/FileUtils.java b/monitor/src/main/java/org/apache/kylin/monitor/FileUtils.java
index 625f2c7..f14ecdf 100644
--- a/monitor/src/main/java/org/apache/kylin/monitor/FileUtils.java
+++ b/monitor/src/main/java/org/apache/kylin/monitor/FileUtils.java
@@ -28,7 +28,7 @@ import org.apache.log4j.Logger;
import au.com.bytecode.opencsv.CSVWriter;
/**
- * Created by jiazhong on 2015/6/18.
+ * @author jiazhong
*/
public class FileUtils {
@@ -113,7 +113,7 @@ public class FileUtils {
*/
public static FileSystem getHdfsFileSystem() throws IOException {
Configuration conf = new Configuration();
- // conf.set("dfs.client.block.write.replace-datanode-on-failure.policy", "NEVER");
+ //conf.set("dfs.client.block.write.replace-datanode-on-failure.policy", "NEVER");
FileSystem fs = null;
try {
fs = FileSystem.newInstance(conf);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/25cbb2c9/monitor/src/main/java/org/apache/kylin/monitor/HiveJdbcClient.java
----------------------------------------------------------------------
diff --git a/monitor/src/main/java/org/apache/kylin/monitor/HiveJdbcClient.java b/monitor/src/main/java/org/apache/kylin/monitor/HiveJdbcClient.java
index ce34db5..185b717 100644
--- a/monitor/src/main/java/org/apache/kylin/monitor/HiveJdbcClient.java
+++ b/monitor/src/main/java/org/apache/kylin/monitor/HiveJdbcClient.java
@@ -19,6 +19,7 @@
package org.apache.kylin.monitor;
import java.io.IOException;
+import java.io.InputStream;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
@@ -29,9 +30,16 @@ import java.util.Calendar;
import org.apache.log4j.Logger;
import org.datanucleus.util.StringUtils;
+import org.w3c.dom.Document;
+import org.w3c.dom.NodeList;
+import org.xml.sax.SAXException;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
/**
- * Created by jiazhong on 2015/6/17.
+ * @author jiazhong
*/
public class HiveJdbcClient {
@@ -56,6 +64,10 @@ public class HiveJdbcClient {
final static Logger logger = Logger.getLogger(HiveJdbcClient.class);
+ final static String KYLIN_JOB_CONF_XML = "kylin_job_conf.xml";
+
+ static String MAP_QUEUE_NAME;
+
private static String driverName = "org.apache.hive.jdbc.HiveDriver";
private static ConfigUtils monitorConfig = ConfigUtils.getInstance();
@@ -79,13 +91,22 @@ public class HiveJdbcClient {
*/
public void start() throws SQLException, IOException {
+ MAP_QUEUE_NAME = getQueueName();
+
+ if(org.apache.commons.lang.StringUtils.isEmpty(MAP_QUEUE_NAME)){
+ MAP_QUEUE_NAME = monitorConfig.getKylinMapJobQueue();
+ }
+
+ logger.info("mapred.job.queue.name:"+MAP_QUEUE_NAME);
+
String CON_URL = monitorConfig.getHiveJdbcConUrl();
String USER_NAME = monitorConfig.getHiveJdbcConUserName();
String PASSWD = monitorConfig.getHiveJdbcConPasswd();
- Connection con = DriverManager.getConnection(CON_URL, USER_NAME, PASSWD);
+ Connection con = DriverManager.getConnection(CON_URL,USER_NAME,PASSWD);
Statement stmt = con.createStatement();
+ stmt.execute("set mapred.job.queue.name="+MAP_QUEUE_NAME);
ResultSet res = null;
SQL_GENERATE_QUERY_LOG_TABLE = generateQueryLogSql();
@@ -205,6 +226,33 @@ public class HiveJdbcClient {
return monthStasticSqlConvert(SQL_EACH_DAY_PERCENTILE);
}
+ public String getQueueName() throws IOException {
+ String queueName = "";
+ InputStream stream = this.getClass().getClassLoader().getResourceAsStream(KYLIN_JOB_CONF_XML);
+ if (stream!=null) {
+ DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
+ DocumentBuilder builder;
+ Document doc;
+ try {
+ builder = factory.newDocumentBuilder();
+ doc = builder.parse(stream);
+ NodeList nl = doc.getElementsByTagName("property");
+ for (int i = 0; i < nl.getLength(); i++) {
+ String name = doc.getElementsByTagName("name").item(i).getFirstChild().getNodeValue();
+ String value = doc.getElementsByTagName("value").item(i).getFirstChild().getNodeValue();
+ if (name.equals("mapreduce.job.queuename")) {
+ queueName = value;
+ }
+ }
+
+ } catch (ParserConfigurationException e) {
+ throw new IOException(e);
+ } catch (SAXException e) {
+ throw new IOException(e);
+ }
+ }
+ return queueName;
+ }
public String monthStasticSqlConvert(String sql) {
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd");
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/25cbb2c9/monitor/src/main/java/org/apache/kylin/monitor/MonitorMetaManager.java
----------------------------------------------------------------------
diff --git a/monitor/src/main/java/org/apache/kylin/monitor/MonitorMetaManager.java b/monitor/src/main/java/org/apache/kylin/monitor/MonitorMetaManager.java
index 0750436..97200fc 100644
--- a/monitor/src/main/java/org/apache/kylin/monitor/MonitorMetaManager.java
+++ b/monitor/src/main/java/org/apache/kylin/monitor/MonitorMetaManager.java
@@ -35,7 +35,7 @@ import org.apache.hadoop.hbase.util.Bytes;
import org.apache.log4j.Logger;
/**
- * Created by jiazhong on 2015/5/25.
+ * @author jiazhong
*/
public class MonitorMetaManager {