You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by zh...@apache.org on 2015/07/02 13:55:08 UTC
[3/4] incubator-kylin git commit: KYLIN-792 ,add performance module
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a935c7ba/server/src/main/java/org/apache/kylin/rest/service/PerformService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/PerformService.java b/server/src/main/java/org/apache/kylin/rest/service/PerformService.java
new file mode 100644
index 0000000..35b1207
--- /dev/null
+++ b/server/src/main/java/org/apache/kylin/rest/service/PerformService.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.rest.service;
+
+import au.com.bytecode.opencsv.CSVReader;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.kylin.common.KylinConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Component;
+
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.*;
+
+/**
+ * @author jiazhong
+ */
+@Component("performService")
+public class PerformService extends BasicService {
+
+ private static final Logger logger = LoggerFactory.getLogger(PerformService.class);
+
+ /*
+ * @return all query user
+ */
+ public List<String[]> getTotalQueryUser() throws IOException {
+ String filePath = KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory()+"/performance/metadata/total_query_user.csv";
+ List<String[]> res = readHdfsFile(filePath);
+ logger.info("Total Query User:"+res.get(0)[0]);
+ return res;
+ }
+
+ /*
+ * @return last 30 daily query num
+ */
+ public List<String[]> dailyQueryCount() throws IOException {
+ String filePath = KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory()+"/performance/metadata/last_30_daily_query_count.csv";
+ List<String[]> res = readHdfsFile(filePath);
+ return res;
+ }
+
+ /*
+ * @return average query count every day
+ */
+ public List<String[]> avgDayQuery() throws IOException {
+ String filePath = KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory()+"/performance/metadata/avg_day_query.csv";
+ List<String[]> res = readHdfsFile(filePath);
+ logger.info("Avg Day Query:"+res.get(0)[0]);
+ return res;
+ }
+
+ /*
+ *@return average latency every day
+ */
+ public List<String[]> last30DayPercentile() throws IOException {
+ String filePath = KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory()+"/performance/metadata/last_30_day_90_percentile_latency.csv";
+ List<String[]> res = readHdfsFile(filePath);
+ return res;
+ }
+
+ /*
+ *@return average latency for every cube
+ */
+ public List<String[]> eachDayPercentile() throws IOException {
+ String filePath = KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory()+"/performance/metadata/each_day_90_95_percentile_latency.csv";
+ List<String[]> res = readHdfsFile(filePath);
+ return res;
+ }
+
+ /*
+ *@return average latency for every cube
+ */
+ public List<String[]> projectPercentile() throws IOException {
+ String filePath = KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory()+"/performance/metadata/project_90_95_percentile_latency.csv";
+ List<String[]> res = readHdfsFile(filePath);
+ return res;
+ }
+
+
+ private List<String[]> readHdfsFile(String filePath) throws IOException {
+ List<String[]> allRows = null;
+ CSVReader reader = null;
+ FileSystem fs = null;
+ Configuration conf = new Configuration();
+
+ try {
+ fs = FileSystem.newInstance(conf);
+ FSDataInputStream inputStream = fs.open(new Path(filePath));
+ reader = new CSVReader(new InputStreamReader (inputStream),'|');
+
+ //Read all rows at once
+ allRows = reader.readAll();
+
+
+ } catch (IOException e) {
+ logger.info("failed to read hdfs file:",e);
+ }
+ finally {
+ fs.close();
+ }
+ return allRows;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a935c7ba/server/src/main/resources/kylinSecurity.xml
----------------------------------------------------------------------
diff --git a/server/src/main/resources/kylinSecurity.xml b/server/src/main/resources/kylinSecurity.xml
index d372240..22eea4c 100644
--- a/server/src/main/resources/kylinSecurity.xml
+++ b/server/src/main/resources/kylinSecurity.xml
@@ -34,6 +34,7 @@
<scr:intercept-url pattern="/api/cubes/src/tables" access="hasAnyRole('ROLE_ANALYST')" />
<scr:intercept-url pattern="/api/cubes*/**" access="isAuthenticated()" />
<scr:intercept-url pattern="/api/job*/**" access="isAuthenticated()" />
+ <scr:intercept-url pattern="/api/performance*/**" access="isAuthenticated()" />
<scr:intercept-url pattern="/api/admin/config" access="permitAll" />
<scr:intercept-url pattern="/api/projects" access="permitAll" />
<scr:intercept-url pattern="/api/admin*/**" access="hasRole('ROLE_ADMIN')" />
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a935c7ba/server/src/main/webapp/WEB-INF/web.xml
----------------------------------------------------------------------
diff --git a/server/src/main/webapp/WEB-INF/web.xml b/server/src/main/webapp/WEB-INF/web.xml
index d12f81a..703c815 100644
--- a/server/src/main/webapp/WEB-INF/web.xml
+++ b/server/src/main/webapp/WEB-INF/web.xml
@@ -14,10 +14,10 @@ limitations under the License. See accompanying LICENSE file.
-->
<web-app xmlns="http://java.sun.com/xml/ns/javaee"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://java.sun.com/xml/ns/javaee
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://java.sun.com/xml/ns/javaee
http://java.sun.com/xml/ns/javaee/web-app_2_5.xsd"
- version="2.5">
+ version="2.5">
<display-name>Kylin RESt Service</display-name>
@@ -37,10 +37,10 @@ limitations under the License. See accompanying LICENSE file.
<context-param>
<param-name>contextConfigLocation</param-name>
<param-value>
- classpath:applicationContext.xml
- classpath:kylinSecurity.xml
- classpath*:kylin-*-plugin.xml
- </param-value>
+ classpath:applicationContext.xml
+ classpath:kylinSecurity.xml
+ classpath*:kylin-*-plugin.xml
+ </param-value>
</context-param>
<listener>
@@ -50,46 +50,58 @@ limitations under the License. See accompanying LICENSE file.
<listener>
<listener-class>org.springframework.web.context.ContextLoaderListener</listener-class>
</listener>
-
+
+ <listener>
+ <listener-class>com.ryantenney.metrics.spring.servlets.MetricsServletsContextListener</listener-class>
+ </listener>
<listener>
- <listener-class>com.ryantenney.metrics.spring.servlets.MetricsServletsContextListener</listener-class>
+ <listener-class>org.apache.kylin.rest.metrics.KylinInstrumentedFilterContextListener</listener-class>
</listener>
+
<listener>
- <listener-class>org.apache.kylin.rest.metrics.KylinInstrumentedFilterContextListener</listener-class>
+ <listener-class>
+ org.springframework.security.web.session.HttpSessionEventPublisher
+ </listener-class>
</listener>
- <listener>
- <listener-class>
- org.springframework.security.web.session.HttpSessionEventPublisher
- </listener-class>
- </listener>
-
-<filter>
- <filter-name>CORS</filter-name>
- <filter-class>com.thetransactioncompany.cors.CORSFilter</filter-class>
- <init-param>
- <param-name>cors.supportedHeaders</param-name>
- <param-value>Authorization,Origin, No-Cache, X-Requested-With, If-Modified-Since, Pragma, Last-Modified, Cache-Control, Expires, Content-Type, X-E4M-With, Accept</param-value>
- </init-param>
- <init-param>
- <param-name>cors.supportedMethods</param-name>
- <param-value>GET, POST, PUT, DELETE, OPTIONS</param-value>
- </init-param>
- <init-param>
- <param-name>cors.supportsCredentials </param-name>
- <param-value>true</param-value>
- </init-param>
-</filter>
-
-<filter-mapping>
- <filter-name>CORS</filter-name>
- <url-pattern>/*</url-pattern>
-</filter-mapping>
-
- <!--
- Apply Spring Security Filter to all Requests
- -->
- <filter>
+ <filter>
+ <filter-name>CORS</filter-name>
+ <filter-class>com.thetransactioncompany.cors.CORSFilter</filter-class>
+ <init-param>
+ <param-name>cors.supportedHeaders</param-name>
+ <param-value>Authorization,Origin, No-Cache, X-Requested-With, If-Modified-Since, Pragma, Last-Modified,
+ Cache-Control, Expires, Content-Type, X-E4M-With, Accept
+ </param-value>
+ </init-param>
+ <init-param>
+ <param-name>cors.supportedMethods</param-name>
+ <param-value>GET, POST, PUT, DELETE, OPTIONS</param-value>
+ </init-param>
+ <init-param>
+ <param-name>cors.supportsCredentials</param-name>
+ <param-value>true</param-value>
+ </init-param>
+ </filter>
+
+ <filter-mapping>
+ <filter-name>CORS</filter-name>
+ <url-pattern>/*</url-pattern>
+ </filter-mapping>
+
+ <filter>
+ <filter-name>loggingFilter</filter-name>
+ <filter-class>org.apache.kylin.rest.filter.KylinApiFilter</filter-class>
+ </filter>
+
+ <filter-mapping>
+ <filter-name>loggingFilter</filter-name>
+ <url-pattern>/api/*</url-pattern>
+ </filter-mapping>
+
+ <!--
+ Apply Spring Security Filter to all Requests
+ -->
+ <filter>
<filter-name>springSecurityFilterChain</filter-name>
<filter-class>org.springframework.web.filter.DelegatingFilterProxy</filter-class>
</filter>
@@ -98,15 +110,15 @@ limitations under the License. See accompanying LICENSE file.
<url-pattern>/*</url-pattern>
</filter-mapping>
- <filter>
- <filter-name>instrumentedFilter</filter-name>
- <filter-class>com.codahale.metrics.servlet.InstrumentedFilter</filter-class>
- </filter>
- <filter-mapping>
- <filter-name>instrumentedFilter</filter-name>
- <url-pattern>/*</url-pattern>
- </filter-mapping>
-
+ <filter>
+ <filter-name>instrumentedFilter</filter-name>
+ <filter-class>com.codahale.metrics.servlet.InstrumentedFilter</filter-class>
+ </filter>
+ <filter-mapping>
+ <filter-name>instrumentedFilter</filter-name>
+ <url-pattern>/*</url-pattern>
+ </filter-mapping>
+
<servlet>
<servlet-name>kylin</servlet-name>
<servlet-class>org.springframework.web.servlet.DispatcherServlet</servlet-class>
@@ -125,7 +137,7 @@ limitations under the License. See accompanying LICENSE file.
<servlet-name>metricsservlet</servlet-name>
<url-pattern>/metrics/*</url-pattern>
</servlet-mapping>
-
+
<servlet>
<servlet-name>metricsadminservlet</servlet-name>
<servlet-class>com.codahale.metrics.servlets.AdminServlet</servlet-class>
@@ -135,9 +147,9 @@ limitations under the License. See accompanying LICENSE file.
<servlet-name>metricsadminservlet</servlet-name>
<url-pattern>/metricsadmin/*</url-pattern>
</servlet-mapping>
-
- <!--
- <servlet>
+
+ <!--
+ <servlet>
<servlet-name>matricsadmin</servlet-name>
<servlet-class>com.codahale.metrics.servlets.MetricsServlet</servlet-class>
<load-on-startup>1</load-on-startup>