You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@chukwa.apache.org by ey...@apache.org on 2010/12/18 08:35:00 UTC
svn commit: r1050592 - in /incubator/chukwa/trunk: ./ conf/
src/docs/src/documentation/content/xdocs/
src/java/org/apache/hadoop/chukwa/datastore/
src/java/org/apache/hadoop/chukwa/hicc/
src/java/org/apache/hadoop/chukwa/util/ src/web/hicc/jsp/
Author: eyang
Date: Sat Dec 18 07:34:59 2010
New Revision: 1050592
URL: http://svn.apache.org/viewvc?rev=1050592&view=rev
Log:
CHUKWA-554. Remove jdbc.conf dependency from Chukwa. (Eric Yang)
Removed:
incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/util/WatchDog.java
incubator/chukwa/trunk/src/web/hicc/jsp/get_user_list.jsp
incubator/chukwa/trunk/src/web/hicc/jsp/hod_job.jsp
incubator/chukwa/trunk/src/web/hicc/jsp/single-series-chart-javascript.jsp
Modified:
incubator/chukwa/trunk/CHANGES.txt
incubator/chukwa/trunk/conf/README
incubator/chukwa/trunk/src/docs/src/documentation/content/xdocs/admin.xml
incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datastore/ChukwaHBaseStore.java
incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/hicc/ClusterConfig.java
incubator/chukwa/trunk/src/web/hicc/jsp/cluster_selector.jsp
incubator/chukwa/trunk/src/web/hicc/jsp/event2_viewer.jsp
Modified: incubator/chukwa/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/CHANGES.txt?rev=1050592&r1=1050591&r2=1050592&view=diff
==============================================================================
--- incubator/chukwa/trunk/CHANGES.txt (original)
+++ incubator/chukwa/trunk/CHANGES.txt Sat Dec 18 07:34:59 2010
@@ -28,6 +28,8 @@ Trunk (unreleased changes)
IMPROVEMENTS
+ CHUKWA-554. Remove jdbc.conf dependency from Chukwa. (Eric Yang)
+
CHUKWA-562. Added RegExDirTailingAdaptor. (Deepak Deshpande via Eric Yang)
CHUKWA-474. Replaced JSON library with JSON-Simple. (Eric Yang)
Modified: incubator/chukwa/trunk/conf/README
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/conf/README?rev=1050592&r1=1050591&r2=1050592&view=diff
==============================================================================
--- incubator/chukwa/trunk/conf/README (original)
+++ incubator/chukwa/trunk/conf/README Sat Dec 18 07:34:59 2010
@@ -29,11 +29,6 @@ chukwa-collector-conf.xml.template
> The name node address, used by collectors to write data to HDFS
> eg: write.hdf.filesystem = hdfs://localhost:9000/
--------------------------------------------
-jdbc.conf.template
-> Modify? yes
-> URL for the MySQL database
-> eg: jdbc:mysql://localhost:3306?user-root
----------------------------------------------
nagios.properties.template
> Modify? yes, optional
> Nagio server that receives NSCA alert
@@ -127,4 +122,4 @@ hadoop-metrics-log4j.properties
hadoop-metrics.properties.template
> Modify? no
> Sample metrics for Chukwa cluster.
-> Sample metrics for source nodes, if set up as a Hadoop cluster.
\ No newline at end of file
+> Sample metrics for source nodes, if set up as a Hadoop cluster.
Modified: incubator/chukwa/trunk/src/docs/src/documentation/content/xdocs/admin.xml
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/src/docs/src/documentation/content/xdocs/admin.xml?rev=1050592&r1=1050591&r2=1050592&view=diff
==============================================================================
--- incubator/chukwa/trunk/src/docs/src/documentation/content/xdocs/admin.xml (original)
+++ incubator/chukwa/trunk/src/docs/src/documentation/content/xdocs/admin.xml Sat Dec 18 07:34:59 2010
@@ -329,56 +329,6 @@ source /path/to/chukwa/conf/database_upg
<source>CHUKWA_HOME/bin/downSampling.sh --config <path to chukwa conf> -n add </source>
</section>
-
-<section>
-<title>Set Up the Database </title>
-<p>Set up and configure the MySQL database.</p>
-
-<section>
-<title>Install MySQL</title>
-
-<p>Download MySQL 5.1 from the <a href="http://dev.mysql.com/downloads/mysql/5.1.html#downloads">MySQL site</a>. </p>
-<source>
-tar fxvz mysql-*.tar.gz -C $CHUKWA_HOME/opt
-cd $CHUKWA_HOME/opt/mysql-*
-</source>
-
-<p>
-Configure and then copy the my.cnf file to the CHUKWA_HOME/opt/mysql-* directory:
-</p>
-<source>
-./scripts/mysql_install_db
-./bin/mysqld_safe&
-./bin/mysqladmin -u root create <clustername>
-./bin/mysql -u root <clustername> < $CHUKWA_HOME/conf/database_create_tables.sql
-</source>
-
-<p>Edit the CHUKWA_HOME/conf/jdbc.conf configuration file. </p>
-<p>Set the clustername to the MYSQL root URL:</p>
-<source>
-<clustername>=jdbc:mysql://localhost:3306/<clustername>?user=root
-</source>
-
-<p>Download the MySQL Connector/J 5.1 from the <a href="http://dev.mysql.com/downloads/connector/j/5.1.html">MySQL site</a>,
-and place the jar file in $CHUKWA_HOME/lib.</p>
-</section>
-
-<section>
-<title>Set Up MySQL for Replication</title>
-<p>Start the MySQL shell:</p>
-<source>
-mysql -u root -p
-Enter password:
-</source>
-<p>From the MySQL shell, enter these commands (replace <username> and <password> with actual values):</p>
-<source>
-GRANT REPLICATION SLAVE ON *.* TO '<username>'@'%' IDENTIFIED BY '<password>';
-FLUSH PRIVILEGES;
-</source>
-</section>
-
-</section>
-
<!--
<section>
<title>Validate the Chukwa Processes </title>
Modified: incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datastore/ChukwaHBaseStore.java
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datastore/ChukwaHBaseStore.java?rev=1050592&r1=1050591&r2=1050592&view=diff
==============================================================================
--- incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datastore/ChukwaHBaseStore.java (original)
+++ incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datastore/ChukwaHBaseStore.java Sat Dec 18 07:34:59 2010
@@ -7,6 +7,8 @@ import java.util.Iterator;
import java.util.List;
import java.util.Set;
import java.util.concurrent.CopyOnWriteArraySet;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import org.apache.hadoop.chukwa.datacollection.writer.hbase.HBaseWriter;
import org.apache.hadoop.chukwa.hicc.bean.Series;
@@ -161,4 +163,32 @@ public class ChukwaHBaseStore {
public static Set<String> getHostnames(String cluster, long startTime, long endTime) {
return getRowNames("SystemMetrics","system:csource", startTime, endTime);
}
+
+ public static Set<String> getClusterNames(long startTime, long endTime) {
+ String tableName = "SystemMetrics";
+ String column = "system:ctags";
+ Set<String> clusters = new HashSet<String>();
+ HTable table = pool.getTable(tableName);
+ Pattern p = Pattern.compile("\\s*cluster=\"(.*?)\"");
+ try {
+ Scan scan = new Scan();
+ scan.addColumn(column.getBytes());
+ scan.setTimeRange(startTime, endTime);
+ ResultScanner results = table.getScanner(scan);
+ Iterator<Result> it = results.iterator();
+ while(it.hasNext()) {
+ Result result = it.next();
+ String buffer = new String(result.getValue(column.getBytes()));
+ Matcher m = p.matcher(buffer);
+ if(m.matches()) {
+ clusters.add(m.group(1));
+ }
+ }
+ results.close();
+ table.close();
+ } catch(Exception e) {
+ log.error(ExceptionUtil.getStackTrace(e));
+ }
+ return clusters;
+ }
}
Modified: incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/hicc/ClusterConfig.java
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/hicc/ClusterConfig.java?rev=1050592&r1=1050591&r2=1050592&view=diff
==============================================================================
--- incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/hicc/ClusterConfig.java (original)
+++ incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/hicc/ClusterConfig.java Sat Dec 18 07:34:59 2010
@@ -22,8 +22,10 @@ package org.apache.hadoop.chukwa.hicc;
import java.io.*;
import java.util.*;
+import org.apache.hadoop.chukwa.datastore.ChukwaHBaseStore;
+
public class ClusterConfig {
- public static HashMap<String, String> clusterMap = new HashMap<String, String>();
+ public static Set<String> clusterMap = null;
private String path = System.getenv("CHUKWA_CONF_DIR") + File.separator;
static public String getContents(File aFile) {
@@ -56,23 +58,15 @@ public class ClusterConfig {
}
public ClusterConfig() {
- File cc = new File(path + "jdbc.conf");
- String buffer = getContents(cc);
- String[] lines = buffer.split("\n");
- for (String line : lines) {
- String[] data = line.split("=", 2);
- clusterMap.put(data[0], data[1]);
+ long end = System.currentTimeMillis();
+ long start = end - 3600000L;
+ if(clusterMap==null) {
+ clusterMap = ChukwaHBaseStore.getClusterNames(start, end);
}
}
- public String getURL(String cluster) {
- String url = clusterMap.get(cluster);
- return url;
- }
-
public Iterator<String> getClusters() {
- Set<String> keys = clusterMap.keySet();
- Iterator<String> i = keys.iterator();
+ Iterator<String> i = clusterMap.iterator();
return i;
}
}
Modified: incubator/chukwa/trunk/src/web/hicc/jsp/cluster_selector.jsp
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/src/web/hicc/jsp/cluster_selector.jsp?rev=1050592&r1=1050591&r2=1050592&view=diff
==============================================================================
--- incubator/chukwa/trunk/src/web/hicc/jsp/cluster_selector.jsp (original)
+++ incubator/chukwa/trunk/src/web/hicc/jsp/cluster_selector.jsp Sat Dec 18 07:34:59 2010
@@ -38,7 +38,6 @@ Cluster
}
}
ClusterConfig cc = new ClusterConfig();
- String jdbc = cc.getURL(cluster);
Iterator i = cc.getClusters();
while(i.hasNext()) {
String label = i.next().toString();
Modified: incubator/chukwa/trunk/src/web/hicc/jsp/event2_viewer.jsp
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/src/web/hicc/jsp/event2_viewer.jsp?rev=1050592&r1=1050591&r2=1050592&view=diff
==============================================================================
--- incubator/chukwa/trunk/src/web/hicc/jsp/event2_viewer.jsp (original)
+++ incubator/chukwa/trunk/src/web/hicc/jsp/event2_viewer.jsp Sat Dec 18 07:34:59 2010
@@ -37,7 +37,6 @@ Filter: <input type="text" id="<%= XssFi
<%
String cluster = (String) session.getAttribute("cluster");
ClusterConfig cc = new ClusterConfig();
- String jdbc = cc.getURL(cluster);
String boxId=XssFilter.filter(request.getParameter("boxId"));
TimeHandler time = new TimeHandler(request);
String startdate = time.getStartTimeText();