You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@chukwa.apache.org by ey...@apache.org on 2009/05/12 18:16:07 UTC
svn commit: r773960 - in /hadoop/chukwa:
branches/chukwa-0.1/bin/exec-data-loader.sh
branches/chukwa-0.1/bin/systemDataLoader.sh
branches/chukwa-0.1/conf/aggregator.sql trunk/bin/exec-data-loader.sh
trunk/bin/systemDataLoader.sh trunk/conf/aggregator.sql
Author: eyang
Date: Tue May 12 16:16:07 2009
New Revision: 773960
URL: http://svn.apache.org/viewvc?rev=773960&view=rev
Log:
CHUKWA-226. Changed HDFS usage collection frequency from 10 minutes to 60 minutes. (Cheng Zhang via Eric Yang)
Modified:
hadoop/chukwa/branches/chukwa-0.1/bin/exec-data-loader.sh
hadoop/chukwa/branches/chukwa-0.1/bin/systemDataLoader.sh
hadoop/chukwa/branches/chukwa-0.1/conf/aggregator.sql
hadoop/chukwa/trunk/bin/exec-data-loader.sh
hadoop/chukwa/trunk/bin/systemDataLoader.sh
hadoop/chukwa/trunk/conf/aggregator.sql
Modified: hadoop/chukwa/branches/chukwa-0.1/bin/exec-data-loader.sh
URL: http://svn.apache.org/viewvc/hadoop/chukwa/branches/chukwa-0.1/bin/exec-data-loader.sh?rev=773960&r1=773959&r2=773960&view=diff
==============================================================================
--- hadoop/chukwa/branches/chukwa-0.1/bin/exec-data-loader.sh (original)
+++ hadoop/chukwa/branches/chukwa-0.1/bin/exec-data-loader.sh Tue May 12 16:16:07 2009
@@ -98,7 +98,7 @@
fi
if [ "X$PARM" == "XHDFSUsage" ]; then
- ${JAVA_HOME}/bin/java $JVM_OPTS -DPERIOD=600 -DCHUKWA_HOME=${CHUKWA_HOME} -DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR} -DCHUKWA_LOG_DIR=${CHUKWA_LOG_DIR} -DRECORD_TYPE=HDFSUsage -Dlog4j.configuration=system-data-loader.properties -classpath ${HADOOP_CONF_DIR}:${CLASSPATH}:${CHUKWA_CORE}:${COMMON}:${HADOOP_JAR}:${CHUKWA_CONF_DIR} org.apache.hadoop.chukwa.inputtools.jplugin.JPluginAgent org.apache.hadoop.chukwa.inputtools.hdfsusage.HDFSUsagePlugin &
+ ${JAVA_HOME}/bin/java $JVM_OPTS -DPERIOD=3600 -DCHUKWA_HOME=${CHUKWA_HOME} -DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR} -DCHUKWA_LOG_DIR=${CHUKWA_LOG_DIR} -DRECORD_TYPE=HDFSUsage -Dlog4j.configuration=system-data-loader.properties -classpath ${HADOOP_CONF_DIR}:${CLASSPATH}:${CHUKWA_CORE}:${COMMON}:${HADOOP_JAR}:${CHUKWA_CONF_DIR} org.apache.hadoop.chukwa.inputtools.jplugin.JPluginAgent org.apache.hadoop.chukwa.inputtools.hdfsusage.HDFSUsagePlugin &
fi
pid=$!
Modified: hadoop/chukwa/branches/chukwa-0.1/bin/systemDataLoader.sh
URL: http://svn.apache.org/viewvc/hadoop/chukwa/branches/chukwa-0.1/bin/systemDataLoader.sh?rev=773960&r1=773959&r2=773960&view=diff
==============================================================================
--- hadoop/chukwa/branches/chukwa-0.1/bin/systemDataLoader.sh (original)
+++ hadoop/chukwa/branches/chukwa-0.1/bin/systemDataLoader.sh Tue May 12 16:16:07 2009
@@ -285,7 +285,7 @@
fi
if [ ${EXISTS} -lt 1 ]; then
- ${JAVA_HOME}/bin/java $JVM_OPTS -DPERIOD=600 -DCHUKWA_HOME=${CHUKWA_HOME} -DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR} -DCHUKWA_LOG_DIR=${CHUKWA_LOG_DIR} -DRECORD_TYPE=HDFSUsage -Dlog4j.configuration=system-data-loader.properties -classpath ${HADOOP_CONF_DIR}:${CLASSPATH}:${CHUKWA_CORE}:${HADOOP_JAR}:${COMMON}:${TOOLS}:${CHUKWA_CONF_DIR} org.apache.hadoop.chukwa.inputtools.jplugin.JPluginAgent org.apache.hadoop.chukwa.inputtools.hdfsusage.HDFSUsagePlugin &
+ ${JAVA_HOME}/bin/java $JVM_OPTS -DPERIOD=3600 -DCHUKWA_HOME=${CHUKWA_HOME} -DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR} -DCHUKWA_LOG_DIR=${CHUKWA_LOG_DIR} -DRECORD_TYPE=HDFSUsage -Dlog4j.configuration=system-data-loader.properties -classpath ${HADOOP_CONF_DIR}:${CLASSPATH}:${CHUKWA_CORE}:${HADOOP_JAR}:${COMMON}:${TOOLS}:${CHUKWA_CONF_DIR} org.apache.hadoop.chukwa.inputtools.jplugin.JPluginAgent org.apache.hadoop.chukwa.inputtools.hdfsusage.HDFSUsagePlugin &
fi
echo "done"
Modified: hadoop/chukwa/branches/chukwa-0.1/conf/aggregator.sql
URL: http://svn.apache.org/viewvc/hadoop/chukwa/branches/chukwa-0.1/conf/aggregator.sql?rev=773960&r1=773959&r2=773960&view=diff
==============================================================================
--- hadoop/chukwa/branches/chukwa-0.1/conf/aggregator.sql (original)
+++ hadoop/chukwa/branches/chukwa-0.1/conf/aggregator.sql Tue May 12 16:16:07 2009
@@ -11,7 +11,7 @@
#replace delayed into [hod_job_digest] (select timestamp,d.hodid,d.userid,[avg(system_metrics)] from (select a.HodID,b.host as machine,a.userid,a.starttime,a.endtime from [HodJob] a join [hod_machine] b on (a.HodID = b.HodID) where endtime between '[past_15_minutes]' and '[past_5_minutes]') as d,[system_metrics] where timestamp between d.starttime and d.endtime and host=d.machine group by hodid,timestamp);
replace into [cluster_hadoop_rpc] (select timestamp, count(host), avg(rpc_processing_time_avg_time), sum(rpc_processing_time_num_ops), avg(rpc_queue_time_avg_time), sum(rpc_queue_time_num_ops), avg(get_build_version_avg_time), sum(get_build_version_num_ops), avg(get_job_counters_avg_time), sum(get_job_counters_num_ops), avg(get_job_profile_avg_time), sum(get_job_profile_num_ops), avg(get_job_status_avg_time), sum(get_job_status_num_ops), avg(get_new_job_id_avg_time), sum(get_new_job_id_num_ops), avg(get_protocol_version_avg_time), sum(get_protocol_version_num_ops), avg(get_system_dir_avg_time), sum(get_system_dir_num_ops), avg(get_task_completion_events_avg_time), sum(get_task_completion_events_num_ops), avg(get_task_diagnostics_avg_time), sum(get_task_diagnostics_num_ops), avg(heartbeat_avg_time), sum(heartbeat_num_ops), avg(killJob_avg_time), sum(killJob_num_ops), avg(submit_job_avg_time), sum(submit_job_num_ops) from [hadoop_rpc] where timestamp between '[past_15_minutes]
' and '[past_5_minutes]' group by timestamp);
#replace into [user_util] (select timestamp, j.UserID as user, sum(j.NumOfMachines) as node_total, sum(cpu_idle_pcnt*j.NumOfMachines) as cpu_unused, sum((cpu_user_pcnt+cpu_system_pcnt)*j.NumOfMachines) as cpu_used, avg(cpu_user_pcnt+cpu_system_pcnt) as cpu_used_pcnt, sum((100-(sda_busy_pcnt+sdb_busy_pcnt+sdc_busy_pcnt+sdd_busy_pcnt)/4)*j.NumOfMachines) as disk_unused, sum(((sda_busy_pcnt+sdb_busy_pcnt+sdc_busy_pcnt+sdd_busy_pcnt)/4)*j.NumOfMachines) as disk_used, avg(((sda_busy_pcnt+sdb_busy_pcnt+sdc_busy_pcnt+sdd_busy_pcnt)/4)) as disk_used_pcnt, sum((100-eth0_busy_pcnt)*j.NumOfMachines) as network_unused, sum(eth0_busy_pcnt*j.NumOfMachines) as network_used, avg(eth0_busy_pcnt) as network_used_pcnt, sum((100-mem_used_pcnt)*j.NumOfMachines) as memory_unused, sum(mem_used_pcnt*j.NumOfMachines) as memory_used, avg(mem_used_pcnt) as memory_used_pcnt from [hod_job_digest] d,[HodJob] j where (d.HodID = j.HodID) and Timestamp between '[past_15_minutes]' and '[past_5_minutes]' grou
p by j.UserID);
-replace into [util] (select [hdfs_usage].timestamp as timestamp, mr_usage.user, queue, sum(bytes) as bytes, sum(slot_time)/3600 as slot_hours from [hdfs_usage], (select job.finish_time as timestamp,job.user,queue,sum(([mr_task].finish_time-[mr_task].start_time)*[mr_task].attempts) as slot_time from [mr_task], (select job_id,user,queue,launch_time,finish_time from [mr_job] where finish_time between '[past_20_minutes]' and '[now]') as job where [mr_task].job_id=job.job_id group by floor(unix_timestamp(timestamp)/600),user,queue) as mr_usage where mr_usage.user=[hdfs_usage].user and [hdfs_usage].timestamp between '[past_20_minutes]' and '[now]' group by floor(unix_timestamp([hdfs_usage].timestamp)/600), user);
+replace into [util] (select [hdfs_usage].timestamp as timestamp, mr_usage.user, queue, sum(bytes) as bytes, sum(slot_time)/3600 as slot_hours from [hdfs_usage], (select job.finish_time as timestamp,job.user,queue,sum(([mr_task].finish_time-[mr_task].start_time)*[mr_task].attempts) as slot_time from [mr_task], (select job_id,user,queue,launch_time,finish_time from [mr_job] where finish_time between '[past_20_minutes]' and '[now]') as job where [mr_task].job_id=job.job_id group by floor(unix_timestamp(timestamp)/3600),user,queue) as mr_usage where mr_usage.user=[hdfs_usage].user and [hdfs_usage].timestamp between '[past_20_minutes]' and '[now]' group by floor(unix_timestamp([hdfs_usage].timestamp)/3600), user);
#
# Down sample metrics for charts
replace into [system_metrics_month] (select [group_avg(system_metrics)] from [system_metrics_week] where timestamp between '[past_15_minutes]' and '[now]' group by FLOOR(UNIX_TIMESTAMP(Timestamp)/300),host);
Modified: hadoop/chukwa/trunk/bin/exec-data-loader.sh
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/bin/exec-data-loader.sh?rev=773960&r1=773959&r2=773960&view=diff
==============================================================================
--- hadoop/chukwa/trunk/bin/exec-data-loader.sh (original)
+++ hadoop/chukwa/trunk/bin/exec-data-loader.sh Tue May 12 16:16:07 2009
@@ -98,7 +98,7 @@
fi
if [ "X$PARM" == "XHDFSUsage" ]; then
- ${JAVA_HOME}/bin/java $JVM_OPTS -DPERIOD=600 -DCHUKWA_HOME=${CHUKWA_HOME} -DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR} -DCHUKWA_LOG_DIR=${CHUKWA_LOG_DIR} -DRECORD_TYPE=HDFSUsage -Dlog4j.configuration=system-data-loader.properties -classpath ${HADOOP_CONF_DIR}:${CLASSPATH}:${CHUKWA_CORE}:${COMMON}:${HADOOP_JAR}:${CHUKWA_CONF_DIR} org.apache.hadoop.chukwa.inputtools.jplugin.JPluginAgent org.apache.hadoop.chukwa.inputtools.hdfsusage.HDFSUsagePlugin &
+ ${JAVA_HOME}/bin/java $JVM_OPTS -DPERIOD=3600 -DCHUKWA_HOME=${CHUKWA_HOME} -DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR} -DCHUKWA_LOG_DIR=${CHUKWA_LOG_DIR} -DRECORD_TYPE=HDFSUsage -Dlog4j.configuration=system-data-loader.properties -classpath ${HADOOP_CONF_DIR}:${CLASSPATH}:${CHUKWA_CORE}:${COMMON}:${HADOOP_JAR}:${CHUKWA_CONF_DIR} org.apache.hadoop.chukwa.inputtools.jplugin.JPluginAgent org.apache.hadoop.chukwa.inputtools.hdfsusage.HDFSUsagePlugin &
fi
pid=$!
Modified: hadoop/chukwa/trunk/bin/systemDataLoader.sh
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/bin/systemDataLoader.sh?rev=773960&r1=773959&r2=773960&view=diff
==============================================================================
--- hadoop/chukwa/trunk/bin/systemDataLoader.sh (original)
+++ hadoop/chukwa/trunk/bin/systemDataLoader.sh Tue May 12 16:16:07 2009
@@ -285,7 +285,7 @@
fi
if [ ${EXISTS} -lt 1 ]; then
- ${JAVA_HOME}/bin/java $JVM_OPTS -DPERIOD=600 -DCHUKWA_HOME=${CHUKWA_HOME} -DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR} -DCHUKWA_LOG_DIR=${CHUKWA_LOG_DIR} -DRECORD_TYPE=HDFSUsage -Dlog4j.configuration=system-data-loader.properties -classpath ${HADOOP_CONF_DIR}:${CLASSPATH}:${CHUKWA_CORE}:${HADOOP_JAR}:${COMMON}:${TOOLS}:${CHUKWA_CONF_DIR} org.apache.hadoop.chukwa.inputtools.jplugin.JPluginAgent org.apache.hadoop.chukwa.inputtools.hdfsusage.HDFSUsagePlugin &
+ ${JAVA_HOME}/bin/java $JVM_OPTS -DPERIOD=3600 -DCHUKWA_HOME=${CHUKWA_HOME} -DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR} -DCHUKWA_LOG_DIR=${CHUKWA_LOG_DIR} -DRECORD_TYPE=HDFSUsage -Dlog4j.configuration=system-data-loader.properties -classpath ${HADOOP_CONF_DIR}:${CLASSPATH}:${CHUKWA_CORE}:${HADOOP_JAR}:${COMMON}:${TOOLS}:${CHUKWA_CONF_DIR} org.apache.hadoop.chukwa.inputtools.jplugin.JPluginAgent org.apache.hadoop.chukwa.inputtools.hdfsusage.HDFSUsagePlugin &
fi
echo "done"
Modified: hadoop/chukwa/trunk/conf/aggregator.sql
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/conf/aggregator.sql?rev=773960&r1=773959&r2=773960&view=diff
==============================================================================
--- hadoop/chukwa/trunk/conf/aggregator.sql (original)
+++ hadoop/chukwa/trunk/conf/aggregator.sql Tue May 12 16:16:07 2009
@@ -11,7 +11,7 @@
#replace delayed into [hod_job_digest] (select timestamp,d.hodid,d.userid,[avg(system_metrics)] from (select a.HodID,b.host as machine,a.userid,a.starttime,a.endtime from [HodJob] a join [hod_machine] b on (a.HodID = b.HodID) where endtime between '[past_15_minutes]' and '[past_5_minutes]') as d,[system_metrics] where timestamp between d.starttime and d.endtime and host=d.machine group by hodid,timestamp);
replace into [cluster_hadoop_rpc] (select timestamp, count(host), avg(rpc_processing_time_avg_time), sum(rpc_processing_time_num_ops), avg(rpc_queue_time_avg_time), sum(rpc_queue_time_num_ops), avg(get_build_version_avg_time), sum(get_build_version_num_ops), avg(get_job_counters_avg_time), sum(get_job_counters_num_ops), avg(get_job_profile_avg_time), sum(get_job_profile_num_ops), avg(get_job_status_avg_time), sum(get_job_status_num_ops), avg(get_new_job_id_avg_time), sum(get_new_job_id_num_ops), avg(get_protocol_version_avg_time), sum(get_protocol_version_num_ops), avg(get_system_dir_avg_time), sum(get_system_dir_num_ops), avg(get_task_completion_events_avg_time), sum(get_task_completion_events_num_ops), avg(get_task_diagnostics_avg_time), sum(get_task_diagnostics_num_ops), avg(heartbeat_avg_time), sum(heartbeat_num_ops), avg(killJob_avg_time), sum(killJob_num_ops), avg(submit_job_avg_time), sum(submit_job_num_ops) from [hadoop_rpc] where timestamp between '[past_15_minutes]
' and '[past_5_minutes]' group by timestamp);
#replace into [user_util] (select timestamp, j.UserID as user, sum(j.NumOfMachines) as node_total, sum(cpu_idle_pcnt*j.NumOfMachines) as cpu_unused, sum((cpu_user_pcnt+cpu_system_pcnt)*j.NumOfMachines) as cpu_used, avg(cpu_user_pcnt+cpu_system_pcnt) as cpu_used_pcnt, sum((100-(sda_busy_pcnt+sdb_busy_pcnt+sdc_busy_pcnt+sdd_busy_pcnt)/4)*j.NumOfMachines) as disk_unused, sum(((sda_busy_pcnt+sdb_busy_pcnt+sdc_busy_pcnt+sdd_busy_pcnt)/4)*j.NumOfMachines) as disk_used, avg(((sda_busy_pcnt+sdb_busy_pcnt+sdc_busy_pcnt+sdd_busy_pcnt)/4)) as disk_used_pcnt, sum((100-eth0_busy_pcnt)*j.NumOfMachines) as network_unused, sum(eth0_busy_pcnt*j.NumOfMachines) as network_used, avg(eth0_busy_pcnt) as network_used_pcnt, sum((100-mem_used_pcnt)*j.NumOfMachines) as memory_unused, sum(mem_used_pcnt*j.NumOfMachines) as memory_used, avg(mem_used_pcnt) as memory_used_pcnt from [hod_job_digest] d,[HodJob] j where (d.HodID = j.HodID) and Timestamp between '[past_15_minutes]' and '[past_5_minutes]' grou
p by j.UserID);
-replace into [util] (select [hdfs_usage].timestamp as timestamp, mr_usage.user, queue, sum(bytes) as bytes, sum(slot_time)/3600 as slot_hours from [hdfs_usage], (select job.finish_time as timestamp,job.user,queue,sum(([mr_task].finish_time-[mr_task].start_time)*[mr_task].attempts) as slot_time from [mr_task], (select job_id,user,queue,launch_time,finish_time from [mr_job] where finish_time between '[past_20_minutes]' and '[now]') as job where [mr_task].job_id=job.job_id group by floor(unix_timestamp(timestamp)/600),user,queue) as mr_usage where mr_usage.user=[hdfs_usage].user and [hdfs_usage].timestamp between '[past_20_minutes]' and '[now]' group by floor(unix_timestamp([hdfs_usage].timestamp)/600), user);
+replace into [util] (select [hdfs_usage].timestamp as timestamp, mr_usage.user, queue, sum(bytes) as bytes, sum(slot_time)/3600 as slot_hours from [hdfs_usage], (select job.finish_time as timestamp,job.user,queue,sum(([mr_task].finish_time-[mr_task].start_time)*[mr_task].attempts) as slot_time from [mr_task], (select job_id,user,queue,launch_time,finish_time from [mr_job] where finish_time between '[past_20_minutes]' and '[now]') as job where [mr_task].job_id=job.job_id group by floor(unix_timestamp(timestamp)/3600),user,queue) as mr_usage where mr_usage.user=[hdfs_usage].user and [hdfs_usage].timestamp between '[past_20_minutes]' and '[now]' group by floor(unix_timestamp([hdfs_usage].timestamp)/3600), user);
#
# Down sample metrics for charts
replace into [system_metrics_month] (select [group_avg(system_metrics)] from [system_metrics_week] where timestamp between '[past_15_minutes]' and '[now]' group by FLOOR(UNIX_TIMESTAMP(Timestamp)/300),host);