You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@chukwa.apache.org by ey...@apache.org on 2012/08/05 05:24:23 UTC
svn commit: r1369521 - in /incubator/chukwa/trunk: ./ conf/ script/pig/
src/main/java/org/apache/hadoop/chukwa/datacollection/adaptor/sigar/
src/main/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/
src/main/web/hicc/descriptors/
Author: eyang
Date: Sun Aug 5 03:24:23 2012
New Revision: 1369521
URL: http://svn.apache.org/viewvc?rev=1369521&view=rev
Log:
CHUKWA-635. Collect swap usage. (Eric Yang)
Modified:
incubator/chukwa/trunk/CHANGES.txt
incubator/chukwa/trunk/conf/hbase.schema
incubator/chukwa/trunk/script/pig/ClusterSummary.pig
incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/datacollection/adaptor/sigar/SigarRunner.java
incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/SystemMetrics.java
incubator/chukwa/trunk/src/main/web/hicc/descriptors/cluster_metrics.descriptor
incubator/chukwa/trunk/src/main/web/hicc/descriptors/system_metrics.descriptor
Modified: incubator/chukwa/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/CHANGES.txt?rev=1369521&r1=1369520&r2=1369521&view=diff
==============================================================================
--- incubator/chukwa/trunk/CHANGES.txt (original)
+++ incubator/chukwa/trunk/CHANGES.txt Sun Aug 5 03:24:23 2012
@@ -4,6 +4,8 @@ Trunk (unreleased changes)
NEW FEATURES
+ CHUKWA-635. Collect swap usage. (Eric Yang)
+
IMPROVEMENTS
CHUKWA-648. Make Chukwa Reduce Type to support hierarchy format. (Jie Huang via asrabkin)
Modified: incubator/chukwa/trunk/conf/hbase.schema
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/conf/hbase.schema?rev=1369521&r1=1369520&r2=1369521&view=diff
==============================================================================
--- incubator/chukwa/trunk/conf/hbase.schema (original)
+++ incubator/chukwa/trunk/conf/hbase.schema Sun Aug 5 03:24:23 2012
@@ -22,6 +22,7 @@ create "SystemMetrics",
{NAME => "system", VERSION => 65535},
{NAME => "disk", VERSION => 65535},
{NAME => "memory", VERSION => 65535},
+{NAME => "swap", VERSION => 65535},
{NAME => "network", VERSION => 65535},
{NAME => "tags", VERSION => 65535}
create "ClusterSummary",
@@ -30,6 +31,7 @@ create "ClusterSummary",
{NAME => "disk", VERSION => 65535},
{NAME => "memory", VERSION => 65535},
{NAME => "network", VERSION => 65535},
+{NAME => "swap", VERSION => 65535},
{NAME => "hdfs", VERSION => 65535},
{NAME => "mapreduce", VERSION => 65535}
create "chukwa",
Modified: incubator/chukwa/trunk/script/pig/ClusterSummary.pig
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/script/pig/ClusterSummary.pig?rev=1369521&r1=1369520&r2=1369521&view=diff
==============================================================================
--- incubator/chukwa/trunk/script/pig/ClusterSummary.pig (original)
+++ incubator/chukwa/trunk/script/pig/ClusterSummary.pig Sun Aug 5 03:24:23 2012
@@ -16,12 +16,12 @@
* limitations under the License.
*/
%default START '1234567890';
-SystemMetrics = load 'hbase://SystemMetrics' USING org.apache.pig.backend.hadoop.hbase.HBaseStorage('tags:cluster cpu:combined cpu:idle cpu:sys cpu:user disk:ReadBytes disk:Reads disk:WriteBytes disk:Writes system:LoadAverage.1 memory:FreePercent memory:UsedPercent network:RxBytes network:RxDropped network:RxErrors network:RxPackets network:TxBytes network:TxCollisions network:TxErrors network:TxPackets','-loadKey -gt $START -caster Utf8StorageConverter') AS (rowKey, cluster, cpuCombined, cpuIdle, cpuSys, cpuUser, diskReadBytes, diskReads, diskWriteBytes, diskWrites, LoadAverage, memoryFreePercent, memoryUsedPercent, networkRxBytes, networkRxDropped, networkRxErrors, networkRxPackets, networkTxBytes, networkTxCollisions, networkTxErrors, networkTxPackets);
-CleanseBuffer = foreach SystemMetrics generate REGEX_EXTRACT($0,'^\\d+',0) as time, cluster, cpuCombined, cpuIdle, cpuSys, cpuUser, diskReadBytes, diskReads, diskWriteBytes, diskWrites, LoadAverage, memoryFreePercent, memoryUsedPercent, networkRxBytes, networkRxDropped, networkRxErrors, networkRxPackets, networkTxBytes, networkTxCollisions, networkTxErrors, networkTxPackets;
-ConcatBuffer = foreach CleanseBuffer generate CONCAT(CONCAT($0, '-'), $1) as rowId, cpuCombined, cpuIdle, cpuSys, cpuUser, diskReadBytes, diskReads, diskWriteBytes, diskWrites, LoadAverage, memoryFreePercent, memoryUsedPercent, networkRxBytes, networkRxDropped, networkRxErrors, networkRxPackets, networkTxBytes, networkTxCollisions, networkTxErrors, networkTxPackets;
+SystemMetrics = load 'hbase://SystemMetrics' USING org.apache.pig.backend.hadoop.hbase.HBaseStorage('tags:cluster cpu:combined cpu:idle cpu:sys cpu:user disk:ReadBytes disk:Reads disk:WriteBytes disk:Writes system:LoadAverage.1 memory:FreePercent memory:UsedPercent network:RxBytes network:RxDropped network:RxErrors network:RxPackets network:TxBytes network:TxCollisions network:TxErrors network:TxPackets swap:Total swap:Used swap:Free swap:PageIn swap:PageOut','-loadKey -gt $START -caster Utf8StorageConverter') AS (rowKey, cluster, cpuCombined, cpuIdle, cpuSys, cpuUser, diskReadBytes, diskReads, diskWriteBytes, diskWrites, LoadAverage, memoryFreePercent, memoryUsedPercent, networkRxBytes, networkRxDropped, networkRxErrors, networkRxPackets, networkTxBytes, networkTxCollisions, networkTxErrors, networkTxPackets, swapTotal, swapUsed, swapFree, swapPageIn, swapPageOut);
+CleanseBuffer = foreach SystemMetrics generate REGEX_EXTRACT($0,'^\\d+',0) as time, cluster, cpuCombined, cpuIdle, cpuSys, cpuUser, diskReadBytes, diskReads, diskWriteBytes, diskWrites, LoadAverage, memoryFreePercent, memoryUsedPercent, networkRxBytes, networkRxDropped, networkRxErrors, networkRxPackets, networkTxBytes, networkTxCollisions, networkTxErrors, networkTxPackets, swapTotal, swapUsed, swapFree, swapPageIn, swapPageOut;
+ConcatBuffer = foreach CleanseBuffer generate CONCAT(CONCAT($0, '-'), $1) as rowId, cpuCombined, cpuIdle, cpuSys, cpuUser, diskReadBytes, diskReads, diskWriteBytes, diskWrites, LoadAverage, memoryFreePercent, memoryUsedPercent, networkRxBytes, networkRxDropped, networkRxErrors, networkRxPackets, networkTxBytes, networkTxCollisions, networkTxErrors, networkTxPackets, swapTotal, swapUsed, swapFree, swapPageIn, swapPageOut;
TimeSeries = GROUP ConcatBuffer BY rowId;
-ComputeBuffer = FOREACH TimeSeries GENERATE group, AVG(ConcatBuffer.cpuCombined), AVG(ConcatBuffer.cpuIdle), AVG(ConcatBuffer.cpuSys), AVG(ConcatBuffer.cpuUser), AVG(ConcatBuffer.diskReadBytes), AVG(ConcatBuffer.diskReads), AVG(ConcatBuffer.diskWriteBytes), AVG(ConcatBuffer.diskWrites), AVG(ConcatBuffer.LoadAverage), AVG(ConcatBuffer.memoryFreePercent), AVG(ConcatBuffer.memoryUsedPercent), AVG(ConcatBuffer.networkRxBytes), AVG(ConcatBuffer.networkRxDropped), AVG(ConcatBuffer.networkRxErrors), AVG(ConcatBuffer.networkRxPackets), AVG(ConcatBuffer.networkTxBytes), AVG(ConcatBuffer.networkTxCollisions), AVG(ConcatBuffer.networkTxErrors), AVG(ConcatBuffer.networkTxPackets);
-STORE ComputeBuffer INTO 'ClusterSummary' USING org.apache.pig.backend.hadoop.hbase.HBaseStorage('cpu:Combined cpu:Idle cpu:Sys cpu:User disk:ReadBytes disk:Reads disk:WriteBytes disk:Writes system:LoadAverage memory:FreePercent memory:UsedPercent network:RxBytes network:RxDropped network:RxErrors network:RxPackets network:TxBytes network:TxCollisions network:TxErrors network:TxPackets');
+ComputeBuffer = FOREACH TimeSeries GENERATE group, AVG(ConcatBuffer.cpuCombined), AVG(ConcatBuffer.cpuIdle), AVG(ConcatBuffer.cpuSys), AVG(ConcatBuffer.cpuUser), AVG(ConcatBuffer.diskReadBytes), AVG(ConcatBuffer.diskReads), AVG(ConcatBuffer.diskWriteBytes), AVG(ConcatBuffer.diskWrites), AVG(ConcatBuffer.LoadAverage), AVG(ConcatBuffer.memoryFreePercent), AVG(ConcatBuffer.memoryUsedPercent), AVG(ConcatBuffer.networkRxBytes), AVG(ConcatBuffer.networkRxDropped), AVG(ConcatBuffer.networkRxErrors), AVG(ConcatBuffer.networkRxPackets), AVG(ConcatBuffer.networkTxBytes), AVG(ConcatBuffer.networkTxCollisions), AVG(ConcatBuffer.networkTxErrors), AVG(ConcatBuffer.networkTxPackets), AVG(ConcatBuffer.swapTotal), AVG(ConcatBuffer.swapUsed), AVG(ConcatBuffer.swapFree), AVG(ConcatBuffer.swapPageIn), AVG(ConcatBuffer.swapPageOut);
+STORE ComputeBuffer INTO 'ClusterSummary' USING org.apache.pig.backend.hadoop.hbase.HBaseStorage('cpu:Combined cpu:Idle cpu:Sys cpu:User disk:ReadBytes disk:Reads disk:WriteBytes disk:Writes system:LoadAverage memory:FreePercent memory:UsedPercent network:RxBytes network:RxDropped network:RxErrors network:RxPackets network:TxBytes network:TxCollisions network:TxErrors network:TxPackets swap:Total swap:Used swap:Free swap:PageIn swap:PageOut');
HDFSMetrics = load 'hbase://Hadoop' USING org.apache.pig.backend.hadoop.hbase.HBaseStorage('dfs_FSNamesystem:cluster dfs_FSNamesystem:CapacityTotalGB dfs_FSNamesystem:CapacityUsedGB dfs_FSNamesystem:CapacityRemainingGB dfs_FSNamesystem:BlockCapacity dfs_FSNamesystem:BlocksTotal dfs_FSNamesystem:MissingBlocks dfs_FSNamesystem:CorruptBlocks dfs_FSNamesystem:UnderReplicatedBlocks dfs_FSNamesystem:FilesTotal','-loadKey -gt $START -caster Utf8StorageConverter') AS (rowKey, cluster, CapacityTotalGB, CapacityUsedGB, CapacityRemainingGB, BlockCapacity, BlocksTotal, MissingBlocks, CorruptBlocks, UnderReplicatedBlocks, FilesTotal);
CleanseBuffer = foreach HDFSMetrics generate REGEX_EXTRACT($0,'^\\d+',0) as time, cluster, CapacityTotalGB, CapacityUsedGB, CapacityRemainingGB, BlockCapacity, BlocksTotal, MissingBlocks, CorruptBlocks, UnderReplicatedBlocks, FilesTotal;
ConcatBuffer = foreach CleanseBuffer generate CONCAT(CONCAT($0, '-'), $1) as rowId, CapacityTotalGB, CapacityUsedGB, CapacityRemainingGB, BlockCapacity, BlocksTotal, MissingBlocks, CorruptBlocks, UnderReplicatedBlocks, FilesTotal;
Modified: incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/datacollection/adaptor/sigar/SigarRunner.java
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/datacollection/adaptor/sigar/SigarRunner.java?rev=1369521&r1=1369520&r2=1369521&view=diff
==============================================================================
--- incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/datacollection/adaptor/sigar/SigarRunner.java (original)
+++ incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/datacollection/adaptor/sigar/SigarRunner.java Sun Aug 5 03:24:23 2012
@@ -30,6 +30,7 @@ import org.hyperic.sigar.CpuPerc;
import org.hyperic.sigar.FileSystem;
import org.hyperic.sigar.FileSystemUsage;
import org.hyperic.sigar.Mem;
+import org.hyperic.sigar.Swap;
import org.hyperic.sigar.NetInterfaceStat;
import org.hyperic.sigar.Sigar;
import org.hyperic.sigar.Uptime;
@@ -61,6 +62,7 @@ public class SigarRunner extends TimerTa
CpuInfo[] cpuinfo = null;
CpuPerc[] cpuPerc = null;
Mem mem = null;
+ Swap swap = null;
FileSystem[] fs = null;
String[] netIf = null;
Uptime uptime = null;
@@ -103,6 +105,12 @@ public class SigarRunner extends TimerTa
JSONObject memMap = new JSONObject();
memMap.putAll(mem.toMap());
json.put("memory", memMap);
+
+ // Swap Utilization
+ swap = sigar.getSwap();
+ JSONObject swapMap = new JSONObject();
+ swapMap.putAll(swap.toMap());
+ json.put("swap", swapMap);
// Network Utilization
netIf = sigar.getNetInterfaceList();
Modified: incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/SystemMetrics.java
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/SystemMetrics.java?rev=1369521&r1=1369520&r2=1369521&view=diff
==============================================================================
--- incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/SystemMetrics.java (original)
+++ incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/SystemMetrics.java Sun Aug 5 03:24:23 2012
@@ -102,6 +102,16 @@ public class SystemMetrics extends Abstr
}
buildGenericRecord(record, null, cal.getTimeInMillis(), "memory");
output.collect(key, record);
+
+ record = new ChukwaRecord();
+ JSONObject swap = (JSONObject) json.get("swap");
+ Iterator<String> swapKeys = swap.keySet().iterator();
+ while(swapKeys.hasNext()) {
+ String key = swapKeys.next();
+ record.add(key, swap.get(key).toString());
+ }
+ buildGenericRecord(record, null, cal.getTimeInMillis(), "swap");
+ output.collect(key, record);
double rxBytes = 0;
double rxDropped = 0;
Modified: incubator/chukwa/trunk/src/main/web/hicc/descriptors/cluster_metrics.descriptor
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/src/main/web/hicc/descriptors/cluster_metrics.descriptor?rev=1369521&r1=1369520&r2=1369521&view=diff
==============================================================================
--- incubator/chukwa/trunk/src/main/web/hicc/descriptors/cluster_metrics.descriptor (original)
+++ incubator/chukwa/trunk/src/main/web/hicc/descriptors/cluster_metrics.descriptor Sun Aug 5 03:24:23 2012
@@ -24,6 +24,11 @@
{"label":"Disk Write Operations","value":"/hicc/v1/metrics/series/ClusterSummary/disk:Writes/session/cluster"},
{"label":"Memory Free Percentage","value":"/hicc/v1/metrics/series/ClusterSummary/memory:FreePercent/session/cluster"},
{"label":"Memory Used Percentage","value":"/hicc/v1/metrics/series/ClusterSummary/memory:UsedPercent/session/cluster"},
+{"label":"Swap Total","value":"/hicc/v1/metrics/series/ClusterSummary/memory:SwapTotal/session/cluster"},
+{"label":"Swap Used","value":"/hicc/v1/metrics/series/ClusterSummary/swap:Used/session/cluster"},
+{"label":"Swap Free","value":"/hicc/v1/metrics/series/ClusterSummary/swap:Free/session/cluster"},
+{"label":"Swap Page In","value":"/hicc/v1/metrics/series/ClusterSummary/swap:PageIn/session/cluster"},
+{"label":"Swap Page Out","value":"/hicc/v1/metrics/series/ClusterSummary/swap:PageOut/session/cluster"},
{"label":"Network Receive Bytes","value":"/hicc/v1/metrics/series/ClusterSummary/network:RxBytes/session/cluster"},
{"label":"Network Receive Dropped","value":"/hicc/v1/metrics/series/ClusterSummary/network:RxDropped/session/cluster"},
{"label":"Network Receive Errors","value":"/hicc/v1/metrics/series/ClusterSummary/network:RxErrors/session/cluster"},
@@ -32,7 +37,7 @@
{"label":"Network Transfer Collisions","value":"/hicc/v1/metrics/series/ClusterSummary/network:TxCollisions/session/cluster"},
{"label":"Network Transfer Errors","value":"/hicc/v1/metrics/series/ClusterSummary/network:TxErrors/session/cluster"},
{"label":"Network Transfer Packets","value":"/hicc/v1/metrics/series/ClusterSummary/network:TxPackets/session/cluster"},
-{"label":"Load Average","value":"/hicc/v1/metrics/series/ClusterSummary/system:LoadAverage/session/cluster"},
+{"label":"Load Average","value":"/hicc/v1/metrics/series/ClusterSummary/system:LoadAverage/session/cluster"}
]},
{"name":"width","type":"select","value":"300","label":"Width","options":[
{"label":"300","value":"300"},
Modified: incubator/chukwa/trunk/src/main/web/hicc/descriptors/system_metrics.descriptor
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/src/main/web/hicc/descriptors/system_metrics.descriptor?rev=1369521&r1=1369520&r2=1369521&view=diff
==============================================================================
--- incubator/chukwa/trunk/src/main/web/hicc/descriptors/system_metrics.descriptor (original)
+++ incubator/chukwa/trunk/src/main/web/hicc/descriptors/system_metrics.descriptor Sun Aug 5 03:24:23 2012
@@ -23,6 +23,11 @@
{"label":"Memory Free","value":"/hicc/v1/metrics/series/SystemMetrics/memory:Free/session/hosts"},
{"label":"Memory Used Percent","value":"/hicc/v1/metrics/series/SystemMetrics/memory:UsedPercent/session/hosts"},
{"label":"Memory Free Percent","value":"/hicc/v1/metrics/series/SystemMetrics/memory:FreePercent/session/hosts"},
+{"label":"Swap Total","value":"/hicc/v1/metrics/series/SystemMetrics/swap/Total/session/hosts"},
+{"label":"Swap Used","value":"/hicc/v1/metrics/series/SystemMetrics/swap/Used/session/hosts"},
+{"label":"Swap Free","value":"/hicc/v1/metrics/series/SystemMetrics/swap/Free/session/hosts"},
+{"label":"Swap Page In","value":"/hicc/v1/metrics/series/SystemMetrics/swap/PageIn/session/hosts"},
+{"label":"Swap Page Out","value":"/hicc/v1/metrics/series/SystemMetrics/swap/PageOut/session/hosts"},
{"label":"Network Card 1 Receive Error/Second","value":"/hicc/v1/metrics/series/SystemMetrics/network:RxErrors.0/session/hosts"},
{"label":"Network Card 1 Receive Bytes/Second","value":"/hicc/v1/metrics/series/SystemMetrics/network:RxBytes.0/session/hosts"},
{"label":"Network Card 1 Receive Packets/Second","value":"/hicc/v1/metrics/series/SystemMetrics/network:RxPackets.0/session/hosts"},