You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@chukwa.apache.org by ey...@apache.org on 2015/06/25 22:48:43 UTC
[12/22] chukwa git commit: CHUKWA-759. Configuration for Chukwa to
monitor HBase. (Eric Yang)
CHUKWA-759. Configuration for Chukwa to monitor HBase. (Eric Yang)
Project: http://git-wip-us.apache.org/repos/asf/chukwa/repo
Commit: http://git-wip-us.apache.org/repos/asf/chukwa/commit/e1f0df55
Tree: http://git-wip-us.apache.org/repos/asf/chukwa/tree/e1f0df55
Diff: http://git-wip-us.apache.org/repos/asf/chukwa/diff/e1f0df55
Branch: refs/heads/master
Commit: e1f0df55dcaadb48398bb680fc85c2983f897279
Parents: d8abe97
Author: Eric Yang <ey...@apache.org>
Authored: Fri Jun 19 16:31:49 2015 -0700
Committer: Eric Yang <ey...@apache.org>
Committed: Fri Jun 19 16:31:49 2015 -0700
----------------------------------------------------------------------
CHANGES.txt | 6 ++-
conf/chukwa-demux-conf.xml | 9 +++-
conf/hadoop-log4j.properties | 2 -
conf/hadoop-metrics2-hbase.properties | 31 ++++++++++++
conf/hbase-log4j.properties | 78 ++++++++++++++++++++++++++++++
conf/initial_adaptors | 7 +++
6 files changed, 128 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/chukwa/blob/e1f0df55/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index adcd624..449dce7 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -14,10 +14,12 @@ Trunk (unreleased changes)
CHUKWA-737. Heartbeat adaptor to push status to remote http server (Shreyas Subramanya)
- CHUKWA-747. Update Widget API to store data in HBase. (Eric Yang)
+ CHUKWA-747. Update Widget API to store data in HBase. (Eric Yang)
IMPROVEMENTS
+ CHUKWA-759. Configuration for Chukwa to monitor HBase. (Eric Yang)
+
CHUKWA-757. Updated Hadoop configuration to stream data to Chukwa. (Eric Yang)
CHUKWA-749. Added Chukwa tags to Solr schema for indexing logs. (Eric Yang)
@@ -42,7 +44,7 @@ Trunk (unreleased changes)
BUGS
- CHUKWA-758. Updated default dashboard to include system metrics. (Eric Yang)
+ CHUKWA-758. Updated default dashboard to include system metrics. (Eric Yang)
CHUKWA-743. Revised pid locking using standard Hadoop approach. (Eric Yang)
http://git-wip-us.apache.org/repos/asf/chukwa/blob/e1f0df55/conf/chukwa-demux-conf.xml
----------------------------------------------------------------------
diff --git a/conf/chukwa-demux-conf.xml b/conf/chukwa-demux-conf.xml
index fca9f1e..fe380c6 100644
--- a/conf/chukwa-demux-conf.xml
+++ b/conf/chukwa-demux-conf.xml
@@ -126,6 +126,12 @@
</property>
<property>
+ <name>HBaseMetrics</name>
+ <value>org.apache.hadoop.chukwa.extraction.hbase.HadoopMetricsProcessor</value>
+ <description>Parser class for HBase Metrics </description>
+ </property>
+
+ <property>
<name>SysLog</name>
<value>org.apache.hadoop.chukwa.extraction.demux.processor.mapper.SysLog</value>
<description>Parser class for SysLog</description>
@@ -246,6 +252,7 @@
<value>org.apache.hadoop.chukwa.extraction.demux.processor.mapper.ZookeeperProcessor</value>
</property>
+<!--
<property>
<name>HBaseMasterProcessor</name>
<value>org.apache.hadoop.chukwa.extraction.demux.processor.mapper.HBaseMasterProcessor</value>
@@ -254,7 +261,7 @@
<property>
<name>HBaseRegionServerProcessor</name>
<value>org.apache.hadoop.chukwa.extraction.demux.processor.mapper.HBaseRegionServerProcessor</value>
- </property>
+ </property> -->
<!-- Demux configs for reducer -->
<property>
http://git-wip-us.apache.org/repos/asf/chukwa/blob/e1f0df55/conf/hadoop-log4j.properties
----------------------------------------------------------------------
diff --git a/conf/hadoop-log4j.properties b/conf/hadoop-log4j.properties
index aad7cc7..6ef5b59 100644
--- a/conf/hadoop-log4j.properties
+++ b/conf/hadoop-log4j.properties
@@ -35,7 +35,6 @@ log4j.appender.RFA=org.apache.log4j.net.SocketAppender
log4j.appender.RFA.Port=${hadoop.log.port}
log4j.appender.RFA.RemoteHost=localhost
log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
-log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
@@ -47,7 +46,6 @@ log4j.appender.DRFA=org.apache.log4j.net.SocketAppender
log4j.appender.DRFA.Port=${hadoop.log.port}
log4j.appender.DRFA.RemoteHost=localhost
log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
-log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
http://git-wip-us.apache.org/repos/asf/chukwa/blob/e1f0df55/conf/hadoop-metrics2-hbase.properties
----------------------------------------------------------------------
diff --git a/conf/hadoop-metrics2-hbase.properties b/conf/hadoop-metrics2-hbase.properties
new file mode 100644
index 0000000..038d31b
--- /dev/null
+++ b/conf/hadoop-metrics2-hbase.properties
@@ -0,0 +1,31 @@
+# syntax: [prefix].[source|sink].[instance].[options]
+# See javadoc of package-info.java for org.apache.hadoop.metrics2 for details
+
+*.sink.socket.class=org.apache.hadoop.chukwa.inputtools.log4j.Log4jMetricsSink
+hbase.sink.socket.host=localhost
+hbase.sink.socket.port=9104
+# *.sink.file*.class=org.apache.hadoop.metrics2.sink.FileSink
+# default sampling period
+*.period=10
+
+# Below are some examples of sinks that could be used
+# to monitor different hbase daemons.
+
+# hbase.sink.file-all.class=org.apache.hadoop.metrics2.sink.FileSink
+# hbase.sink.file-all.filename=all.metrics
+
+# hbase.sink.file0.class=org.apache.hadoop.metrics2.sink.FileSink
+# hbase.sink.file0.context=hmaster
+# hbase.sink.file0.filename=master.metrics
+
+# hbase.sink.file1.class=org.apache.hadoop.metrics2.sink.FileSink
+# hbase.sink.file1.context=thrift-one
+# hbase.sink.file1.filename=thrift-one.metrics
+
+# hbase.sink.file2.class=org.apache.hadoop.metrics2.sink.FileSink
+# hbase.sink.file2.context=thrift-two
+# hbase.sink.file2.filename=thrift-one.metrics
+
+# hbase.sink.file3.class=org.apache.hadoop.metrics2.sink.FileSink
+# hbase.sink.file3.context=rest
+# hbase.sink.file3.filename=rest.metrics
http://git-wip-us.apache.org/repos/asf/chukwa/blob/e1f0df55/conf/hbase-log4j.properties
----------------------------------------------------------------------
diff --git a/conf/hbase-log4j.properties b/conf/hbase-log4j.properties
new file mode 100644
index 0000000..d1ba3fa
--- /dev/null
+++ b/conf/hbase-log4j.properties
@@ -0,0 +1,78 @@
+# Define some default values that can be overridden by system properties
+hbase.root.logger=INFO,console
+hbase.security.logger=INFO,console
+hbase.log.dir=.
+hbase.log.file=hbase.log
+
+# Define the root logger to the system property "hbase.root.logger".
+log4j.rootLogger=${hbase.root.logger}
+
+# Logging Threshold
+log4j.threshold=ALL
+
+#
+# Daily Rolling File Appender
+#
+log4j.appender.DRFA=org.apache.log4j.net.SocketAppender
+log4j.appender.DRFA.Port=${hbase.log.port}
+log4j.appender.DRFA.RemoteHost=localhost
+log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
+
+# Pattern format: Date LogLevel LoggerName LogMessage
+log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %m%n
+
+# Rolling File Appender
+log4j.appender.RFA=org.apache.log4j.net.SocketAppender
+log4j.appender.RFA.Port=${hbase.log.port}
+log4j.appender.RFA.RemoteHost=localhost
+log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
+log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %m%n
+
+#
+# Security audit appender
+#
+log4j.appender.RFAS=org.apache.log4j.net.SocketAppender
+log4j.appender.RFAS.Port=${hbase.security.log.port}
+log4j.appender.RFAS.RemoteHost=localhost
+log4j.appender.RFAS.layout=org.apache.log4j.PatternLayout
+log4j.appender.RFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
+log4j.category.SecurityLogger=${hbase.security.logger}
+log4j.additivity.SecurityLogger=false
+#log4j.logger.SecurityLogger.org.apache.hadoop.hbase.security.access.AccessController=TRACE
+#log4j.logger.SecurityLogger.org.apache.hadoop.hbase.security.visibility.VisibilityController=TRACE
+
+#
+# Null Appender
+#
+log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender
+
+#
+# console
+# Add "console" to rootlogger above if you want to use this
+#
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.target=System.err
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %m%n
+
+# Custom Logging levels
+
+log4j.logger.org.apache.zookeeper=INFO
+#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG
+log4j.logger.org.apache.hadoop.hbase=INFO
+# Make these two classes INFO-level. Make them DEBUG to see more zk debug.
+log4j.logger.org.apache.hadoop.hbase.zookeeper.ZKUtil=INFO
+log4j.logger.org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher=INFO
+#log4j.logger.org.apache.hadoop.dfs=DEBUG
+# Set this class to log INFO only otherwise its OTT
+# Enable this to get detailed connection error/retry logging.
+# log4j.logger.org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation=TRACE
+
+
+# Uncomment this line to enable tracing on _every_ RPC call (this can be a lot of output)
+#log4j.logger.org.apache.hadoop.ipc.HBaseServer.trace=DEBUG
+
+# Uncomment the below if you want to remove logging of client region caching'
+# and scan of hbase:meta messages
+# log4j.logger.org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation=INFO
+# log4j.logger.org.apache.hadoop.hbase.client.MetaScanner=INFO
http://git-wip-us.apache.org/repos/asf/chukwa/blob/e1f0df55/conf/initial_adaptors
----------------------------------------------------------------------
diff --git a/conf/initial_adaptors b/conf/initial_adaptors
index 5bc26be..08b1e34 100644
--- a/conf/initial_adaptors
+++ b/conf/initial_adaptors
@@ -8,3 +8,10 @@ add SocketAdaptor HadoopNMLog 9100 0
add SocketAdaptor HadoopHSLog 9101 0
add SocketAdaptor HadoopRMSUMMARY 9102 0
add SocketAdaptor HadoopTLA 9103 0
+add SocketAdaptor HBaseMetrics 9104 0
+add SocketAdaptor HBaseSecurityLog 9105 0
+add SocketAdaptor HBaseMSLog 9106 0
+add SocketAdaptor HBaseRSLog 9107 0
+add SocketAdaptor HBaseTFLog 9108 0
+add SocketAdaptor HBaseZKLog 9109 0
+add SocketAdaptor HBaseRESTLog 9110 0