You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@chukwa.apache.org by ey...@apache.org on 2010/12/05 22:09:22 UTC
svn commit: r1042449 - in /incubator/chukwa/trunk: CHANGES.txt
conf/chukwa-collector-conf.xml.template
src/java/org/apache/hadoop/chukwa/datacollection/writer/hbase/HBaseWriter.java
Author: eyang
Date: Sun Dec 5 21:09:22 2010
New Revision: 1042449
URL: http://svn.apache.org/viewvc?rev=1042449&view=rev
Log:
CHUKWA-563. Added configuration to skip HBase schema validation for HBaseWriter. (Eric Yang)
Modified:
incubator/chukwa/trunk/CHANGES.txt
incubator/chukwa/trunk/conf/chukwa-collector-conf.xml.template
incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/writer/hbase/HBaseWriter.java
Modified: incubator/chukwa/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/CHANGES.txt?rev=1042449&r1=1042448&r2=1042449&view=diff
==============================================================================
--- incubator/chukwa/trunk/CHANGES.txt (original)
+++ incubator/chukwa/trunk/CHANGES.txt Sun Dec 5 21:09:22 2010
@@ -4,6 +4,8 @@ Trunk (unreleased changes)
NEW FEATURES
+ CHUKWA-563. Added configuration to skip HBase schema validation for HBaseWriter. (Eric Yang)
+
CHUKWA-558. Update default Chukwa configuration files to use SocketAdaptor for Hadoop metrics streaming. (Eric Yang)
CHUKWA-527 Configuration spellcheck. (Ari Rabkin)
Modified: incubator/chukwa/trunk/conf/chukwa-collector-conf.xml.template
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/conf/chukwa-collector-conf.xml.template?rev=1042449&r1=1042448&r2=1042449&view=diff
==============================================================================
--- incubator/chukwa/trunk/conf/chukwa-collector-conf.xml.template (original)
+++ incubator/chukwa/trunk/conf/chukwa-collector-conf.xml.template Sun Dec 5 21:09:22 2010
@@ -59,7 +59,7 @@
<property>
<name>hbase.zookeeper.quorum</name>
- <value>host1.mydomain.com,host2.mydomain.com,host3.mydomain.com</value>
+ <value>localhost</value>
<description>Comma separated list of servers in the ZooKeeper Quorum.
For example, "host1.mydomain.com,host2.mydomain.com,host3.mydomain.com".
By default this is set to localhost for local and pseudo-distributed modes
@@ -69,6 +69,22 @@
</description>
</property>
+ <property>
+ <name>hbase.writer.verify.schema</name>
+ <value>false</value>
+ <description>Verify HBase Table schema with demux parser schema, log
+ warning if there are mismatch between hbase schema and demux parsers.
+ </description>
+ </property>
+
+ <property>
+ <name>hbase.writer.halt.on.schema.mismatch</name>
+ <value>false</value>
+ <description>If this option is set to true, and HBase table schema
+ is mismatched with demux parser, collector will shut down itself.
+ </description>
+ </property>
+
-->
<property>
Modified: incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/writer/hbase/HBaseWriter.java
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/writer/hbase/HBaseWriter.java?rev=1042449&r1=1042448&r2=1042449&view=diff
==============================================================================
--- incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/writer/hbase/HBaseWriter.java (original)
+++ incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/writer/hbase/HBaseWriter.java Sun Dec 5 21:09:22 2010
@@ -113,7 +113,9 @@ public class HBaseWriter extends Pipelin
statTimer.schedule(new StatReportingTask(), 1000, 10 * 1000);
output = new OutputCollector();
reporter = new Reporter();
- verifyHbaseSchema();
+ if(conf.getBoolean("hbase.writer.verify.schema", false)) {
+ verifyHbaseSchema();
+ }
pool = new HTablePool(hconf, 60);
}
@@ -141,7 +143,7 @@ public class HBaseWriter extends Pipelin
private void verifyHbaseSchema() {
log.debug("Verify Demux parser with HBase schema");
- boolean schemaVerified = false;
+ boolean schemaVerified = true;
try {
HBaseAdmin admin = new HBaseAdmin(hconf);
List<Class> demuxParsers = ClassUtils.getClassesForPackage(conf.get("hbase.demux.package"));
@@ -150,23 +152,25 @@ public class HBaseWriter extends Pipelin
Tables list = x.getAnnotation(Tables.class);
for(Table table : list.annotations()) {
if(!verifyHbaseTable(admin, table)) {
- throw new Exception("Validation failed - table: "+table.name()+" column family: "+table.columnFamily()+" does not exist.");
+ schemaVerified = false;
+ log.warn("Validation failed - table: "+table.name()+" column family: "+table.columnFamily()+" does not exist.");
}
}
} else if(x.isAnnotationPresent(Table.class)) {
Table table = x.getAnnotation(Table.class);
if(!verifyHbaseTable(admin, table)) {
- throw new Exception("Validation failed - table: "+table.name()+" column family: "+table.columnFamily()+" does not exist.");
+ schemaVerified = false;
+ log.warn("Validation failed - table: "+table.name()+" column family: "+table.columnFamily()+" does not exist.");
}
}
}
- schemaVerified = true;
} catch (Exception e) {
+ schemaVerified = false;
log.error(ExceptionUtil.getStackTrace(e));
}
if(!schemaVerified) {
log.error("Hbase schema mismatch with demux parser.");
- if(conf.getBoolean("halt.on.schema.mismatch", true)) {
+ if(conf.getBoolean("hbase.writer.halt.on.schema.mismatch", true)) {
log.error("Exiting...");
DaemonWatcher.bailout(-1);
}