You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@chukwa.apache.org by as...@apache.org on 2010/10/12 06:27:32 UTC
svn commit: r1021625 - in /incubator/chukwa/trunk: ./ lib/
src/java/org/apache/hadoop/chukwa/conf/
src/java/org/apache/hadoop/chukwa/datacollection/agent/
src/java/org/apache/hadoop/chukwa/datacollection/collector/
Author: asrabkin
Date: Tue Oct 12 04:27:32 2010
New Revision: 1021625
URL: http://svn.apache.org/viewvc?rev=1021625&view=rev
Log:
CHUKWA-527 Configuration spellcheck. Contributed by Ari Rabkin.
Added:
incubator/chukwa/trunk/lib/agent.dict
incubator/chukwa/trunk/lib/collector.dict
incubator/chukwa/trunk/lib/confspellcheck-LICENSE
incubator/chukwa/trunk/lib/confspellcheck-README
incubator/chukwa/trunk/lib/confspellcheck.jar (with props)
Modified:
incubator/chukwa/trunk/NOTICE.txt
incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/conf/ChukwaConfiguration.java
incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/agent/ChukwaAgent.java
incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/collector/CollectorStub.java
Modified: incubator/chukwa/trunk/NOTICE.txt
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/NOTICE.txt?rev=1021625&r1=1021624&r2=1021625&view=diff
==============================================================================
--- incubator/chukwa/trunk/NOTICE.txt (original)
+++ incubator/chukwa/trunk/NOTICE.txt Tue Oct 12 04:27:32 2010
@@ -29,3 +29,5 @@ Copyright is held by its developers.
JChronic is under an MIT license, copyright is held by the developers.
* Available from https://jchronic.dev.java.net/
+The configuration spellchecker was written by Ari Rabkin [a Chukwa committer], and is available under a 2-clause BSD-style license.
+* Available from http://code.google.com/p/jchord/source/browse/#svn/trunk/conf_spellchecker
\ No newline at end of file
Added: incubator/chukwa/trunk/lib/agent.dict
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/lib/agent.dict?rev=1021625&view=auto
==============================================================================
--- incubator/chukwa/trunk/lib/agent.dict (added)
+++ incubator/chukwa/trunk/lib/agent.dict Tue Oct 12 04:27:32 2010
@@ -0,0 +1,35 @@
+$CHUKWA_CONF_DIR
+$CHUKWA_HOME File
+$CHUKWA_PID_DIR File
+CONF-adaptor.dirscan.intervalMs Integral
+CONF-adaptor.memBufWrapper.size Integral
+CONF-chukwaAgent.adaptor.context.switch.time Time
+CONF-chukwaAgent.adaptor.fileadaptor.timeoutperiod Time
+CONF-chukwaAgent.checkpoint.dir File
+CONF-chukwaAgent.checkpoint.enabled Boolean
+CONF-chukwaAgent.checkpoint.interval Time
+CONF-chukwaAgent.checkpoint.name
+CONF-chukwaAgent.connector.retryRate Time
+CONF-chukwaAgent.control.port Portno
+CONF-chukwaAgent.control.remote Boolean
+CONF-chukwaAgent.fileTailingAdaptor.maxReadSize Integral
+CONF-chukwaAgent.initial_adaptors File
+CONF-chukwaAgent.sender.collectorTimeout Integral
+CONF-chukwaAgent.sender.fastRetries Integral
+CONF-chukwaAgent.sender.retries Integral
+CONF-chukwaAgent.sender.retryInterval Integral
+CONF-chukwaAgent.tags
+CONF-chukwaCollector.asyncAcks.scanperiod Time
+CONF-chukwaCollector.http.port Portno
+CONF-chukwaCollector.rotateInterval Time
+CONF-connector.commitpoll.hostfile
+CONF-connector.commitpoll.period Time
+CONF-connector.commitpoll.timeout Time
+CONF-constAdaptor.minSleep Integral
+CONF-constAdaptor.sleepVariance Integral
+CONF-httpConnector.asyncAcks Boolean
+CONF-httpConnector.maxPostSize Integral
+CONF-httpConnector.minPostInterval Time
+PROP-CHUKWA_CONF_DIR File
+PROP-CHUKWA_HOME
+PROP-user.name
Added: incubator/chukwa/trunk/lib/collector.dict
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/lib/collector.dict?rev=1021625&view=auto
==============================================================================
--- incubator/chukwa/trunk/lib/collector.dict (added)
+++ incubator/chukwa/trunk/lib/collector.dict Tue Oct 12 04:27:32 2010
@@ -0,0 +1,28 @@
+$CHUKWA_CONF_DIR
+$CHUKWA_DATA_DIR
+$CHUKWA_HOME
+$CHUKWA_PID_DIR File
+CONF-chukwaCollector.asyncAcks.purgedelay Time
+CONF-chukwaCollector.asyncAcks.scanpaths
+CONF-chukwaCollector.asyncAcks.scanperiod Time
+CONF-chukwaCollector.chunkSuppressBufferSize Integral
+CONF-chukwaCollector.http.port Integral
+CONF-chukwaCollector.http.threads Integral
+CONF-chukwaCollector.localOutputDir File
+CONF-chukwaCollector.minPercentFreeDisk Integral
+CONF-chukwaCollector.outputDir
+CONF-chukwaCollector.pipeline ClassName list
+CONF-chukwaCollector.rotateInterval Time
+CONF-chukwaCollector.showLogs.enabled Boolean
+CONF-chukwaCollector.stats.period Time
+CONF-chukwaCollector.tee.keepalive Boolean
+CONF-chukwaCollector.tee.port Integral
+CONF-chukwaCollector.writeChunkRetries Integral
+CONF-chukwaCollector.writerClass ClassName
+CONF-chukwaRootDir
+CONF-fs.default.name URI
+CONF-httpConnector.asyncAcks Boolean
+CONF-localToRemoteHdfsMover.exitIfHDFSNotavailable Boolean
+CONF-nullWriter.dataRate Time
+CONF-writer.hdfs.filesystem URI
+PROP-CHUKWA_HOME
Added: incubator/chukwa/trunk/lib/confspellcheck-LICENSE
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/lib/confspellcheck-LICENSE?rev=1021625&view=auto
==============================================================================
--- incubator/chukwa/trunk/lib/confspellcheck-LICENSE (added)
+++ incubator/chukwa/trunk/lib/confspellcheck-LICENSE Tue Oct 12 04:27:32 2010
@@ -0,0 +1,26 @@
+Copyright (c) 2008-2010, Ariel Rabkin
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+* Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the
+ distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
Added: incubator/chukwa/trunk/lib/confspellcheck-README
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/lib/confspellcheck-README?rev=1021625&view=auto
==============================================================================
--- incubator/chukwa/trunk/lib/confspellcheck-README (added)
+++ incubator/chukwa/trunk/lib/confspellcheck-README Tue Oct 12 04:27:32 2010
@@ -0,0 +1,29 @@
+This directory holds a configuration spellcheck tool, being developed by Ari Rabkin at UC Bekeley. (Contact asrabkin@gmail.com for details)
+
+The checker has two basic tasks. First, to make sure that every option set in a configuration file is an option actually supported by the program in question. Second, to check that the values specified for the option make sense. (In case, e.g., a user confuses a numeric from boolean parameter).
+
+
+
+DICTIONARY FORMAT
+
+Each line of the dictionary files consists of a regular expression, followed by an option type, followed by an optional annotation. These sections are separated by a single tab character.
+
+Each regular expression may match one or more valid options. (This is to cover cases like Hadoop's fs.x.impl, where x can be any string).
+
+The list of valid option types is as follows:
+ Address
+ Boolean
+ ClassName
+ File
+ Fraction
+ Integral
+ NetworkInterface
+ Portno
+ Special
+ URI
+ URL
+
+Most of these are self-explanatory. Special means that the option must be one of a small number of string options. The list of valid options goes in the annotation field. It should be comma-separated and wrapped in {}.
+
+If a dictionary includes a type not on that list, the checker will assume that all values are acceptable for that option.
+
Added: incubator/chukwa/trunk/lib/confspellcheck.jar
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/lib/confspellcheck.jar?rev=1021625&view=auto
==============================================================================
Binary file - no diff available.
Propchange: incubator/chukwa/trunk/lib/confspellcheck.jar
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Modified: incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/conf/ChukwaConfiguration.java
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/conf/ChukwaConfiguration.java?rev=1021625&r1=1021624&r2=1021625&view=diff
==============================================================================
--- incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/conf/ChukwaConfiguration.java (original)
+++ incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/conf/ChukwaConfiguration.java Tue Oct 12 04:27:32 2010
@@ -27,6 +27,16 @@ import org.apache.log4j.Logger;
public class ChukwaConfiguration extends Configuration {
static Logger log = Logger.getLogger(ChukwaConfiguration.class);
+ private String chukwaHome, chukwaConf;
+
+ public String getChukwaHome() {
+ return chukwaHome;
+ }
+
+ public String getChukwaConf() {
+ return chukwaConf;
+ }
+
public ChukwaConfiguration() {
this(true);
}
@@ -35,7 +45,7 @@ public class ChukwaConfiguration extends
super();
if (loadDefaults) {
- String chukwaHome = System.getenv("CHUKWA_HOME");
+ chukwaHome = System.getenv("CHUKWA_HOME");
if (chukwaHome == null) {
chukwaHome = ".";
}
@@ -43,7 +53,7 @@ public class ChukwaConfiguration extends
if (!chukwaHome.endsWith("/")) {
chukwaHome = chukwaHome + File.separator;
}
- String chukwaConf = System.getenv("CHUKWA_CONF_DIR");
+ chukwaConf = System.getenv("CHUKWA_CONF_DIR");
if (chukwaConf == null) {
chukwaConf = chukwaHome + "conf" + File.separator;
}
Modified: incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/agent/ChukwaAgent.java
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/agent/ChukwaAgent.java?rev=1021625&r1=1021624&r2=1021625&view=diff
==============================================================================
--- incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/agent/ChukwaAgent.java (original)
+++ incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/agent/ChukwaAgent.java Tue Oct 12 04:27:32 2010
@@ -51,12 +51,14 @@ import org.apache.hadoop.chukwa.util.Dae
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.log4j.Logger;
+
import org.mortbay.jetty.Server;
import org.mortbay.jetty.servlet.Context;
import org.mortbay.jetty.servlet.ServletHolder;
import org.mortbay.jetty.nio.SelectChannelConnector;
import org.mortbay.thread.BoundedThreadPool;
import com.sun.jersey.spi.container.servlet.ServletContainer;
+import edu.berkeley.confspell.*;
/**
* The local agent daemon that runs on each machine. This class is designed to
@@ -161,7 +163,9 @@ public class ChukwaAgent implements Adap
+ "[default collector URL]");
System.exit(0);
}
+
Configuration conf = readConfig();
+
ChukwaAgent localAgent = new ChukwaAgent(conf);
if (agent.anotherAgentIsRunning()) {
@@ -220,7 +224,7 @@ public class ChukwaAgent implements Adap
public ChukwaAgent(Configuration conf) throws AlreadyRunningException {
ChukwaAgent.agent = this;
this.conf = conf;
-
+
// almost always just reading this; so use a ConcurrentHM.
// since we wrapped the offset, it's not a structural mod.
adaptorPositions = new ConcurrentHashMap<Adaptor, Offset>();
@@ -739,6 +743,15 @@ public class ChukwaAgent implements Adap
.getAbsolutePath());
conf.set("chukwaAgent.initial_adaptors", new File(chukwaConf,
"initial_adaptors").getAbsolutePath());
+
+
+ try {
+ Configuration chukwaAgentConf = new Configuration(false);
+ chukwaAgentConf.addResource(new Path(agentConf.getAbsolutePath()));
+ Checker.checkConf(new OptDictionary(new File(new File(chukwaHome, "lib"), "agent.dict")),
+ HSlurper.fromHConf(chukwaAgentConf));
+ } catch(Exception e) {e.printStackTrace();}
+
return conf;
}
Modified: incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/collector/CollectorStub.java
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/collector/CollectorStub.java?rev=1021625&r1=1021624&r2=1021625&view=diff
==============================================================================
--- incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/collector/CollectorStub.java (original)
+++ incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/collector/CollectorStub.java Tue Oct 12 04:27:32 2010
@@ -27,7 +27,13 @@ import org.apache.hadoop.chukwa.datacoll
import org.apache.hadoop.chukwa.datacollection.writer.*;
import org.apache.hadoop.chukwa.util.DaemonWatcher;
import org.apache.hadoop.chukwa.conf.ChukwaConfiguration;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import edu.berkeley.confspell.Checker;
+import edu.berkeley.confspell.HSlurper;
+import edu.berkeley.confspell.OptDictionary;
import javax.servlet.http.HttpServlet;
+import java.io.File;
import java.util.*;
public class CollectorStub {
@@ -49,6 +55,15 @@ public class CollectorStub {
}
ChukwaConfiguration conf = new ChukwaConfiguration();
+
+ try {
+ Configuration collectorConf = new Configuration(false);
+ collectorConf.addResource(new Path(conf.getChukwaConf() + "/chukwa-common.xml"));
+ collectorConf.addResource(new Path(conf.getChukwaConf() + "/chukwa-collector-conf.xml"));
+ Checker.checkConf(new OptDictionary(new File(new File(conf.getChukwaHome(), "lib"), "collector.dict")),
+ HSlurper.fromHConf(collectorConf));
+ } catch(Exception e) {e.printStackTrace();}
+
int portNum = conf.getInt("chukwaCollector.http.port", 9999);
THREADS = conf.getInt("chukwaCollector.http.threads", THREADS);