You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by om...@apache.org on 2008/08/13 00:35:23 UTC
svn commit: r685353 [1/13] - in /hadoop/core/trunk: ./ src/contrib/chukwa/
src/contrib/chukwa/bin/ src/contrib/chukwa/build/ src/contrib/chukwa/conf/
src/contrib/chukwa/dist/ src/contrib/chukwa/docs/
src/contrib/chukwa/docs/paper/ src/contrib/chukwa/ha...
Author: omalley
Date: Tue Aug 12 15:35:16 2008
New Revision: 685353
URL: http://svn.apache.org/viewvc?rev=685353&view=rev
Log:
HADOOP-3719. Initial checkin of Chukwa, which is a data collection and
analysis framework. (Jerome Boulon, Andy Konwinski, Ari Rabkin,
and Eric Yang)
Added:
hadoop/core/trunk/src/contrib/chukwa/
hadoop/core/trunk/src/contrib/chukwa/LICENSE.txt
hadoop/core/trunk/src/contrib/chukwa/bin/
hadoop/core/trunk/src/contrib/chukwa/bin/VERSION
hadoop/core/trunk/src/contrib/chukwa/bin/agent.sh (with props)
hadoop/core/trunk/src/contrib/chukwa/bin/buildDailyArchive.sh
hadoop/core/trunk/src/contrib/chukwa/bin/buildHourlyArchive.sh
hadoop/core/trunk/src/contrib/chukwa/bin/chukwa-config.sh (with props)
hadoop/core/trunk/src/contrib/chukwa/bin/jettyCollector.sh (with props)
hadoop/core/trunk/src/contrib/chukwa/bin/jobhisttailstarter.sh
hadoop/core/trunk/src/contrib/chukwa/bin/processSinkFiles.sh (with props)
hadoop/core/trunk/src/contrib/chukwa/bin/shutdown.sh (with props)
hadoop/core/trunk/src/contrib/chukwa/bin/startup.sh (with props)
hadoop/core/trunk/src/contrib/chukwa/bin/systemDataLoader.sh
hadoop/core/trunk/src/contrib/chukwa/bin/watchdog.sh (with props)
hadoop/core/trunk/src/contrib/chukwa/build/
hadoop/core/trunk/src/contrib/chukwa/build.xml
hadoop/core/trunk/src/contrib/chukwa/chukwafy_jobhist.awk
hadoop/core/trunk/src/contrib/chukwa/conf/
hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-agent-conf.xml
hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-agent-conf.xml.template
hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-collector-conf.xml
hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-collector-conf.xml.template
hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-env.sh
hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-env.sh.template
hadoop/core/trunk/src/contrib/chukwa/conf/collectors
hadoop/core/trunk/src/contrib/chukwa/conf/collectors.template
hadoop/core/trunk/src/contrib/chukwa/conf/commons-logging.properties
hadoop/core/trunk/src/contrib/chukwa/conf/fields.spec
hadoop/core/trunk/src/contrib/chukwa/conf/joblog.properties
hadoop/core/trunk/src/contrib/chukwa/conf/log4j.properties
hadoop/core/trunk/src/contrib/chukwa/conf/mdl.xml
hadoop/core/trunk/src/contrib/chukwa/conf/mdl.xml.template
hadoop/core/trunk/src/contrib/chukwa/conf/mysql_create_tables
hadoop/core/trunk/src/contrib/chukwa/conf/mysql_upgrade_tables
hadoop/core/trunk/src/contrib/chukwa/conf/nodeActivity.properties
hadoop/core/trunk/src/contrib/chukwa/conf/queueinfo.properties
hadoop/core/trunk/src/contrib/chukwa/conf/system-data-loader.properties
hadoop/core/trunk/src/contrib/chukwa/conf/torque.properties
hadoop/core/trunk/src/contrib/chukwa/conf/util.properties
hadoop/core/trunk/src/contrib/chukwa/dist/
hadoop/core/trunk/src/contrib/chukwa/docs/
hadoop/core/trunk/src/contrib/chukwa/docs/README
hadoop/core/trunk/src/contrib/chukwa/docs/paper/
hadoop/core/trunk/src/contrib/chukwa/docs/paper/chukwa0.jpg (with props)
hadoop/core/trunk/src/contrib/chukwa/docs/paper/chukwa1.jpg (with props)
hadoop/core/trunk/src/contrib/chukwa/docs/paper/chukwa_08.tex
hadoop/core/trunk/src/contrib/chukwa/docs/paper/hicc_in_action2.png (with props)
hadoop/core/trunk/src/contrib/chukwa/docs/paper/usenix.sty (with props)
hadoop/core/trunk/src/contrib/chukwa/docs/paper/widget.png (with props)
hadoop/core/trunk/src/contrib/chukwa/hadoop-packaging/
hadoop/core/trunk/src/contrib/chukwa/hadoop-packaging/build-tarball.sh (with props)
hadoop/core/trunk/src/contrib/chukwa/hadoop-packaging/log4j.properties.templ
hadoop/core/trunk/src/contrib/chukwa/hadoop-packaging/new-chukwa-conf/
hadoop/core/trunk/src/contrib/chukwa/hadoop-packaging/patchhadoop.patch
hadoop/core/trunk/src/contrib/chukwa/hadoopjars/
hadoop/core/trunk/src/contrib/chukwa/hadoopjars/hadoop-0.18.0-core.jar (with props)
hadoop/core/trunk/src/contrib/chukwa/lib/
hadoop/core/trunk/src/contrib/chukwa/lib/ant-LICENSE.txt
hadoop/core/trunk/src/contrib/chukwa/lib/ant-README.txt
hadoop/core/trunk/src/contrib/chukwa/lib/ant.jar (with props)
hadoop/core/trunk/src/contrib/chukwa/lib/commons-codec-1.3-LICENSE.txt
hadoop/core/trunk/src/contrib/chukwa/lib/commons-codec-1.3-README.txt
hadoop/core/trunk/src/contrib/chukwa/lib/commons-codec-1.3.jar (with props)
hadoop/core/trunk/src/contrib/chukwa/lib/commons-fileupload-1.2-LICENSE.txt
hadoop/core/trunk/src/contrib/chukwa/lib/commons-fileupload-1.2-README.txt
hadoop/core/trunk/src/contrib/chukwa/lib/commons-fileupload-1.2.jar (with props)
hadoop/core/trunk/src/contrib/chukwa/lib/commons-httpclient-3.1-LICENSE.txt
hadoop/core/trunk/src/contrib/chukwa/lib/commons-httpclient-3.1-README.txt
hadoop/core/trunk/src/contrib/chukwa/lib/commons-httpclient-3.1.jar (with props)
hadoop/core/trunk/src/contrib/chukwa/lib/commons-io-1.4-LICENSE.txt
hadoop/core/trunk/src/contrib/chukwa/lib/commons-io-1.4-README.txt
hadoop/core/trunk/src/contrib/chukwa/lib/commons-io-1.4.jar (with props)
hadoop/core/trunk/src/contrib/chukwa/lib/commons-logging-LICENSE.txt
hadoop/core/trunk/src/contrib/chukwa/lib/commons-logging-README.txt
hadoop/core/trunk/src/contrib/chukwa/lib/commons-logging-adapters-1.1-LICENSE.txt
hadoop/core/trunk/src/contrib/chukwa/lib/commons-logging-adapters-1.1.jar (with props)
hadoop/core/trunk/src/contrib/chukwa/lib/commons-logging-api-1.1-LICENSE.txt
hadoop/core/trunk/src/contrib/chukwa/lib/commons-logging-api-1.1.jar (with props)
hadoop/core/trunk/src/contrib/chukwa/lib/commons-logging.jar (with props)
hadoop/core/trunk/src/contrib/chukwa/lib/jetty-6.1.11-LICENSE.txt
hadoop/core/trunk/src/contrib/chukwa/lib/jetty-6.1.11-README.txt
hadoop/core/trunk/src/contrib/chukwa/lib/jetty-6.1.11.jar (with props)
hadoop/core/trunk/src/contrib/chukwa/lib/jetty-util-6.1.11.jar (with props)
hadoop/core/trunk/src/contrib/chukwa/lib/json-LICENSE.txt
hadoop/core/trunk/src/contrib/chukwa/lib/json-README.txt
hadoop/core/trunk/src/contrib/chukwa/lib/json.jar (with props)
hadoop/core/trunk/src/contrib/chukwa/lib/jsp-api-LICENSE.txt
hadoop/core/trunk/src/contrib/chukwa/lib/jsp-api-README.txt
hadoop/core/trunk/src/contrib/chukwa/lib/jsp-api.jar (with props)
hadoop/core/trunk/src/contrib/chukwa/lib/junit-LICENSE.txt
hadoop/core/trunk/src/contrib/chukwa/lib/junit-README.html
hadoop/core/trunk/src/contrib/chukwa/lib/junit.jar (with props)
hadoop/core/trunk/src/contrib/chukwa/lib/log4j-1.2.13-LICENSE.txt
hadoop/core/trunk/src/contrib/chukwa/lib/log4j-1.2.13-README.txt
hadoop/core/trunk/src/contrib/chukwa/lib/log4j-1.2.13.jar (with props)
hadoop/core/trunk/src/contrib/chukwa/lib/servlet-LICENSE.txt
hadoop/core/trunk/src/contrib/chukwa/lib/servlet-README.txt
hadoop/core/trunk/src/contrib/chukwa/lib/servlet-api-LICENSE.txt
hadoop/core/trunk/src/contrib/chukwa/lib/servlet-api-README.txt
hadoop/core/trunk/src/contrib/chukwa/lib/servlet-api.jar (with props)
hadoop/core/trunk/src/contrib/chukwa/lib/servlet.jar (with props)
hadoop/core/trunk/src/contrib/chukwa/lib/taglibs-LICENSE.txt
hadoop/core/trunk/src/contrib/chukwa/lib/taglibs-README.txt
hadoop/core/trunk/src/contrib/chukwa/lib/taglibs.jar (with props)
hadoop/core/trunk/src/contrib/chukwa/opt/
hadoop/core/trunk/src/contrib/chukwa/src/
hadoop/core/trunk/src/contrib/chukwa/src/java/
hadoop/core/trunk/src/contrib/chukwa/src/java/org/
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/ChukwaArchiveKey.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/Chunk.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/ChunkBuilder.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/ChunkImpl.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/conf/
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/conf/ChukwaConfiguration.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/ChunkQueue.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/ChunkReceiver.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/DataFactory.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/Adaptor.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/AdaptorException.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/ExecAdaptor.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/CharFileTailingAdaptorUTF8.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/CharFileTailingAdaptorUTF8NewLineEscaped.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/FileTailer.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/FileTailingAdaptor.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/agent/
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/agent/AdaptorFactory.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/agent/AgentControlSocketListener.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/agent/ChukwaAgent.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/agent/MemLimitQueue.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/agent/WaitingQueue.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/collector/
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/collector/CollectorStub.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/collector/servlet/
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/collector/servlet/ServletCollector.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/collector/servlet/ServletDiagnostics.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/connector/
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/connector/Connector.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/connector/http/
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/connector/http/HttpConnector.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/controller/
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/controller/ChukwaAgentController.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/protocol/
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/protocol/Protocol.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/sender/
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/sender/ChukwaHttpSender.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/sender/ChukwaSender.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/sender/RetryListOfCollectors.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/test/
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/test/ConsoleOutConnector.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/test/FileTailerStressTest.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/test/SinkFileValidator.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/writer/
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/writer/ChukwaWriter.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/writer/ConsoleWriter.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/writer/InMemoryWriter.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/writer/PipelineStageWriter.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/writer/SeqFileWriter.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/Consolidator.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/archive/
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/archive/ChuckwaArchiveBuilder.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/archive/ChukwaArchiveDailyOutputFormat.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/archive/ChukwaArchiveDailyPartitioner.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/archive/ChukwaArchiveHourlyOutputFormat.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/archive/ChukwaArchiveHourlyPartitioner.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/archive/ChukwaArchiveMerger.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/database/
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/database/DataExpiration.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/database/DatabaseHelper.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/database/DatabaseLoader.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/database/MetricDataLoader.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/demux/
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/demux/ChukwaRecordOutputFormat.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/demux/ChukwaRecordPartitioner.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/demux/Demux.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/demux/MoveOrMergeRecordFile.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/demux/TaggerPlugin.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/AbstractProcessor.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/ChunkProcessor.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/DFInvalidRecord.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/DFProcessor.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/DebugOutputProcessor.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/DuplicateProcessorException.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/HadoopLogProcessor.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/HadoopMetricsProcessor.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/Iostat.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/JobLogHistoryProcessor.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/PbsInvalidEntry.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/PbsNodesProcessor.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/ProcessorFactory.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/Sar.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/Top.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/UnknownRecordTypeException.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/YWatchProcessor.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/YwatchInvalidEntry.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/engine/
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/engine/ChukwaRecord.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/engine/ChukwaRecordJT.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/engine/ChukwaSearchResult.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/engine/ChukwaSearchService.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/engine/DatabaseRecord.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/engine/Record.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/engine/SearchResult.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/engine/SearchService.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/engine/datasource/
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/engine/datasource/DataSource.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/engine/datasource/DataSourceException.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/engine/datasource/DataSourceFactory.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/engine/datasource/DsDirectory.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/engine/datasource/database/
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/engine/datasource/database/DatabaseDS.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/engine/datasource/record/
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/engine/datasource/record/ChukwaFileParser.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/engine/datasource/record/ChukwaSequenceFileParser.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/engine/datasource/record/MalformedFileFormat.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/engine/datasource/record/RecordDS.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/hicc/
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/hicc/ClusterConfig.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/hicc/ColorPicker.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/hicc/Config.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/hicc/DatasetMapper.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/hicc/JSONLoader.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/hicc/TimeHandler.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/hicc/Views.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/hicc/ViewsTag.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/hicc/Workspace.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/inputtools/
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/inputtools/ChukwaTTInstru.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/inputtools/log4j/
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/inputtools/log4j/ChukwaDailyRollingFileAppender.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/inputtools/log4j/OneLineLogLayout.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/inputtools/mdl/
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/inputtools/mdl/DataConfig.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/inputtools/mdl/LoaderServer.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/inputtools/plugin/
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/inputtools/plugin/ExecPlugin.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/inputtools/plugin/IPlugin.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/inputtools/plugin/metrics/
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/inputtools/plugin/metrics/Exec.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/inputtools/plugin/metrics/Log4JMetricsContext.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/inputtools/plugin/nodeactivity/
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/inputtools/plugin/nodeactivity/NodeActivityPlugin.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/inputtools/plugin/pbsnode/
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/inputtools/plugin/pbsnode/PbsNodePlugin.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/util/
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/util/ConstRateAdaptor.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/util/DatabaseWriter.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/util/ExceptionUtil.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/util/MaxRateSender.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/util/RecordConstants.java
hadoop/core/trunk/src/contrib/chukwa/src/test/
hadoop/core/trunk/src/contrib/chukwa/src/test/org/
hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/
hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/
hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/
hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/TestChunkBuilder.java
hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/datacollection/
hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/datacollection/TempFileUtil.java
hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/datacollection/TestAgentCollector.java
hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/datacollection/adaptor/
hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/datacollection/adaptor/TestExecAdaptor.java
hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/
hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/TestFileTailingAdaptors.java
hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/TestStartAtOffset.java
hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/datacollection/agent/
hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/datacollection/agent/TestAgent.java
hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/datacollection/connector/
hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/datacollection/connector/ChunkCatcherConnector.java
hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/datacollection/connector/TestFailedCollector.java
hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/datacollection/controller/
hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/datacollection/controller/TestAgentClient.java
hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/datacollection/sender/
hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/datacollection/sender/TestRetryListOfCollectors.java
hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/extraction/
hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/extraction/demux/
hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/extraction/demux/TestDemux.java
hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/extraction/demux/processor/
hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/inputtools/
hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/inputtools/log4j/
hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/inputtools/log4j/TestChukwaAppender.java
hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/util/
hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/util/TestRecordConsts.java
hadoop/core/trunk/src/contrib/chukwa/src/web/
hadoop/core/trunk/src/contrib/chukwa/src/web/collector/
hadoop/core/trunk/src/contrib/chukwa/src/web/collector/WEB-INF/
hadoop/core/trunk/src/contrib/chukwa/src/web/collector/WEB-INF/lib/
hadoop/core/trunk/src/contrib/chukwa/src/web/collector/WEB-INF/web.xml
hadoop/core/trunk/src/contrib/chukwa/test/
hadoop/core/trunk/src/contrib/chukwa/tools/
hadoop/core/trunk/src/contrib/chukwa/tools/expire.sh (with props)
hadoop/core/trunk/src/contrib/chukwa/var/
hadoop/core/trunk/src/contrib/chukwa/var/run/
Modified:
hadoop/core/trunk/CHANGES.txt
Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=685353&r1=685352&r2=685353&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Tue Aug 12 15:35:16 2008
@@ -78,6 +78,10 @@
HADOOP-657. Free disk space should be modelled and used by the scheduler
to make scheduling decisions. (Ari Rabkin via omalley)
+ HADOOP-3719. Initial checkin of Chukwa, which is a data collection and
+ analysis framework. (Jerome Boulon, Andy Konwinski, Ari Rabkin,
+ and Eric Yang)
+
IMPROVEMENTS
HADOOP-3732. Delay intialization of datanode block verification till
Added: hadoop/core/trunk/src/contrib/chukwa/LICENSE.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/LICENSE.txt?rev=685353&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/LICENSE.txt (added)
+++ hadoop/core/trunk/src/contrib/chukwa/LICENSE.txt Tue Aug 12 15:35:16 2008
@@ -0,0 +1,202 @@
+
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
Added: hadoop/core/trunk/src/contrib/chukwa/bin/VERSION
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/VERSION?rev=685353&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/VERSION (added)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/VERSION Tue Aug 12 15:35:16 2008
@@ -0,0 +1 @@
+0.0.1
Added: hadoop/core/trunk/src/contrib/chukwa/bin/agent.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/agent.sh?rev=685353&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/agent.sh (added)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/agent.sh Tue Aug 12 15:35:16 2008
@@ -0,0 +1,26 @@
+#!/bin/sh
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+pid=$$
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+
+. "$bin"/chukwa-config.sh
+
+echo "${pid}" > "$CHUKWA_HOME/var/run/Agent.pid"
+
+${JAVA_HOME}/bin/java -DCHUKWA_HOME=${CHUKWA_HOME} -classpath ${CLASSPATH}:${chukwaCore}:${chukwaAgent}:${HADOOP_JAR}:${CHUKWA_HOME}/conf:${HADOOP_HOME}/conf:${common} org.apache.hadoop.chukwa.datacollection.agent.ChukwaAgent $@
Propchange: hadoop/core/trunk/src/contrib/chukwa/bin/agent.sh
------------------------------------------------------------------------------
svn:executable = *
Added: hadoop/core/trunk/src/contrib/chukwa/bin/buildDailyArchive.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/buildDailyArchive.sh?rev=685353&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/buildDailyArchive.sh (added)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/buildDailyArchive.sh Tue Aug 12 15:35:16 2008
@@ -0,0 +1,30 @@
+#!/bin/sh
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+pid=$$
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+. "$bin"/chukwa-config.sh
+
+echo "${pid}" > "$CHUKWA_HOME/var/run/BuildDailyArchive.pid"
+
+HADOOP_CONF_DIR="${HADOOP_HOME}/conf/"
+HADOOP_CMDE="${HADOOP_HOME}/bin/hadoop "
+
+
+ $HADOOP_CMDE jar ${chukwaCore} org.apache.hadoop.chukwa.extraction.archive.ChuckwaArchiveBuilder Daily $srcEventHdfsDir ${chuwaRecordsRepository}
+
\ No newline at end of file
Added: hadoop/core/trunk/src/contrib/chukwa/bin/buildHourlyArchive.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/buildHourlyArchive.sh?rev=685353&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/buildHourlyArchive.sh (added)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/buildHourlyArchive.sh Tue Aug 12 15:35:16 2008
@@ -0,0 +1,30 @@
+#!/bin/sh
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+pid=$$
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+. "$bin"/chukwa-config.sh
+
+echo "${pid}" > "$CHUKWA_HOME/var/run/BuildDailyArchive.pid"
+
+HADOOP_CONF_DIR="${HADOOP_HOME}/conf/"
+HADOOP_CMDE="${HADOOP_HOME}/bin/hadoop "
+
+
+ $HADOOP_CMDE jar ${chukwaCore} org.apache.hadoop.chukwa.extraction.archive.ChuckwaArchiveBuilder Hourly $srcEventHdfsDir ${chuwaRecordsRepository}
+
\ No newline at end of file
Added: hadoop/core/trunk/src/contrib/chukwa/bin/chukwa-config.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/chukwa-config.sh?rev=685353&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/chukwa-config.sh (added)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/chukwa-config.sh Tue Aug 12 15:35:16 2008
@@ -0,0 +1,104 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# included in all the hadoop scripts with source command
+# should not be executable directly
+# also should not be passed any arguments, since we need original $*
+
+# resolve links - $0 may be a softlink
+
+this="$0"
+while [ -h "$this" ]; do
+ ls=`ls -ld "$this"`
+ link=`expr "$ls" : '.*-> \(.*\)$'`
+ if expr "$link" : '.*/.*' > /dev/null; then
+ this="$link"
+ else
+ this=`dirname "$this"`/"$link"
+ fi
+done
+
+# convert relative path to absolute path
+bin=`dirname "$this"`
+script=`basename "$this"`
+bin=`cd "$bin"; pwd`
+this="$bin/$script"
+
+. ${bin}/../conf/chukwa-env.sh
+
+export HADOOP_HOME="${HADOOP_HOME:-${bin}/../../../..}"
+
+# the root of the Chukwa installation
+if [ -z $CHUKWA_HOME ] ; then
+CHUKWA_HOME=`dirname "$this"`/..
+export CHUKWA_HOME=`cd $CHUKWA_HOME; pwd`
+fi
+
+chukwaVersion=`cat ${CHUKWA_HOME}/bin/VERSION`
+DEFAULT_CHUKWA_HOME=${CHUKWA_HOME}/logs/
+export CHUKWA_LOG_DIR="${CHUKWA_LOG_DIR:-$DEFAULT_CHUKWA_HOME}"
+if [ ! -d $CHUKWA_LOG_DIR ]; then
+ mkdir -p $CHUKWA_LOG_DIR
+fi
+
+export chuwaRecordsRepository="/chukwa/repos/demo"
+
+export DATACONFIG=${CHUKWA_HOME}/conf/mdl.xml
+common=`ls ${CHUKWA_HOME}/lib/*.jar`
+export common=`echo ${common} | sed 'y/ /:/'`
+
+#chukwaCore=${HADOOP_HOME}/build/contrib/chukwa/chukwa-core-${chukwaVersion}.jar
+chukwaCore=${HADOOP_HOME}/build/contrib/chukwa
+if [ -a $chukwaCore ] ; then
+ export chukwaCore
+else
+ echo ${chukwaCore} does not exist
+ export chukwaCore=${CHUKWA_HOME}/chukwa-core-${chukwaVersion}.jar
+fi
+
+#chukwaAgent=${HADOOP_HOME}/build/contrib/chukwa/chukwa-agent-${chukwaVersion}.jar
+chukwaAgent=${HADOOP_HOME}/build/contrib/chukwa
+if [ -a $chukwaAgent ] ; then
+ export chukwaAgent
+else
+ echo ${chukwaAgent} does not exist
+ export chukwaAgent=${CHUKWA_HOME}/chukwa-agent-${chukwaVersion}.jar
+fi
+
+echo chukwaCore is ${chukwaCore} and chukwaAgent is ${chukwaAgent}
+
+export CURRENT_DATE=`date +%Y%m%d%H%M`
+export TS_CONFIG=${CHUKWA_HOME}/conf/ts
+export tomcat=${CHUKWA_HOME}/opt/apache-tomcat-6.0.16
+if [ -d ${HADOOP_HOME}/build/classes ]; then
+ DEFAULT_HADOOP_JAR=${HADOOP_HOME}/build/classes
+# this doesn't work, but needs to be replaced with something that does
+#elif [ls ${HADOOP_HOME}/build/hadoop-*-core.jar` ]; then
+# echo setting DEFAULT_HADOOP_JAR to `ls ${HADOOP_HOME}/build/hadoop-*-core.jar`
+# DEFAULT_HADOOP_JAR=`ls ${HADOOP_HOME}/build/hadoop-*-core.jar`
+else
+ DEFAULT_HADOOP_JAR=${CHUKWA_HOME}/hadoopjars/hadoop-0.18.0-core.jar
+fi
+export HADOOP_JAR=${HADOOP_JAR:-$DEFAULT_HADOOP_JAR}
+
+echo
+echo HADOOP_JAR is $HADOOP_JAR
+echo
+
+export CHUKWA_LOG_DIR="${CHUKWA_HOME}/logs/"
+DEFAULT_PID_DIR=${CHUKWA_HOME}/var/run
+export CHUKWA_PID_DIR="${CHUKWA_PID_DIR:-$DEFAULT_PID_DIR}"
+export chuwaRecordsRepository="/chukwa/repos/demo"
+
Propchange: hadoop/core/trunk/src/contrib/chukwa/bin/chukwa-config.sh
------------------------------------------------------------------------------
svn:executable = *
Added: hadoop/core/trunk/src/contrib/chukwa/bin/jettyCollector.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/jettyCollector.sh?rev=685353&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/jettyCollector.sh (added)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/jettyCollector.sh Tue Aug 12 15:35:16 2008
@@ -0,0 +1,31 @@
+#!/bin/sh
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+pid=$$
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+
+. "$bin"/chukwa-config.sh
+
+echo "${pid}" > "${CHUKWA_PID_DIR}/Collector.pid"
+
+
+if [ $# -lt 1 ]; then
+${JAVA_HOME}/bin/java -classpath ${CLASSPATH}:${chukwaCore}:${common}:${HADOOP_JAR}:${CHUKWA_HOME}/conf:${HADOOP_HOME}/conf org.apache.hadoop.chukwa.datacollection.collector.CollectorStub 9310
+else
+${JAVA_HOME}/bin/java -classpath ${CLASSPATH}:${chukwaCore}:${common}:${HADOOP_JAR}:${CHUKWA_HOME}/conf:${HADOOP_HOME}/conf org.apache.hadoop.chukwa.datacollection.collector.CollectorStub $@
+fi
Propchange: hadoop/core/trunk/src/contrib/chukwa/bin/jettyCollector.sh
------------------------------------------------------------------------------
svn:executable = *
Added: hadoop/core/trunk/src/contrib/chukwa/bin/jobhisttailstarter.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/jobhisttailstarter.sh?rev=685353&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/jobhisttailstarter.sh (added)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/jobhisttailstarter.sh Tue Aug 12 15:35:16 2008
@@ -0,0 +1,59 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# A script to tell chukwa to monitor job history files.
+# Rather simpleminded -- gets a list of what's being monitored,
+# and tells chukwa to watch everything in job hist that it isn't already scanning.
+# Relies on having netcat. Also, control socket portno is currently hardcoded,
+# as are hostname and adaptor name.
+
+if [ $# -lt 1 ]; then
+ echo 1>&2 Usage: $0 '<path to job history files>'
+ exit 127
+ fi
+
+
+JOB_HIST=`(cd $1; pwd)` #returns an absolute path
+echo "assuming job history logs live in $JOB_HIST"
+JOBHISTFILES=/tmp/jobhistfiles
+TAILEDFILES=/tmp/tailedhists
+
+#Step 1 -- get a list of currently watched files
+(nc localhost 9093 | grep -o "[^/]*$" | grep -o '^[^ ]*' | sort > $TAILEDFILES) <<HERE
+list
+close
+HERE
+
+#step 2 -- get the list of history files
+ls $JOB_HIST | grep -v '\.xml' | sort > $JOBHISTFILES
+#step 3 -- start watching each new history file
+#find files that aren't being watched, and are in job history dir
+#NEWHISTFILES=`cat $JOBHISTFILES`
+#NEWHISTFILES=`sort /tmp/both | uniq -u > /tmp/one`| uniq -d - $JOBHISTFILES`
+
+cat $JOBHISTFILES $TAILEDFILES | sort | uniq -u > /tmp/either
+#either not tailed, or not a history file
+NEWHISTFILES=`cat /tmp/either $JOBHISTFILES | sort | uniq -d`
+#better be a job history file -- hence, not being tailed
+
+for job in $NEWHISTFILES ; do
+ #new jobs are rare, safe to create socket per job hist file
+nc localhost 9093 <<HERE
+add LineFileTailUTF8 $JOB_HIST$job 0
+close
+HERE
+ echo "told Chukwa agent to start watching $job"
+done
Added: hadoop/core/trunk/src/contrib/chukwa/bin/processSinkFiles.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/processSinkFiles.sh?rev=685353&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/processSinkFiles.sh (added)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/processSinkFiles.sh Tue Aug 12 15:35:16 2008
@@ -0,0 +1,59 @@
+#!/bin/sh
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+pid=$$
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+. "$bin"/chukwa-config.sh
+
+echo "${pid}" > "${CHUKWA_PID_DIR}/ProcessSinkFiles.pid"
+
+HADOOP_CONF_DIR="${HADOOP_HOME}/conf/"
+HADOOP_CMDE="${HADOOP_HOME}/bin/hadoop "
+
+while [ 1 ]
+do
+
+
+ now=`date +%s`
+ strDate=`date +%m/%d/%y%n`
+ srcDoneHdfsDir="/chukwa/tmp/srcDone$now/"
+ srcEventHdfsDir="/chukwa/tmp/srcEvent$now/"
+
+ echo "Running $strDate $now" >> "${CHUKWA_LOG_DIR}/mr.log"
+
+ echo "srcDoneHdfsDir: $srcDoneHdfsDir srcEventHdfsDir: $srcEventHdfsDir" >> "${MR_LOG}"
+
+ $HADOOP_CMDE dfs -mkdir $srcDoneHdfsDir
+ echo "done with mkdir" >> "${CHUKWA_LOG_DIR}/mr.log"
+
+ $HADOOP_CMDE dfs -mv "/chukwa/logs/*/*.done" $srcDoneHdfsDir
+ echo "done with mv logs" >> "${CHUKWA_LOG_DIR}/mr.log"
+
+ $HADOOP_CMDE jar ${chukwaCore} org.apache.hadoop.chukwa.extraction.demux.Demux -r 2 $srcDoneHdfsDir $srcEventHdfsDir
+ echo "done with demux job" >> "${CHUKWA_LOG_DIR}/mr.log"
+
+ $HADOOP_CMDE jar ${chukwaCore} org.apache.hadoop.chukwa.extraction.demux.MoveOrMergeLogFile $srcEventHdfsDir ${chuwaRecordsRepository}
+ echo "done with MoveOrMergeLogFile" >> "${CHUKWA_LOG_DIR}/mr.log"
+
+ now=`date +%s`
+ strDate=`date +%m/%d/%y%n`
+ echo "Stopping ${strDate} ${now}" >> "${CHUKWA_LOG_DIR}/mr.log"
+
+ sleep 300
+
+done
Propchange: hadoop/core/trunk/src/contrib/chukwa/bin/processSinkFiles.sh
------------------------------------------------------------------------------
svn:executable = *
Added: hadoop/core/trunk/src/contrib/chukwa/bin/shutdown.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/shutdown.sh?rev=685353&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/shutdown.sh (added)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/shutdown.sh Tue Aug 12 15:35:16 2008
@@ -0,0 +1,81 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+java=$JAVA_HOME/bin/java
+
+. "$bin"/chukwa-config.sh
+
+# remove watchdog
+crontab -l | grep -v ${CHUKWA_HOME}/bin/watchdog.sh > /tmp/cron.${CURRENT_DATE}
+crontab /tmp/cron.${CURRENT_DATE}
+rm -f /tmp/cron.${CURRENT_DATE}
+
+# stop torque data loader
+pidFile=$CHUKWA_HOME/var/run/TorqueDataLoader.pid
+if [ -f $pidFile ]; then
+ echo -n "Shutting down Torque Data Loader.."
+ torquepid=`head ${pidFile}`
+ kill -HUP ${torquepid}
+ # kill -HUP `ps eww |grep TorqueDataLoader |grep -v grep |cut -b 1-5` >/dev/null 2>&1
+ rm ${pidFile}
+ echo "done"
+else
+ echo " no $pidFile"
+fi
+
+# stop util data loader
+pidFile=$CHUKWA_HOME/var/run/UtilDataLoader.pid
+if [ -f $pidFile ]; then
+ echo -n "Shutting down Util Data Loader.."
+ utilpid=`head ${pidFile}`
+ #kill -HUP `ps eww |grep UtilDataLoader |grep -v grep |cut -b 1-5` >/dev/null 2>&1
+ kill -HUP ${utilpid}
+ rm ${pidFile}
+ echo "done"
+else
+ echo " no $pidFile"
+fi
+
+# stop queue info data loader
+pidFile=$CHUKWA_HOME/var/run/QueueInfoDataLoader.pid
+if [ -f $pidFile ]; then
+ echo -n "Shutting down Queue Info Data Loader.."
+ queuepid=`head ${pidFile}`
+ #kill -HUP `ps eww |grep QueueInfoDataLoader |grep -v grep |cut -b 1-5` >/dev/null 2>&1
+ kill -HUP ${queuepid}
+ rm ${pidFile}
+ echo "done"
+else
+ echo " no $pidFile"
+fi
+
+
+# stop queue info data loader
+pidFile=$CHUKWA_HOME/var/run/MapReduceLogLoader.pid
+if [ -f $pidFile ]; then
+ echo -n "Shutting down Map Reduce Log Loader.."
+ logpid=`head ${pidFile}`
+ #kill -HUP `ps eww |grep MapReduceLogLoader |grep -v grep |cut -b 1-5` >/dev/null 2>&1
+ kill -HUP ${logpid}
+ rm ${pidFile}
+ echo "done"
+else
+ echo " no $pidFile"
+fi
+
Propchange: hadoop/core/trunk/src/contrib/chukwa/bin/shutdown.sh
------------------------------------------------------------------------------
svn:executable = *
Added: hadoop/core/trunk/src/contrib/chukwa/bin/startup.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/startup.sh?rev=685353&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/startup.sh (added)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/startup.sh Tue Aug 12 15:35:16 2008
@@ -0,0 +1,54 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+java=$JAVA_HOME/bin/java
+
+. "$bin"/chukwa-config.sh
+crontab -l > /tmp/cron.${CURRENT_DATE}
+crontest=$?
+
+if [ ! -d ${CHUKWA_HOME}/opt/apache-tomcat-6.0.16 ]; then
+ tar fxz ${CHUKWA_HOME}/opt/apache-tomcat-6.0.16.tar.gz -C ${CHUKWA_HOME}/opt
+fi
+
+if [ ! -f ${CHUKWA_HOME}/opt/apache-tomcat-6.0.16/webapps/ts.war ]; then
+ cp ${CHUKWA_HOME}/ts.war ${CHUKWA_HOME}/opt/apache-tomcat-6.0.16/webapps
+fi
+
+if [ "X$?" != "X0" ]; then
+ cat > /tmp/cron.${CURRENT_DATE} << CRON
+* * * * * ${CHUKWA_HOME}/bin/watchdog.sh
+10 * * * * ${CHUKWA_HOME}/tools/expire.sh 3 /grid/0/tmp/mdl
+CRON
+else
+ grep -v "${CHUKWA_HOME}/bin/watchdog.sh" /tmp/cron.${CURRENT_DATE} | \
+ grep -v "${CHUKWA_HOME}/tools/expire.sh 3 /grid/0/tmp/mdl" > /tmp/cron.${CURRENT_DATE}.2
+ mv /tmp/cron.${CURRENT_DATE}.2 /tmp/cron.${CURRENT_DATE}
+ cat >> /tmp/cron.${CURRENT_DATE} << CRON
+* * * * * ${CHUKWA_HOME}/bin/watchdog.sh
+10 * * * * ${CHUKWA_HOME}/tools/expire.sh 3 /grid/0/tmp/mdl
+CRON
+fi
+
+# save crontab
+echo -n "Registering data loader cron jobs.."
+crontab /tmp/cron.${CURRENT_DATE} > /dev/null 2>&1
+rm -f /tmp/cron.${CURRENT_DATE}
+echo "done"
+
Propchange: hadoop/core/trunk/src/contrib/chukwa/bin/startup.sh
------------------------------------------------------------------------------
svn:executable = *
Added: hadoop/core/trunk/src/contrib/chukwa/bin/systemDataLoader.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/systemDataLoader.sh?rev=685353&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/systemDataLoader.sh (added)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/systemDataLoader.sh Tue Aug 12 15:35:16 2008
@@ -0,0 +1,28 @@
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+pid=$$
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+
+. "$bin"/chukwa-config.sh
+
+echo "${pid}" > "$CHUKWA_HOME/var/run/systemDataLoader.pid"
+
+${JAVA_HOME}/bin/java -DCHUKWA_HOME=${CHUKWA_HOME} -DRECORD_TYPE=Sar -Dlog4j.configuration=system-data-loader.properties -classpath ${CLASSPATH}:${chukwaCore}:${hadoop_jar}:${common}:${tools}:${CHUKWA_HOME}/conf org.apache.hadoop.chukwa.inputtools.plugin.metrics.Exec sar -q -r -n FULL 55 &
+${JAVA_HOME}/bin/java -DCHUKWA_HOME=${CHUKWA_HOME} -DRECORD_TYPE=Iostat -Dlog4j.configuration=system-data-loader.properties -classpath ${CLASSPATH}:${chukwaCore}:${hadoop_jar}:${common}:${tools}:${CHUKWA_HOME}/conf org.apache.hadoop.chukwa.inputtools.plugin.metrics.Exec iostat -x 55 2 &
+${JAVA_HOME}/bin/java -DCHUKWA_HOME=${CHUKWA_HOME} -DRECORD_TYPE=Top -Dlog4j.configuration=system-data-loader.properties -classpath ${CLASSPATH}:${chukwaCore}:${hadoop_jar}:${common}:${tools}:${CHUKWA_HOME}/conf org.apache.hadoop.chukwa.inputtools.plugin.metrics.Exec top -b -n 1 -c &
+#${JAVA_HOME}/bin/java -DRECORD_TYPE=Df -Dlog4j.configuration=system-data-loader.properties -classpath ${CLASSPATH}:${chukwaAgent}:${hadoop_jar}:${common}:${tools}:${CHUKWA_HOME}/conf org.apache.hadoop.chukwa.inputtools.plugin.metrics.Exec df -x nfs -x none &
+
Added: hadoop/core/trunk/src/contrib/chukwa/bin/watchdog.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/bin/watchdog.sh?rev=685353&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/bin/watchdog.sh (added)
+++ hadoop/core/trunk/src/contrib/chukwa/bin/watchdog.sh Tue Aug 12 15:35:16 2008
@@ -0,0 +1,112 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+
+. "$bin"/chukwa-config.sh
+
+java=$JAVA_HOME/bin/java
+jps=$JAVA_HOME/bin/jps
+
+
+min=`date +%M`
+
+
+# start torque data loader
+pidFile=$CHUKWA_HOME/var/run/TorqueDataLoader.pid
+if [ -f $pidFile ]; then
+ pid=`head ${pidFile}`
+ ChildPIDRunningStatus=`${jps} | grep ${pid} | grep TorqueDataLoader | grep -v grep | wc -l`
+ #ChildPIDRunningStatus=`ps -ef | grep TorqueDataLoader | grep -v grep | wc -l`
+ if [ $ChildPIDRunningStatus -lt 1 ]; then
+ ${java} -DCHUKWA_HOME=${CHUKWA_HOME} -Dlog4j.configuration=torque.properties -classpath ${CLASSPATH}:${chukwa}:${ikit}:${common} org.apache.hadoop.chukwa.sources.mdl.TorqueDataLoader&
+ fi
+else
+ ${java} -DCHUKWA_HOME=${CHUKWA_HOME} -Dlog4j.configuration=torque.properties -classpath ${CLASSPATH}:${chukwa}:${ikit}:${common} org.apache.hadoop.chukwa.sources.mdl.TorqueDataLoader&
+fi
+# start util data loader
+pidFile=$CHUKWA_HOME/var/run/UtilDataLoader.pid
+if [ -f $pidFile ]; then
+ pid=`head ${pidFile}`
+ ChildPIDRunningStatus=`${jps} | grep ${pid} | grep UtilDataLoader | grep -v grep | wc -l`
+ #ChildPIDRunningStatus=`ps -ef | grep UtilDataLoader | grep -v grep | wc -l`
+ if [ $ChildPIDRunningStatus -lt 1 ]; then
+ ${java} -DCHUKWA_HOME=${CHUKWA_HOME} -Dlog4j.configuration=util.properties -classpath ${CLASSPATH}:${chukwa}:${ikit}:${common} org.apache.hadoop.chukwa.sources.mdl.UtilDataLoader&
+ fi
+else
+ ${java} -DCHUKWA_HOME=${CHUKWA_HOME} -Dlog4j.configuration=util.properties -classpath ${CLASSPATH}:${chukwa}:${ikit}:${common} org.apache.hadoop.chukwa.sources.mdl.UtilDataLoader&
+fi
+
+# start queue info data loader
+pidFile=$CHUKWA_HOME/var/run/QueueInfoDataLoader.pid
+if [ -f $pidFile ]; then
+ pid=`head ${pidFile}`
+ ChildPIDRunningStatus=`${jps} | grep ${pid} | grep QueueInfoDataLoader | grep -v grep | wc -l`
+ #ChildPIDRunningStatus=`ps -ef | grep QueueInfoDataLoader | grep -v grep | wc -l`
+ if [ $ChildPIDRunningStatus -lt 1 ]; then
+ ${java} -DCHUKWA_HOME=${CHUKWA_HOME} -Dlog4j.configuration=queueinfo.properties -classpath ${CLASSPATH}:${chukwa}:${ikit}:${common} org.apache.hadoop.chukwa.sources.mdl.QueueInfoDataLoader&
+ fi
+else
+ ${java} -DCHUKWA_HOME=${CHUKWA_HOME} -Dlog4j.configuration=queueinfo.properties -classpath ${CLASSPATH}:${chukwa}:${ikit}:${common} org.apache.hadoop.chukwa.sources.mdl.QueueInfoDataLoader&
+fi
+
+# start map reduce log data loader
+tenmin=`echo ${min} | cut -b 2-`
+if [ "X${tenmin}" == "X0" ]; then
+ pidFile=$CHUKWA_HOME/var/run/JobLogDataLoader.pid
+ if [ -f $pidFile ]; then
+ pid=`head ${pidFile}`
+ ChildPIDRunningStatus=`${jps} | grep ${pid} | grep JobLogDataLoader | wc -l`
+ if [ $ChildPIDRunningStatus -lt 1 ]; then
+ ${java} -Xms128m -Xmx1280m -DCHUKWA_HOME=${CHUKWA_HOME} -Dlog4j.configuration=joblog.properties -classpath ${CLASSPATH}:${chukwa}:${ikit}:${common} org.apache.hadoop.chukwa.sources.mdl.JobLogDataLoader &
+ fi
+ else
+ ${java} -Xms128m -Xmx1280m -DCHUKWA_HOME=${CHUKWA_HOME} -Dlog4j.configuration=joblog.properties -classpath ${CLASSPATH}:${chukwa}:${ikit}:${common} org.apache.hadoop.chukwa.sources.mdl.JobLogDataLoader &
+ fi
+fi
+
+# start node activity plugin
+tenmin=`echo ${min} | cut -b 2-`
+if [ "X${tenmin}" == "X0" ]; then
+ pidFile=$CHUKWA_HOME/var/run/NodeActivityPlugin.pid
+ if [ -f $pidFile ]; then
+ pid=`head ${pidFile}`
+ ChildPIDRunningStatus=`${jps} | grep ${pid} | grep NodeActivityMDL | wc -l`
+ if [ $ChildPIDRunningStatus -lt 1 ]; then
+ ${java} -DCHUKWA_HOME=${CHUKWA_HOME} -Dlog4j.configuration=nodeActivity.properties -classpath ${CLASSPATH}:${chukwa}:${ikit}:${common} org.apache.hadoop.chukwa.sources.plugin.nodeactivity.NodeActivityMDL&
+ fi
+ else
+ ${java} -DCHUKWA_HOME=${CHUKWA_HOME} -Dlog4j.configuration=nodeActivity.properties -classpath ${CLASSPATH}:${chukwa}:${ikit}:${common} org.apache.hadoop.chukwa.sources.plugin.nodeactivity.NodeActivityMDL&
+ fi
+fi
+
+# start database summary loader
+tenmin=`echo ${min} | cut -b 2-`
+if [ "X${tenmin}" == "X0" ]; then
+ pidFile=$CHUKWA_HOME/var/run/DBSummaryLoader.pid
+ if [ -f $pidFile ]; then
+ pid=`head ${pidFile}`
+ ChildPIDRunningStatus=`${jps} | grep ${pid} | grep DBSummaryLoader | wc -l`
+ if [ $ChildPIDRunningStatus -lt 1 ]; then
+ ${java} -Xms128m -Xmx1280m -DCHUKWA_HOME=${CHUKWA_HOME} -Dlog4j.configuration=log4j.properties -classpath ${CLASSPATH}:${chukwa}:${ikit}:${ckit}:${common} org.apache.hadoop.chukwa.extraction.DBSummaryLoader &
+ fi
+ else
+ ${java} -Xms128m -Xmx1280m -DCHUKWA_HOME=${CHUKWA_HOME} -Dlog4j.configuration=log4j.properties -classpath ${CLASSPATH}:${chukwa}:${ikit}:${ckit}:${common} org.apache.hadoop.chukwa.extraction.DBSummaryLoader &
+ fi
+fi
+
Propchange: hadoop/core/trunk/src/contrib/chukwa/bin/watchdog.sh
------------------------------------------------------------------------------
svn:executable = *
Added: hadoop/core/trunk/src/contrib/chukwa/build.xml
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/build.xml?rev=685353&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/build.xml (added)
+++ hadoop/core/trunk/src/contrib/chukwa/build.xml Tue Aug 12 15:35:16 2008
@@ -0,0 +1,359 @@
+<?xml version="1.0" ?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<project name="chukwa" default="main">
+
+ <property name="chukwaVersion" value="0.0.1"/>
+
+ <property name="lib.dir" value="${basedir}/lib"/>
+ <property name="src.dir" value="${basedir}/src"/>
+ <property name="conf.dir" value="${basedir}/conf"/>
+ <property name="docs.dir" value="${basedir}/docs"/>
+ <property name="tools.dir" value="${basedir}/tools"/>
+ <property name="dist.dir" value="${basedir}/dist"/>
+ <property name="opt.dir" value="${basedir}/opt"/>
+ <property name="javac.debug" value="on"/>
+
+ <import file="../build-contrib.xml"/>
+
+ <target name="init">
+ <mkdir dir="${build.dir}"/>
+ <mkdir dir="${build.dir}/test"/>
+ <!--
+ <copy todir="${build.dir}">
+ <fileset dir="${basedir}">
+ <exclude name="**/VERSION"/>
+ </fileset>
+ </copy>
+
+-->
+ <exec executable="echo" output="${basedir}/bin/VERSION">
+ <arg line="${chukwaVersion}" />
+ </exec>
+
+ <mkdir dir="${dist.dir}"/>
+ <property name="final.name" value="${name}-${chukwaVersion}"/>
+
+ </target>
+
+ <target name="main" depends="init, compile, compress" description="Main target">
+ <echo>
+ Building the .jar files.
+ </echo>
+ </target>
+
+ <target name="compile" depends="init" description="Compilation target">
+ <javac srcdir="src/java/org/apache/hadoop/chukwa" destdir="${build.dir}" excludes="**/ChukwaTTInstru.java" debug="${javac.debug}">
+ <classpath refid="chukwaHadoopClasspath" />
+ </javac>
+ </target>
+
+ <target name="test" depends="init" description="Test target">
+ <javac srcdir="src/test/org/apache/hadoop/chukwa" destdir="${build.dir}" excludes="**/TestDemux.java" debug="${javac.debug}">
+ <classpath refid="chukwaHadoopClasspath" />
+ </javac>
+ </target>
+
+ <target name="collector" depends="compile" description="Prepare collector.war">
+ <mkdir dir="${build.dir}/collector"/>
+ <mkdir dir="${build.dir}/collector/WEB-INF"/>
+ <mkdir dir="${build.dir}/collector/WEB-INF/classes"/>
+ <mkdir dir="${build.dir}/collector/WEB-INF/lib"/>
+ <mkdir dir="${build.dir}/collector/META-INF"/>
+ <copy todir="${build.dir}/collector/WEB-INF/classes">
+ <fileset dir="${build.dir}">
+ <include name="org/apache/hadoop/chukwa/conf/**/*.class" />
+ </fileset>
+ <fileset dir="${build.dir}">
+ <include name="org/apache/hadoop/chukwa/datacollection/**/*.class" />
+ </fileset>
+ <fileset dir="${build.dir}">
+ <include name="org/apache/hadoop/chukwa/extraction/**/*.class" />
+ </fileset>
+ <fileset dir="${build.dir}">
+ <include name="org/apache/hadoop/chukwa/util/**/*.class" />
+ </fileset>
+ <fileset dir="${basedir}/src/java">
+ <include name="org/apache/hadoop/chukwa/conf/**/*.java" />
+ </fileset>
+ <fileset dir="${basedir}/src/java">
+ <include name="org/apache/hadoop/chukwa/datacollection/**/*.java" />
+ </fileset>
+ <fileset dir="${basedir}/src/java">
+ <include name="org/apache/hadoop/chukwa/extraction/**/*.java" />
+ </fileset>
+ <fileset dir="${basedir}/src/java">
+ <include name="org/apache/hadoop/chukwa/util/**/*.java" />
+ </fileset>
+ </copy>
+
+ <copy todir="${build.dir}/collector">
+ <fileset dir="${basedir}/src/web/collector">
+ <include name="**" />
+ </fileset>
+ </copy>
+
+ <copy todir="${build.dir}/collector/WEB-INF/lib">
+ <fileset dir="${basedir}/lib">
+ <include name="log4j-1.2.13.jar" />
+ <include name="${build.dir}/${final.name}-core.jar" />
+ </fileset>
+ </copy>
+ </target>
+
+ <target name="collector_jar" depends="compile, collector" description="Create collector jar">
+ <jar jarfile="${build.dir}/collector-${chukwaVersion}.war" basedir="${build.dir}/collector">
+ <fileset dir="${build.dir}/collector" includes="**" />
+ </jar>
+ </target>
+ <target name="tools_jar" depends="compile, collector" description="Create tools jar">
+ <jar jarfile="${build.dir}/tools-${chukwaVersion}.jar" basedir="${build.dir}" includes="org/apache/hadoop/chukwa/inputtools/**/*.class">
+ <fileset dir="${basedir}/src/java">
+ <include name="org/apache/hadoop/chukwa/inputtools/**/*.java"/>
+ </fileset>
+ </jar>
+ </target>
+
+ <target name="agent_jar" depends="compile, collector" description="Create agent jar">
+ <jar jarfile="${build.dir}/chukwa-agent-${chukwaVersion}.jar" basedir="${build.dir}" includes="org/apache/hadoop/chukwa/client/**/*.class" >
+ <fileset dir="${build.dir}">
+ <include name="org/apache/hadoop/chukwa/conf/**/*.class"/>
+ </fileset>
+ <fileset dir="${build.dir}">
+ <include name="org/apache/hadoop/chukwa/util/**/*.class"/>
+ </fileset>
+ <fileset dir="${build.dir}">
+ <include name="org/apache/hadoop/chukwa/inputtools/**/*.class"/>
+ </fileset>
+ <fileset dir="${build.dir}">
+ <include name="org/apache/hadoop/chukwa/datacollection/**/*.class"/>
+ </fileset>
+
+ <fileset dir="${basedir}/src/java">
+ <include name="org/apache/hadoop/chukwa/client/**/*.java"/>
+ </fileset>
+ <fileset dir="${basedir}/src/java">
+ <include name="org/apache/hadoop/chukwa/util/**/*.java"/>
+ </fileset>
+ <fileset dir="${basedir}/src/java">
+ <include name="org/apache/hadoop/chukwa/inputtools/**/*.java"/>
+ </fileset>
+ <fileset dir="${basedir}/src/java">
+ <include name="org/apache/hadoop/chukwa/inputtools/**/*.java"/>
+ </fileset>
+ <fileset dir="${basedir}/src/java">
+ <include name="org/apache/hadoop/chukwa/datacollection/**/*.java"/>
+ <exclude name="org/apache/hadoop/chukwa/datacollection/collector/**/*.java"/>
+ <exclude name="org/apache/hadoop/chukwa/datacollection/writer/**/*.java"/>
+ </fileset>
+
+ </jar>
+ </target>
+
+ <target name="chukwa_jar" depends="compile, collector" description="Create chukwa-core jar">
+ <jar jarfile="${build.dir}/chukwa-core-${chukwaVersion}.jar" basedir="${build.dir}" includes="org/apache/hadoop/chukwa/datacollection/**/*.class" >
+ <manifest>
+ <section name="org/apache/hadoop/chukwa">
+ <attribute name="Implementation-Title" value="Chukwa"/>
+ <attribute name="Implementation-Version" value="${version}"/>
+ <attribute name="Implementation-Vendor" value="Apache"/>
+ </section>
+ </manifest>
+
+ <fileset dir="${build.dir}">
+ <include name="org/apache/hadoop/chukwa/**/*.class"/>
+ </fileset>
+ <fileset dir="${basedir}/src/java">
+ <include name="org/apache/hadoop/chukwa/**/*.java"/>
+ </fileset>
+ </jar>
+ </target>
+
+ <target name="chukwa-hadoop_jar" depends="compile" description="Create chukwa_hadoop jar for use with getting hadoop to use chukwa">
+
+ <jar jarfile="${build.dir}/chukwa-hadoop-${chukwaVersion}-client.jar" basedir="${build.dir}" includes="org/apache/hadoop/chukwa/inputtools/log4j/**/*.class">
+ <fileset dir="${basedir}/src/java">
+ <include name="org/apache/hadoop/chukwa/inputtools/log4j/**/*.java"/>
+ <include name="org/apache/hadoop/chukwa/datacollection/client/**/*.java"/>
+ <include name="org/apache/hadoop/chukwa/util/**/*.java"/>
+ </fileset>
+ <fileset dir="${build.dir}">
+ <include name="org/apache/hadoop/chukwa/datacollection/client/**/*.class"/>
+ <include name="org/apache/hadoop/chukwa/util/**/*.class"/>
+ <include name="org/apache/hadoop/chukwa/datacollection/controller/*.class"/>
+ </fileset>
+ </jar>
+ </target>
+
+ <target name="compress" depends="compile,collector,collector_jar,tools_jar,agent_jar,chukwa_jar,chukwa-hadoop_jar" description="Compression target">
+ <!-- <jar jarfile="${build.dir}/chukwa.jar" basedir="${build.dir}" includes="org/apache/hadoop/chukwa/**/*.class" /> -->
+
+ <copy todir="." includeEmptyDirs="false">
+ <fileset dir="${build.dir}">
+ <exclude name="**" />
+ <include name="**/*.jar" />
+ </fileset>
+ </copy>
+ </target>
+
+ <!-- ====================================================== -->
+ <!-- Macro definitions -->
+ <!-- ====================================================== -->
+ <macrodef name="macro_tar" description="Worker Macro for tar">
+ <attribute name="param.destfile"/>
+ <element name="param.listofitems"/>
+ <sequential>
+ <tar compression="gzip" longfile="gnu"
+ destfile="@{param.destfile}">
+ <param.listofitems/>
+ </tar>
+ </sequential>
+ </macrodef>
+
+ <!-- ================================================================== -->
+ <!-- D I S T R I B U T I O N -->
+ <!-- ================================================================== -->
+ <!-- -->
+ <!-- ================================================================== -->
+ <target name="package" depends="compress" description="Build distribution">
+ <mkdir dir="${build.dir}/${final.name}"/>
+ <mkdir dir="${build.dir}/${final.name}/lib"/>
+ <mkdir dir="${build.dir}/${final.name}/bin"/>
+ <mkdir dir="${build.dir}/${final.name}/opt"/>
+ <mkdir dir="${build.dir}/${final.name}/tools"/>
+ <mkdir dir="${build.dir}/${final.name}/var"/>
+ <mkdir dir="${build.dir}/${final.name}/var/run"/>
+ <mkdir dir="${build.dir}/${final.name}/logs"/>
+
+ <copy todir="${build.dir}/${final.name}" includeEmptyDirs="false">
+ <fileset dir="${build.dir}">
+ <include name="**/*.jar" />
+ <include name="**/*.war" />
+ </fileset>
+ </copy>
+ <copy todir="${build.dir}/${final.name}/lib" includeEmptyDirs="false">
+ <fileset dir="lib">
+ <exclude name="**/native/**"/>
+ </fileset>
+ </copy>
+
+ <exec dir="${dist.dir}" executable="sh" failonerror="true">
+ <env key="BASE_NATIVE_LIB_DIR" value="${lib.dir}/native"/>
+ <env key="BUILD_NATIVE_DIR" value="${build.dir}/native"/>
+ <env key="DIST_LIB_DIR" value="${dist.dir}/lib/native"/>
+ </exec>
+
+ <copy todir="${build.dir}/${final.name}/bin">
+ <fileset dir="bin"/>
+ </copy>
+
+ <copy todir="${build.dir}/${final.name}/conf">
+ <fileset dir="${conf.dir}" excludes="**/*.template"/>
+ </copy>
+
+ <copy todir="${build.dir}/${final.name}/docs">
+ <fileset dir="${docs.dir}" />
+ </copy>
+
+ <copy todir="${build.dir}/${final.name}/opt">
+ <fileset dir="${opt.dir}" />
+ </copy>
+
+ <copy todir="${build.dir}/${final.name}/tools">
+ <fileset dir="${tools.dir}" />
+ </copy>
+
+ <chmod perm="ugo+x" type="file" parallel="false">
+ <fileset dir="${build.dir}/${final.name}/tools">
+ <include name="**" />
+ </fileset>
+ </chmod>
+
+ <copy todir="${build.dir}/${final.name}">
+ <fileset dir=".">
+ <include name="*.txt" />
+ </fileset>
+ </copy>
+
+ <copy todir="${build.dir}/${final.name}/src" includeEmptyDirs="true">
+ <fileset dir="src" excludes="**/*.template **/docs/build/**/*"/>
+ </copy>
+
+ <copy todir="${build.dir}/${final.name}" file="build.xml"/>
+
+ <chmod perm="ugo+x" type="file" parallel="false">
+ <fileset dir="${build.dir}/${final.name}/bin">
+ <include name="*/bin/*" />
+ </fileset>
+ </chmod>
+
+ </target>
+
+ <!-- ================================================================== -->
+ <!-- Make release tarball -->
+ <!-- ================================================================== -->
+ <target name="tar" depends="package" description="Make release tarball">
+ <macro_tar param.destfile="${dist.dir}/${final.name}.tar.gz">
+ <param.listofitems>
+ <tarfileset dir="${build.dir}" mode="664">
+ <exclude name="${final.name}/org/*" />
+ <include name="${final.name}/**" />
+ </tarfileset>
+ <tarfileset dir="${build.dir}" mode="755">
+ <include name="${final.name}/bin/*" />
+ </tarfileset>
+ </param.listofitems>
+ </macro_tar>
+ </target>
+
+ <target name="binary" depends="package" description="Make tarball without source and documentation">
+ <macro_tar param.destfile="${dist.dir}/${final.name}-bin.tar.gz">
+ <param.listofitems>
+ <tarfileset dir="${build.dir}" mode="664">
+ <exclude name="${final.name}/src/**" />
+ <exclude name="${final.name}/docs/**" />
+ <include name="${final.name}/**" />
+ </tarfileset>
+ <tarfileset dir="${build.dir}" mode="755">
+ <include name="${final.name}/bin/*" />
+ </tarfileset>
+ </param.listofitems>
+ </macro_tar>
+ </target>
+
+ <!-- the normal classpath -->
+ <path id="classpath">
+ <pathelement location="${build.classes}"/>
+ <fileset dir="${lib.dir}">
+ <include name="**/*.jar" />
+ <exclude name="**/excluded/" />
+ </fileset>
+ <pathelement location="${conf.dir}"/>
+ </path>
+
+ <!-- the hadoop classpath -->
+ <path id="chukwaHadoopClasspath">
+ <pathelement location="${hadoop.root}/build/classes"/>
+ <fileset dir="${lib.dir}">
+ <include name="**/*.jar" />
+ <exclude name="**/excluded/" />
+ </fileset>
+ </path>
+
+</project>
Added: hadoop/core/trunk/src/contrib/chukwa/chukwafy_jobhist.awk
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/chukwafy_jobhist.awk?rev=685353&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/chukwafy_jobhist.awk (added)
+++ hadoop/core/trunk/src/contrib/chukwa/chukwafy_jobhist.awk Tue Aug 12 15:35:16 2008
@@ -0,0 +1,24 @@
+#!/usr/bin/gawk
+
+#A small awk script to take normal job history logs and chukwafy
+#In particular, they get prefixed with something of the form
+#"2008-07-28 23:30:38,865 INFO org.apache.hadoop.chukwa.ChukwaJobHistory:
+
+BEGIN {
+OFS = ""
+}
+
+{ # for each record
+ #get Timestamp
+for(i =1; i <= NF; ++i) {
+ if( $i ~ /_TIME=/) {
+ split($i, halves, "=");
+ ts_msec = substr(halves[2], 2 , length(halves[2]) - 2)
+ break;
+ }
+}
+if(ts_msec == 0)
+ print "WARNING: no timestamp in line " > /dev/stderr
+
+print strftime("%Y-%m-%d %H:%M:%S", ts_msec/1000) , "," , (ts_msec%1000) , " INFO org.apache.hadoop.chukwa.ChukwaJobHistory: " , $0
+}
\ No newline at end of file
Added: hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-agent-conf.xml
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-agent-conf.xml?rev=685353&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-agent-conf.xml (added)
+++ hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-agent-conf.xml Tue Aug 12 15:35:16 2008
@@ -0,0 +1,20 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="nutch-conf.xsl"?>
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+
+ <property>
+ <name>chukwaAgent.control.port</name>
+ <value>9093</value>
+ <description>The socket port number the agent's control interface can be contacted at.</description>
+ </property>
+
+ <property>
+ <name>chukwaAgent.hostname</name>
+ <value>localhost</value>
+ <description>The hostname of the agent on this node. Usually localhost, this is used by the chukwa instrumentation agent-control interface library</description>
+ </property>
+
+</configuration>