You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@chukwa.apache.org by as...@apache.org on 2009/03/11 23:39:32 UTC
svn commit: r752666 [16/16] - in /hadoop/chukwa/trunk: ./
src/java/org/apache/hadoop/chukwa/ src/java/org/apache/hadoop/chukwa/conf/
src/java/org/apache/hadoop/chukwa/database/
src/java/org/apache/hadoop/chukwa/datacollection/
src/java/org/apache/hadoo...
Modified: hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/validationframework/ChukwaAgentToCollectorValidator.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/validationframework/ChukwaAgentToCollectorValidator.java?rev=752666&r1=752665&r2=752666&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/validationframework/ChukwaAgentToCollectorValidator.java (original)
+++ hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/validationframework/ChukwaAgentToCollectorValidator.java Wed Mar 11 22:39:26 2009
@@ -1,8 +1,8 @@
package org.apache.hadoop.chukwa.validationframework;
+
import java.io.File;
import java.io.IOException;
-
import org.apache.hadoop.chukwa.conf.ChukwaConfiguration;
import org.apache.hadoop.chukwa.datacollection.agent.ChukwaAgent;
import org.apache.hadoop.chukwa.datacollection.agent.ChukwaAgent.AlreadyRunningException;
@@ -12,163 +12,142 @@
import org.apache.hadoop.chukwa.validationframework.interceptor.SetupTestClasses;
import org.apache.hadoop.chukwa.validationframework.util.DataOperations;
-public class ChukwaAgentToCollectorValidator
-{
- public static final int ADD = 100;
- public static final int VALIDATE = 200;
-
-
- private static void usage()
- {
- System.out.println("usage ...");
- System.exit(-1);
- }
- /**
- * @param args
- * @throws Throwable
- * @throws AlreadyRunningException
- * @throws IOException
- */
- public static void main(String[] args) throws Throwable
- {
- if (args.length != 2)
- {
- usage();
- }
-
-
- int command = -1;
-
- if ("-add".equalsIgnoreCase(args[0]))
- {
- command = ChukwaAgentToCollectorValidator.ADD;
- }
- else if ("-validate".equalsIgnoreCase(args[0]))
- {
- command = ChukwaAgentToCollectorValidator.VALIDATE;
- }
- else
- {
- usage();
- }
-
- String chukwaTestRepository = System.getenv("chukwaTestRepository");
- if (chukwaTestRepository == null)
- {
- chukwaTestRepository = "/tmp/chukwaTestRepository/";
- }
-
- if (!chukwaTestRepository.endsWith("/"))
- {
- chukwaTestRepository += "/";
- }
-
- String fileName = args[1];
-
- String name = null;
- if (fileName.indexOf("/") >= 0)
- {
- name = fileName.substring(fileName.lastIndexOf("/"));
- }
- else
- {
- name = fileName;
- }
-
- String chukwaTestDirectory = chukwaTestRepository + name ;
- String inputFile = chukwaTestDirectory + "/input/" + name;
- String outputDir = null;
-
- if (command == ChukwaAgentToCollectorValidator.ADD)
- {
- File dir = new File(chukwaTestDirectory + "/input/");
- if (dir.exists())
- {
- throw new RuntimeException("a test with the same input file is already there, remove it first");
- }
- dir.mkdirs();
- DataOperations.copyFile(fileName, inputFile);
- outputDir = "/gold";
- }
- else
- {
- outputDir = "/" + System.currentTimeMillis();
- }
-
- System.out.println("chukwaTestDirectory [" + chukwaTestDirectory + "]");
- System.out.println("command [" + ( (command == ChukwaAgentToCollectorValidator.ADD)?"ADD":"VALIDATE") + "]");
- System.out.println("fileName [" + inputFile + "]");
-
-
- ChukwaConfiguration conf = new ChukwaConfiguration(true);
- String collectorOutputDir = conf.get("chukwaCollector.outputDir");
-
-
- prepareAndSendData(chukwaTestDirectory+ outputDir,inputFile,collectorOutputDir);
- extractRawLog(chukwaTestDirectory+ outputDir,name,collectorOutputDir);
- boolean rawLogTestResult = validateRawLogs(chukwaTestDirectory+outputDir,name) ;
-
-
- boolean binLogTestResult = true;
-
- if (command == ChukwaAgentToCollectorValidator.VALIDATE)
- {
- binLogTestResult = validateOutputs(chukwaTestDirectory+outputDir,name);
- }
-
-
- if (rawLogTestResult == true && binLogTestResult == true)
- {
- System.out.println("test OK");
- System.exit(10);
- }
- else
- {
- System.out.println("test KO");
- throw new RuntimeException("test failed for file [" + name +"]" );
- }
- }
-
- public static void prepareAndSendData(String dataRootFolder,String inputFile,String dataSinkDirectory) throws Throwable
- {
-
- ChunkDumper.testRepositoryDumpDir = dataRootFolder + "/";
-
- SetupTestClasses.setupClasses();
-
- // clean up the collector outputDir.
- File collectorDir = new File(dataSinkDirectory);
- String[] files = collectorDir.list();
- for(String f: files)
- {
- File file = new File(dataSinkDirectory+ File.separator +f);
- file.delete();
- System.out.println("Deleting previous collectors files: " + f);
- }
-
- System.out.println("Starting agent");
+public class ChukwaAgentToCollectorValidator {
+ public static final int ADD = 100;
+ public static final int VALIDATE = 200;
+
+ private static void usage() {
+ System.out.println("usage ...");
+ System.exit(-1);
+ }
+
+ /**
+ * @param args
+ * @throws Throwable
+ * @throws AlreadyRunningException
+ * @throws IOException
+ */
+ public static void main(String[] args) throws Throwable {
+ if (args.length != 2) {
+ usage();
+ }
+
+ int command = -1;
+
+ if ("-add".equalsIgnoreCase(args[0])) {
+ command = ChukwaAgentToCollectorValidator.ADD;
+ } else if ("-validate".equalsIgnoreCase(args[0])) {
+ command = ChukwaAgentToCollectorValidator.VALIDATE;
+ } else {
+ usage();
+ }
+
+ String chukwaTestRepository = System.getenv("chukwaTestRepository");
+ if (chukwaTestRepository == null) {
+ chukwaTestRepository = "/tmp/chukwaTestRepository/";
+ }
+
+ if (!chukwaTestRepository.endsWith("/")) {
+ chukwaTestRepository += "/";
+ }
+
+ String fileName = args[1];
+
+ String name = null;
+ if (fileName.indexOf("/") >= 0) {
+ name = fileName.substring(fileName.lastIndexOf("/"));
+ } else {
+ name = fileName;
+ }
+
+ String chukwaTestDirectory = chukwaTestRepository + name;
+ String inputFile = chukwaTestDirectory + "/input/" + name;
+ String outputDir = null;
+
+ if (command == ChukwaAgentToCollectorValidator.ADD) {
+ File dir = new File(chukwaTestDirectory + "/input/");
+ if (dir.exists()) {
+ throw new RuntimeException(
+ "a test with the same input file is already there, remove it first");
+ }
+ dir.mkdirs();
+ DataOperations.copyFile(fileName, inputFile);
+ outputDir = "/gold";
+ } else {
+ outputDir = "/" + System.currentTimeMillis();
+ }
+
+ System.out.println("chukwaTestDirectory [" + chukwaTestDirectory + "]");
+ System.out.println("command ["
+ + ((command == ChukwaAgentToCollectorValidator.ADD) ? "ADD"
+ : "VALIDATE") + "]");
+ System.out.println("fileName [" + inputFile + "]");
+
+ ChukwaConfiguration conf = new ChukwaConfiguration(true);
+ String collectorOutputDir = conf.get("chukwaCollector.outputDir");
+
+ prepareAndSendData(chukwaTestDirectory + outputDir, inputFile,
+ collectorOutputDir);
+ extractRawLog(chukwaTestDirectory + outputDir, name, collectorOutputDir);
+ boolean rawLogTestResult = validateRawLogs(chukwaTestDirectory + outputDir,
+ name);
+
+ boolean binLogTestResult = true;
+
+ if (command == ChukwaAgentToCollectorValidator.VALIDATE) {
+ binLogTestResult = validateOutputs(chukwaTestDirectory + outputDir, name);
+ }
+
+ if (rawLogTestResult == true && binLogTestResult == true) {
+ System.out.println("test OK");
+ System.exit(10);
+ } else {
+ System.out.println("test KO");
+ throw new RuntimeException("test failed for file [" + name + "]");
+ }
+ }
+
+ public static void prepareAndSendData(String dataRootFolder,
+ String inputFile, String dataSinkDirectory) throws Throwable {
+
+ ChunkDumper.testRepositoryDumpDir = dataRootFolder + "/";
+
+ SetupTestClasses.setupClasses();
+
+ // clean up the collector outputDir.
+ File collectorDir = new File(dataSinkDirectory);
+ String[] files = collectorDir.list();
+ for (String f : files) {
+ File file = new File(dataSinkDirectory + File.separator + f);
+ file.delete();
+ System.out.println("Deleting previous collectors files: " + f);
+ }
+
+ System.out.println("Starting agent");
String[] agentArgs = new String[0];
ChukwaAgent.main(agentArgs);
-
+
// Start the collector
System.out.println("Starting collector");
CollectorStub.main(new String[0]);
-
+
// Start the agent
ChukwaAgent agent = ChukwaAgent.getAgent();
-
-
+
int portno = 9093; // Default
- ChukwaAgentController cli = new ChukwaAgentController("localhost",portno);
- // ADD
- // org.apache.hadoop.chukwa.datacollection.adaptor.filetailer.CharFileTailingAdaptorUTF8NewLineEscaped
- // SysLog
- // 0 /var/log/messages
+ ChukwaAgentController cli = new ChukwaAgentController("localhost", portno);
+ // ADD
+ // org.apache.hadoop.chukwa.datacollection.adaptor.filetailer.
+ // CharFileTailingAdaptorUTF8NewLineEscaped
+ // SysLog
+ // 0 /var/log/messages
// 0
System.out.println("Adding adaptor");
- long adaptor = cli.add("org.apache.hadoop.chukwa.datacollection.adaptor.filetailer.CharFileTailingAdaptorUTF8NewLineEscaped",
- "AutomatedTestType", "0 " + inputFile, 0);
-
+ long adaptor = cli
+ .add(
+ "org.apache.hadoop.chukwa.datacollection.adaptor.filetailer.CharFileTailingAdaptorUTF8NewLineEscaped",
+ "AutomatedTestType", "0 " + inputFile, 0);
+
cli.remove(adaptor);
System.out.println("Adaptor removed");
agent.shutdown();
@@ -176,94 +155,99 @@
CollectorStub.jettyServer.stop();
System.out.println("Shutting down collector");
Thread.sleep(2000);
- }
-
- public static void extractRawLog(String dataRootFolder,String fileName,String dataSinkDirectory) throws Exception
- {
- // Adaptor output
- DataOperations.extractRawLogFromDump(dataRootFolder + "/adaptor/", fileName);
- // Sender output
- DataOperations.extractRawLogFromDump(dataRootFolder + "/sender/", fileName);
-
- // Collector output
- File dir = new File(dataRootFolder + "/collector/");
- dir.mkdirs();
-
- File dataSinkDir = new File(dataSinkDirectory);
- String[] doneFiles = dataSinkDir.list();
+ }
+
+ public static void extractRawLog(String dataRootFolder, String fileName,
+ String dataSinkDirectory) throws Exception {
+ // Adaptor output
+ DataOperations
+ .extractRawLogFromDump(dataRootFolder + "/adaptor/", fileName);
+ // Sender output
+ DataOperations.extractRawLogFromDump(dataRootFolder + "/sender/", fileName);
+
+ // Collector output
+ File dir = new File(dataRootFolder + "/collector/");
+ dir.mkdirs();
+
+ File dataSinkDir = new File(dataSinkDirectory);
+ String[] doneFiles = dataSinkDir.list();
// Move done file to the final directory
- for(String f: doneFiles)
- {
+ for (String f : doneFiles) {
String outputFile = null;
- if (f.endsWith(".done"))
- {
- outputFile = fileName +".done";
- }
- else
- {
+ if (f.endsWith(".done")) {
+ outputFile = fileName + ".done";
+ } else {
outputFile = fileName + ".crc";
}
- System.out.println("Moving that file [" + dataSinkDirectory+ File.separator +f + "] to [" +dataRootFolder + "/collector/" + outputFile +"]");
- DataOperations.copyFile(dataSinkDirectory+ File.separator +f, dataRootFolder + "/collector/" + outputFile);
- }
-
- DataOperations.extractRawLogFromdataSink(ChunkDumper.testRepositoryDumpDir + "/collector/",fileName);
- }
-
-
- public static boolean validateRawLogs(String dataRootFolder,String fileName)
- {
- boolean result = true;
- // Validate Adaptor
- boolean adaptorMD5 = DataOperations.validateMD5(dataRootFolder + "/../input/" + fileName, dataRootFolder + "/adaptor/" + fileName + ".raw");
- if (!adaptorMD5)
- {
- System.out.println("Adaptor validation failed");
- result = false;
- }
- // Validate Sender
- boolean senderMD5 = DataOperations.validateMD5(dataRootFolder + "/../input/" + fileName, dataRootFolder + "/sender/" + fileName + ".raw");
- if (!senderMD5)
- {
- System.out.println("Sender validation failed");
- result = false;
- }
- // Validate DataSink
- boolean collectorMD5 = DataOperations.validateMD5(dataRootFolder + "/../input/" + fileName , dataRootFolder + "/collector/" + fileName + ".raw");
- if (!collectorMD5)
- {
- System.out.println("collector validation failed");
- result = false;
- }
-
- return result;
- }
-
- public static boolean validateOutputs(String dataRootFolder,String fileName)
- {
- boolean result = true;
- // Validate Adaptor
- boolean adaptorMD5 = DataOperations.validateMD5(dataRootFolder + "/../gold/adaptor/" + fileName + ".bin", dataRootFolder + "/adaptor/" + fileName + ".bin");
- if (!adaptorMD5)
- {
- System.out.println("Adaptor bin validation failed");
- result = false;
- }
- // Validate Sender
- boolean senderMD5 = DataOperations.validateMD5(dataRootFolder + "/../gold/sender/" + fileName+ ".bin", dataRootFolder + "/sender/" + fileName + ".bin");
- if (!senderMD5)
- {
- System.out.println("Sender bin validation failed");
- result = false;
- }
- // Validate DataSink
-// boolean collectorMD5 = DataOperations.validateRawLog(dataRootFolder + "/../gold/collector/" + fileName + ".done", dataRootFolder + "/collector/" + fileName + ".done");
-// if (!collectorMD5)
-// {
-// System.out.println("collector bin validation failed");
-// result = false;
-// }
-
- return result;
- }
+ System.out.println("Moving that file [" + dataSinkDirectory
+ + File.separator + f + "] to [" + dataRootFolder + "/collector/"
+ + outputFile + "]");
+ DataOperations.copyFile(dataSinkDirectory + File.separator + f,
+ dataRootFolder + "/collector/" + outputFile);
+ }
+
+ DataOperations.extractRawLogFromdataSink(ChunkDumper.testRepositoryDumpDir
+ + "/collector/", fileName);
+ }
+
+ public static boolean validateRawLogs(String dataRootFolder, String fileName) {
+ boolean result = true;
+ // Validate Adaptor
+ boolean adaptorMD5 = DataOperations.validateMD5(dataRootFolder
+ + "/../input/" + fileName, dataRootFolder + "/adaptor/" + fileName
+ + ".raw");
+ if (!adaptorMD5) {
+ System.out.println("Adaptor validation failed");
+ result = false;
+ }
+ // Validate Sender
+ boolean senderMD5 = DataOperations.validateMD5(dataRootFolder
+ + "/../input/" + fileName, dataRootFolder + "/sender/" + fileName
+ + ".raw");
+ if (!senderMD5) {
+ System.out.println("Sender validation failed");
+ result = false;
+ }
+ // Validate DataSink
+ boolean collectorMD5 = DataOperations.validateMD5(dataRootFolder
+ + "/../input/" + fileName, dataRootFolder + "/collector/" + fileName
+ + ".raw");
+ if (!collectorMD5) {
+ System.out.println("collector validation failed");
+ result = false;
+ }
+
+ return result;
+ }
+
+ public static boolean validateOutputs(String dataRootFolder, String fileName) {
+ boolean result = true;
+ // Validate Adaptor
+ boolean adaptorMD5 = DataOperations.validateMD5(dataRootFolder
+ + "/../gold/adaptor/" + fileName + ".bin", dataRootFolder + "/adaptor/"
+ + fileName + ".bin");
+ if (!adaptorMD5) {
+ System.out.println("Adaptor bin validation failed");
+ result = false;
+ }
+ // Validate Sender
+ boolean senderMD5 = DataOperations.validateMD5(dataRootFolder
+ + "/../gold/sender/" + fileName + ".bin", dataRootFolder + "/sender/"
+ + fileName + ".bin");
+ if (!senderMD5) {
+ System.out.println("Sender bin validation failed");
+ result = false;
+ }
+ // Validate DataSink
+ // boolean collectorMD5 = DataOperations.validateRawLog(dataRootFolder +
+ // "/../gold/collector/" + fileName + ".done", dataRootFolder +
+ // "/collector/" + fileName + ".done");
+ // if (!collectorMD5)
+ // {
+ // System.out.println("collector bin validation failed");
+ // result = false;
+ // }
+
+ return result;
+ }
}
\ No newline at end of file
Modified: hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/validationframework/DemuxDirectoryValidator.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/validationframework/DemuxDirectoryValidator.java?rev=752666&r1=752665&r2=752666&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/validationframework/DemuxDirectoryValidator.java (original)
+++ hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/validationframework/DemuxDirectoryValidator.java Wed Mar 11 22:39:26 2009
@@ -18,9 +18,9 @@
package org.apache.hadoop.chukwa.validationframework;
+
import java.io.File;
import java.net.URI;
-
import org.apache.hadoop.chukwa.conf.ChukwaConfiguration;
import org.apache.hadoop.chukwa.validationframework.util.DataOperations;
import org.apache.hadoop.conf.Configuration;
@@ -28,173 +28,158 @@
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-public class DemuxDirectoryValidator
-{
+public class DemuxDirectoryValidator {
static Configuration conf = null;
static FileSystem fs = null;
-
- public static void usage()
- {
+
+ public static void usage() {
System.out.println("Usage ...");
System.exit(-1);
}
-
- public static void validate(boolean isLocal,FileSystem fs,Configuration conf,String[] directories)
- {
+
+ public static void validate(boolean isLocal, FileSystem fs,
+ Configuration conf, String[] directories) {
DemuxDirectoryValidator.fs = fs;
DemuxDirectoryValidator.conf = conf;
- try
- {
- if (isLocal)
- {
- compareLocalDirectory(directories[0],directories[1]);
- }
- else
- {
+ try {
+ if (isLocal) {
+ compareLocalDirectory(directories[0], directories[1]);
+ } else {
DemuxDirectoryValidator.fs = fs;
- compareHDFSDirectory(directories[0],directories[1]);
+ compareHDFSDirectory(directories[0], directories[1]);
}
- }
- catch(Exception e)
- {
+ } catch (Exception e) {
e.printStackTrace();
- throw new RuntimeException ("Validation failed! ["
- + directories[0] +"][" + directories[1] + "]" ,e);
+ throw new RuntimeException("Validation failed! [" + directories[0] + "]["
+ + directories[1] + "]", e);
}
}
-
+
/**
* @param args
*/
- public static void main(String[] args)
- {
+ public static void main(String[] args) {
- if (args.length != 3)
- {
+ if (args.length != 3) {
usage();
}
-
+
String demuxGoldDirectory = args[1];
String demuxTestDirectory = args[2];
boolean isLocal = true;
-
- if ("-local".equalsIgnoreCase(args[0]))
- {
- compareLocalDirectory(demuxGoldDirectory,demuxTestDirectory);
- }
- else if ("-hdfs".equalsIgnoreCase(args[0]))
- {
+
+ if ("-local".equalsIgnoreCase(args[0])) {
+ compareLocalDirectory(demuxGoldDirectory, demuxTestDirectory);
+ } else if ("-hdfs".equalsIgnoreCase(args[0])) {
isLocal = false;
conf = new ChukwaConfiguration();
String fsName = conf.get("writer.hdfs.filesystem");
- try
- {
+ try {
fs = FileSystem.get(new URI(fsName), conf);
- } catch (Exception e)
- {
+ } catch (Exception e) {
e.printStackTrace();
throw new RuntimeException(e);
- }
- }
- else
- {
+ }
+ } else {
System.out.println("Wrong first argument");
usage();
}
-
- String[] dirs = { demuxGoldDirectory,demuxTestDirectory};
- validate(isLocal,fs,conf,dirs);
-
+
+ String[] dirs = { demuxGoldDirectory, demuxTestDirectory };
+ validate(isLocal, fs, conf, dirs);
+
System.out.println("Gold and test directories are equivalent");
System.exit(10);
}
-
- public static void compareHDFSDirectory(String gold,String test)
- {
- try
- {
+
+ public static void compareHDFSDirectory(String gold, String test) {
+ try {
Path goldDirectory = new Path(gold);
FileStatus[] goldFiles = fs.listStatus(goldDirectory);
- // Path testDirectory = new Path(test);
-// FileStatus[] testFiles = fs.listStatus(testDirectory);
-//
-
- for(int i=0;i<goldFiles.length;i++)
- {
-
- //Skip the crc files
- if (goldFiles[i].getPath().getName().endsWith(".crc") )
- { continue; }
-
- System.out.println("Testing [" + goldFiles[i].getPath().getName().intern() + "]" );
-
-// if (goldFiles[i].getPath().getName().intern() != testFiles[i].getPath().getName().intern())
-// {
-// throw new RuntimeException("Gold & test dirrectories [" + gold +"/" +goldFiles[i].getPath().getName() +"] are not the same");
-// }
-
- if (goldFiles[i].isDir())
- {
- //Skip the _logs directory
- if (goldFiles[i].getPath().getName().equalsIgnoreCase("_logs"))
- { continue; }
-
- compareHDFSDirectory(gold +"/" +goldFiles[i].getPath().getName(),test +"/" +goldFiles[i].getPath().getName());
+ // Path testDirectory = new Path(test);
+ // FileStatus[] testFiles = fs.listStatus(testDirectory);
+ //
+
+ for (int i = 0; i < goldFiles.length; i++) {
+
+ // Skip the crc files
+ if (goldFiles[i].getPath().getName().endsWith(".crc")) {
+ continue;
}
- else
- {
- boolean isTheSme = DataOperations.validateChukwaRecords(fs,conf,goldFiles[i].getPath(), new Path(test + "/" +goldFiles[i].getPath().getName()));
- if (!isTheSme)
- {
-// System.out.println("MD5 failed on [" + gold +"/" +goldFiles[i] +"]");
- throw new RuntimeException("ChukwaRecords validation error: for Gold & test [" + gold +"/" +goldFiles[i].getPath().getName()+"] ["
- + test +"/" +goldFiles[i].getPath().getName()+ "] are not the same");
+
+ System.out.println("Testing ["
+ + goldFiles[i].getPath().getName().intern() + "]");
+
+ // if (goldFiles[i].getPath().getName().intern() !=
+ // testFiles[i].getPath().getName().intern())
+ // {
+ // throw new RuntimeException("Gold & test dirrectories [" + gold +"/"
+ // +goldFiles[i].getPath().getName() +"] are not the same");
+ // }
+
+ if (goldFiles[i].isDir()) {
+ // Skip the _logs directory
+ if (goldFiles[i].getPath().getName().equalsIgnoreCase("_logs")) {
+ continue;
+ }
+
+ compareHDFSDirectory(gold + "/" + goldFiles[i].getPath().getName(),
+ test + "/" + goldFiles[i].getPath().getName());
+ } else {
+ boolean isTheSme = DataOperations.validateChukwaRecords(fs, conf,
+ goldFiles[i].getPath(), new Path(test + "/"
+ + goldFiles[i].getPath().getName()));
+ if (!isTheSme) {
+ // System.out.println("MD5 failed on [" + gold +"/" +goldFiles[i]
+ // +"]");
+ throw new RuntimeException(
+ "ChukwaRecords validation error: for Gold & test [" + gold
+ + "/" + goldFiles[i].getPath().getName() + "] [" + test
+ + "/" + goldFiles[i].getPath().getName()
+ + "] are not the same");
}
}
}
- } catch (Exception e)
- {
+ } catch (Exception e) {
e.printStackTrace();
throw new RuntimeException(e);
}
}
- public static void compareLocalDirectory(String gold,String test)
- {
+ public static void compareLocalDirectory(String gold, String test) {
File goldDirectory = new File(gold);
String[] goldFiles = goldDirectory.list();
File testDirectory = new File(test);
String[] testFiles = testDirectory.list();
-
- for(int i=0;i<goldFiles.length;i++)
- {
- if (goldFiles[i].intern() != testFiles[i].intern())
- {
- throw new RuntimeException("Gold & test dirrectories [" + gold +"/" +goldFiles[i] +"] are not the same");
- }
- File g = new File(gold +"/" +goldFiles[i]);
- if (g.isDirectory())
- {
- //Skip the _logs directory
- if (goldFiles[i].equalsIgnoreCase("_logs"))
- { continue; }
-
- compareLocalDirectory(gold +"/" +goldFiles[i],test +"/" +goldFiles[i]);
- }
- else
- {
- boolean md5 = DataOperations.validateMD5(gold +"/" +goldFiles[i], test +"/" +goldFiles[i]);
- if (!md5)
- {
-// System.out.println("MD5 failed on [" + gold +"/" +goldFiles[i] +"]");
- throw new RuntimeException("MD5 for Gold & test [" + gold +"/" +goldFiles[i]+"] are not the same");
+
+ for (int i = 0; i < goldFiles.length; i++) {
+ if (goldFiles[i].intern() != testFiles[i].intern()) {
+ throw new RuntimeException("Gold & test dirrectories [" + gold + "/"
+ + goldFiles[i] + "] are not the same");
+ }
+ File g = new File(gold + "/" + goldFiles[i]);
+ if (g.isDirectory()) {
+ // Skip the _logs directory
+ if (goldFiles[i].equalsIgnoreCase("_logs")) {
+ continue;
+ }
+
+ compareLocalDirectory(gold + "/" + goldFiles[i], test + "/"
+ + goldFiles[i]);
+ } else {
+ boolean md5 = DataOperations.validateMD5(gold + "/" + goldFiles[i],
+ test + "/" + goldFiles[i]);
+ if (!md5) {
+ // System.out.println("MD5 failed on [" + gold +"/" +goldFiles[i]
+ // +"]");
+ throw new RuntimeException("MD5 for Gold & test [" + gold + "/"
+ + goldFiles[i] + "] are not the same");
}
}
}
-
-
+
}
}
Modified: hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/validationframework/interceptor/ChunkDumper.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/validationframework/interceptor/ChunkDumper.java?rev=752666&r1=752665&r2=752666&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/validationframework/interceptor/ChunkDumper.java (original)
+++ hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/validationframework/interceptor/ChunkDumper.java Wed Mar 11 22:39:26 2009
@@ -1,5 +1,6 @@
package org.apache.hadoop.chukwa.validationframework.interceptor;
+
import java.io.DataOutputStream;
import java.io.File;
import java.io.FileNotFoundException;
@@ -7,73 +8,62 @@
import java.io.IOException;
import java.util.HashMap;
import java.util.Iterator;
-
import org.apache.hadoop.chukwa.Chunk;
-public class ChunkDumper
-{
- static public String testRepositoryDumpDir = "/tmp/chukwaDump/";
- static HashMap<String, DataOutputStream> hash = new HashMap<String, DataOutputStream>();
-
- public static void dump(String component,Chunk chunk)
- {
-
- String fileName = chunk.getApplication();
-
- if (!hash.containsKey(component + "-" +fileName))
- {
- File directory = new File(testRepositoryDumpDir+ "/"+ component );
- if (!directory.exists())
- {
- directory.mkdirs();
- }
- String name = fileName;
- if (fileName.indexOf("/") >= 0)
- {
- name = fileName.substring(fileName.lastIndexOf("/"));
- }
- name += ".bin";
-
- synchronized(name.intern())
- {
- System.out.println("FileName [" + name + "]");
- try
- {
- DataOutputStream dos=new DataOutputStream(new FileOutputStream(new File(testRepositoryDumpDir+ "/"+ component + "/" + name)));
- System.out.println("Writing to [" + testRepositoryDumpDir+ "/"+ component + "/" + name + "]");
- hash.put(component + "-" +fileName, dos);
- } catch (FileNotFoundException e)
- {
- e.printStackTrace();
- }
- }
- }
- String key = component + "-" +fileName;
- synchronized(key.intern())
- {
- DataOutputStream dos = hash.get(key);
- try
- {
- chunk.write(dos);
- dos.flush();
- }
- catch (IOException e)
- {
- e.printStackTrace();
- }
- }
- }
-
- static void close()
- {
- Iterator<String> it = hash.keySet().iterator();
- while(it.hasNext())
- {
- String key = it.next();
- DataOutputStream dos = hash.get(key);
- try{ dos.close();}
- catch (Exception e)
- { e.printStackTrace();}
- }
- }
+public class ChunkDumper {
+ static public String testRepositoryDumpDir = "/tmp/chukwaDump/";
+ static HashMap<String, DataOutputStream> hash = new HashMap<String, DataOutputStream>();
+
+ public static void dump(String component, Chunk chunk) {
+
+ String fileName = chunk.getApplication();
+
+ if (!hash.containsKey(component + "-" + fileName)) {
+ File directory = new File(testRepositoryDumpDir + "/" + component);
+ if (!directory.exists()) {
+ directory.mkdirs();
+ }
+ String name = fileName;
+ if (fileName.indexOf("/") >= 0) {
+ name = fileName.substring(fileName.lastIndexOf("/"));
+ }
+ name += ".bin";
+
+ synchronized (name.intern()) {
+ System.out.println("FileName [" + name + "]");
+ try {
+ DataOutputStream dos = new DataOutputStream(new FileOutputStream(
+ new File(testRepositoryDumpDir + "/" + component + "/" + name)));
+ System.out.println("Writing to [" + testRepositoryDumpDir + "/"
+ + component + "/" + name + "]");
+ hash.put(component + "-" + fileName, dos);
+ } catch (FileNotFoundException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ String key = component + "-" + fileName;
+ synchronized (key.intern()) {
+ DataOutputStream dos = hash.get(key);
+ try {
+ chunk.write(dos);
+ dos.flush();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+
+ static void close() {
+ Iterator<String> it = hash.keySet().iterator();
+ while (it.hasNext()) {
+ String key = it.next();
+ DataOutputStream dos = hash.get(key);
+ try {
+ dos.close();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ }
}
Modified: hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/validationframework/interceptor/ChunkQueueInterceptor.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/validationframework/interceptor/ChunkQueueInterceptor.java?rev=752666&r1=752665&r2=752666&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/validationframework/interceptor/ChunkQueueInterceptor.java (original)
+++ hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/validationframework/interceptor/ChunkQueueInterceptor.java Wed Mar 11 22:39:26 2009
@@ -1,40 +1,36 @@
package org.apache.hadoop.chukwa.validationframework.interceptor;
-import java.util.List;
+import java.util.List;
import org.apache.hadoop.chukwa.Chunk;
import org.apache.hadoop.chukwa.datacollection.ChunkQueue;
public class ChunkQueueInterceptor implements
- org.apache.hadoop.chukwa.datacollection.ChunkQueue
-{
- private ChunkQueue defaultQueue = null;
-
- public ChunkQueueInterceptor(ChunkQueue defaultQueue)
- {
- this.defaultQueue = defaultQueue;
- }
-
- @Override
- public void add(Chunk chunk) throws InterruptedException
- {
- ChunkDumper.dump("adaptor", chunk);
- defaultQueue.add(chunk);
- }
-
- @Override
- public void collect(List<Chunk> chunks, int count)
- throws InterruptedException
- {
- defaultQueue.collect(chunks, count);
- for(Chunk chunk: chunks)
- { ChunkDumper.dump("sender", chunk);}
- }
-
- @Override
- public int size()
- {
- return defaultQueue.size();
- }
+ org.apache.hadoop.chukwa.datacollection.ChunkQueue {
+ private ChunkQueue defaultQueue = null;
+
+ public ChunkQueueInterceptor(ChunkQueue defaultQueue) {
+ this.defaultQueue = defaultQueue;
+ }
+
+ @Override
+ public void add(Chunk chunk) throws InterruptedException {
+ ChunkDumper.dump("adaptor", chunk);
+ defaultQueue.add(chunk);
+ }
+
+ @Override
+ public void collect(List<Chunk> chunks, int count)
+ throws InterruptedException {
+ defaultQueue.collect(chunks, count);
+ for (Chunk chunk : chunks) {
+ ChunkDumper.dump("sender", chunk);
+ }
+ }
+
+ @Override
+ public int size() {
+ return defaultQueue.size();
+ }
}
Modified: hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/validationframework/interceptor/SetupTestClasses.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/validationframework/interceptor/SetupTestClasses.java?rev=752666&r1=752665&r2=752666&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/validationframework/interceptor/SetupTestClasses.java (original)
+++ hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/validationframework/interceptor/SetupTestClasses.java Wed Mar 11 22:39:26 2009
@@ -1,34 +1,29 @@
package org.apache.hadoop.chukwa.validationframework.interceptor;
-import java.lang.reflect.Field;
+import java.lang.reflect.Field;
import org.apache.hadoop.chukwa.datacollection.ChunkQueue;
import org.apache.hadoop.chukwa.datacollection.DataFactory;
-public class SetupTestClasses
-{
- public static void setupClasses() throws Throwable
- {
- setupChunkQueueInterceptor();
- }
-
- static protected void setupChunkQueueInterceptor() throws Throwable
- {
- DataFactory da = DataFactory.getInstance();
- ChunkQueue chunkQueue = da.getEventQueue();
-
- final Field fields[] = DataFactory.class.getDeclaredFields();
- for (int i = 0; i < fields.length; ++i)
- {
- if ("chunkQueue".equals(fields[i].getName()))
- {
- Field f = fields[i];
- f.setAccessible(true);
- ChunkQueue ci = new ChunkQueueInterceptor(chunkQueue);
- f.set(da, ci);
- System.out.println("Adding QueueInterceptor");
- break;
- }
- }
- }
+public class SetupTestClasses {
+ public static void setupClasses() throws Throwable {
+ setupChunkQueueInterceptor();
+ }
+
+ static protected void setupChunkQueueInterceptor() throws Throwable {
+ DataFactory da = DataFactory.getInstance();
+ ChunkQueue chunkQueue = da.getEventQueue();
+
+ final Field fields[] = DataFactory.class.getDeclaredFields();
+ for (int i = 0; i < fields.length; ++i) {
+ if ("chunkQueue".equals(fields[i].getName())) {
+ Field f = fields[i];
+ f.setAccessible(true);
+ ChunkQueue ci = new ChunkQueueInterceptor(chunkQueue);
+ f.set(da, ci);
+ System.out.println("Adding QueueInterceptor");
+ break;
+ }
+ }
+ }
}
Modified: hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/validationframework/util/DataOperations.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/validationframework/util/DataOperations.java?rev=752666&r1=752665&r2=752666&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/validationframework/util/DataOperations.java (original)
+++ hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/validationframework/util/DataOperations.java Wed Mar 11 22:39:26 2009
@@ -1,5 +1,6 @@
package org.apache.hadoop.chukwa.validationframework.util;
+
import java.io.DataInputStream;
import java.io.EOFException;
import java.io.File;
@@ -8,7 +9,6 @@
import java.io.FileWriter;
import java.io.IOException;
import java.net.URI;
-
import org.apache.hadoop.chukwa.ChukwaArchiveKey;
import org.apache.hadoop.chukwa.Chunk;
import org.apache.hadoop.chukwa.ChunkImpl;
@@ -21,20 +21,17 @@
import org.apache.hadoop.io.SequenceFile;
import org.apache.log4j.Logger;
-public class DataOperations
-{
+public class DataOperations {
static Logger log = Logger.getLogger(DataOperations.class);
public static void copyFile(String fromFileName, String toFileName)
- throws IOException
- {
+ throws IOException {
File fromFile = new File(fromFileName);
File toFile = new File(toFileName);
FileInputStream from = null;
FileOutputStream to = null;
- try
- {
+ try {
from = new FileInputStream(fromFile);
to = new FileOutputStream(toFile);
byte[] buffer = new byte[4096];
@@ -42,29 +39,23 @@
while ((bytesRead = from.read(buffer)) != -1)
to.write(buffer, 0, bytesRead); // write
- } finally
- {
+ } finally {
if (from != null)
- try
- {
+ try {
from.close();
- } catch (IOException e)
- {
+ } catch (IOException e) {
;
}
if (to != null)
- try
- {
+ try {
to.close();
- } catch (IOException e)
- {
+ } catch (IOException e) {
// ;
}
}
}
- public static boolean validateMD5(String inputFile, String testFile)
- {
+ public static boolean validateMD5(String inputFile, String testFile) {
// System.out.println("validateMD5 [" + inputFile + "] [" + testFile
// + "]");
String md5_1 = MD5.checksum(new File(inputFile));
@@ -73,8 +64,7 @@
return md5_1.intern() == md5_2.intern();
}
- public static boolean validateMD5(FileSystem fs, Path inputFile, Path testFile)
- {
+ public static boolean validateMD5(FileSystem fs, Path inputFile, Path testFile) {
// System.out.println("validateMD5 [" + inputFile + "] [" + testFile
// + "]");
String md5_1 = MD5.checksum(fs, inputFile);
@@ -84,12 +74,10 @@
}
public static boolean validateChukwaRecords(FileSystem fs,
- Configuration conf, Path inputFile, Path testFile)
- {
+ Configuration conf, Path inputFile, Path testFile) {
SequenceFile.Reader goldReader = null;
SequenceFile.Reader testReader = null;
- try
- {
+ try {
// log.info(">>>>>>>>>>>>>> Openning records [" + inputFile.getName()
// +"][" + testFile.getName() +"]");
goldReader = new SequenceFile.Reader(fs, inputFile, conf);
@@ -102,12 +90,10 @@
ChukwaRecord testRecord = new ChukwaRecord();
// log.info(">>>>>>>>>>>>>> Start reading");
- while (goldReader.next(goldKey, goldRecord))
- {
+ while (goldReader.next(goldKey, goldRecord)) {
testReader.next(testKey, testRecord);
- if (goldKey.compareTo(testKey) != 0)
- {
+ if (goldKey.compareTo(testKey) != 0) {
log.info(">>>>>>>>>>>>>> Not the same Key");
log.info(">>>>>>>>>>>>>> Record [" + goldKey.getKey() + "] ["
+ goldKey.getReduceType() + "]");
@@ -116,8 +102,7 @@
return false;
}
- if (goldRecord.compareTo(testRecord) != 0)
- {
+ if (goldRecord.compareTo(testRecord) != 0) {
log.info(">>>>>>>>>>>>>> Not the same Value");
log.info(">>>>>>>>>>>>>> Record [" + goldKey.getKey() + "] ["
+ goldKey.getReduceType() + "]");
@@ -125,32 +110,27 @@
+ testKey.getReduceType() + "]");
log.info(">>>>>>>>>>>>>> Gold Value [" + goldRecord.toString() + "]");
log.info(">>>>>>>>>>>>>> Test value [" + testRecord.toString() + "]");
-
+
return false;
}
}
// log.info(">>>>>>>>>>>>>> Same File");
return true;
- } catch (IOException e)
- {
+ } catch (IOException e) {
e.printStackTrace();
return false;
- } finally
- {
- try
- {
+ } finally {
+ try {
goldReader.close();
testReader.close();
- } catch (IOException e)
- {
+ } catch (IOException e) {
}
}
}
public static void extractRawLogFromdataSink(String directory, String fileName)
- throws Exception
- {
+ throws Exception {
ChukwaConfiguration conf = new ChukwaConfiguration();
String fsName = conf.get("writer.hdfs.filesystem");
FileSystem fs = FileSystem.get(new URI(fsName), conf);
@@ -163,14 +143,11 @@
ChukwaArchiveKey key = new ChukwaArchiveKey();
ChunkImpl chunk = ChunkImpl.getBlankChunk();
FileWriter out = new FileWriter(outputFile);
- try
- {
- while (r.next(key, chunk))
- {
+ try {
+ while (r.next(key, chunk)) {
out.write(new String(chunk.getData()));
}
- } finally
- {
+ } finally {
out.close();
r.close();
}
@@ -178,22 +155,18 @@
}
public static void extractRawLogFromDump(String directory, String fileName)
- throws Exception
- {
+ throws Exception {
File inputFile = new File(directory + fileName + ".bin");
File outputFile = new File(directory + fileName + ".raw");
DataInputStream dis = new DataInputStream(new FileInputStream(inputFile));
Chunk chunk = null;
FileWriter out = new FileWriter(outputFile);
boolean eof = false;
- do
- {
- try
- {
+ do {
+ try {
chunk = ChunkImpl.read(dis);
out.write(new String(chunk.getData()));
- } catch (EOFException e)
- {
+ } catch (EOFException e) {
eof = true;
}
Modified: hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/validationframework/util/MD5.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/validationframework/util/MD5.java?rev=752666&r1=752665&r2=752666&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/validationframework/util/MD5.java (original)
+++ hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/validationframework/util/MD5.java Wed Mar 11 22:39:26 2009
@@ -1,48 +1,45 @@
package org.apache.hadoop.chukwa.validationframework.util;
+
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import java.security.MessageDigest;
-
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-public class MD5
-{
- public static String checksum(File file) {
- try {
- InputStream fin = new FileInputStream(file);
- java.security.MessageDigest md5er =
- MessageDigest.getInstance("MD5");
- byte[] buffer = new byte[1024];
- int read;
- do {
- read = fin.read(buffer);
- if (read > 0)
- md5er.update(buffer, 0, read);
- } while (read != -1);
- fin.close();
- byte[] digest = md5er.digest();
- if (digest == null)
- return null;
- String strDigest = "0x";
- for (int i = 0; i < digest.length; i++) {
- strDigest += Integer.toString((digest[i] & 0xff)
- + 0x100, 16).substring(1).toUpperCase();
- }
- return strDigest;
- } catch (Exception e) {
- return null;
- }
- }
-
- public static String checksum(FileSystem fs,Path file) {
+public class MD5 {
+ public static String checksum(File file) {
+ try {
+ InputStream fin = new FileInputStream(file);
+ java.security.MessageDigest md5er = MessageDigest.getInstance("MD5");
+ byte[] buffer = new byte[1024];
+ int read;
+ do {
+ read = fin.read(buffer);
+ if (read > 0)
+ md5er.update(buffer, 0, read);
+ } while (read != -1);
+ fin.close();
+ byte[] digest = md5er.digest();
+ if (digest == null)
+ return null;
+ String strDigest = "0x";
+ for (int i = 0; i < digest.length; i++) {
+ strDigest += Integer.toString((digest[i] & 0xff) + 0x100, 16)
+ .substring(1).toUpperCase();
+ }
+ return strDigest;
+ } catch (Exception e) {
+ return null;
+ }
+ }
+
+ public static String checksum(FileSystem fs, Path file) {
try {
FSDataInputStream fin = fs.open(file);
- java.security.MessageDigest md5er =
- MessageDigest.getInstance("MD5");
+ java.security.MessageDigest md5er = MessageDigest.getInstance("MD5");
byte[] buffer = new byte[1024];
int read;
do {
@@ -56,8 +53,8 @@
return null;
String strDigest = "0x";
for (int i = 0; i < digest.length; i++) {
- strDigest += Integer.toString((digest[i] & 0xff)
- + 0x100, 16).substring(1).toUpperCase();
+ strDigest += Integer.toString((digest[i] & 0xff) + 0x100, 16)
+ .substring(1).toUpperCase();
}
return strDigest;
} catch (Exception e) {