You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@chukwa.apache.org by ey...@apache.org on 2012/11/20 20:39:04 UTC
svn commit: r1411813 - in /incubator/chukwa/trunk: ./
src/main/java/org/apache/hadoop/chukwa/database/
src/main/java/org/apache/hadoop/chukwa/datacollection/writer/
src/main/java/org/apache/hadoop/chukwa/dataloader/
src/main/java/org/apache/hadoop/chuk...
Author: eyang
Date: Tue Nov 20 19:39:03 2012
New Revision: 1411813
URL: http://svn.apache.org/viewvc?rev=1411813&view=rev
Log:
CHUKWA-642. Added regular expression validation. (Eric Spishak via Eric Yang)
Modified:
incubator/chukwa/trunk/CHANGES.txt
incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/database/DataExpiration.java
incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/database/TableCreator.java
incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/datacollection/writer/SocketTeeWriter.java
incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/dataloader/MetricDataLoader.java
incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/TsProcessor.java
incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/inputtools/ChukwaInputFormat.java
incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/util/DumpChunks.java
incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/util/Filter.java
incubator/chukwa/trunk/src/test/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/TestTsProcessor.java
incubator/chukwa/trunk/src/test/java/org/apache/hadoop/chukwa/inputtools/TestInputFormat.java
incubator/chukwa/trunk/src/test/java/org/apache/hadoop/chukwa/util/TestDumpChunks.java
incubator/chukwa/trunk/src/test/java/org/apache/hadoop/chukwa/util/TestFilter.java
Modified: incubator/chukwa/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/CHANGES.txt?rev=1411813&r1=1411812&r2=1411813&view=diff
==============================================================================
--- incubator/chukwa/trunk/CHANGES.txt (original)
+++ incubator/chukwa/trunk/CHANGES.txt Tue Nov 20 19:39:03 2012
@@ -24,6 +24,8 @@ Trunk (unreleased changes)
BUGS
+ CHUKWA-642. Added regular expression validation. (Eric Spishak via Eric Yang)
+
CHUKWA-632. Added getter method for startOffset in AbstractProcessor. (Abhijit Dhar via Eric Yang)
CHUKWA-637. Fixed default cluster selection in hicc. (Eric Yang)
Modified: incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/database/DataExpiration.java
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/database/DataExpiration.java?rev=1411813&r1=1411812&r2=1411813&view=diff
==============================================================================
--- incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/database/DataExpiration.java (original)
+++ incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/database/DataExpiration.java Tue Nov 20 19:39:03 2012
@@ -27,6 +27,7 @@ import java.util.Iterator;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.chukwa.util.DatabaseWriter;
+import org.apache.hadoop.chukwa.util.RegexUtil;
public class DataExpiration {
private static DatabaseConfig dbc = null;
@@ -50,6 +51,12 @@ public class DataExpiration {
while (ki.hasNext()) {
String name = ki.next();
String tableName = dbNames.get(name);
+ if (!RegexUtil.isRegex(tableName)) {
+ log.warn("Skipping tableName: '" + tableName
+ + "' because there was an error parsing it as a regex: "
+ + RegexUtil.regexError(tableName));
+ return;
+ }
String[] tableList = dbc.findTableName(tableName, start, end);
for (String tl : tableList) {
log.debug("table name: " + tableList[0]);
Modified: incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/database/TableCreator.java
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/database/TableCreator.java?rev=1411813&r1=1411812&r2=1411813&view=diff
==============================================================================
--- incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/database/TableCreator.java (original)
+++ incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/database/TableCreator.java Tue Nov 20 19:39:03 2012
@@ -29,6 +29,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.chukwa.util.DatabaseWriter;
import org.apache.hadoop.chukwa.util.ExceptionUtil;
+import org.apache.hadoop.chukwa.util.RegexUtil;
public class TableCreator {
private static DatabaseConfig dbc = null;
@@ -56,6 +57,12 @@ public class TableCreator {
while (ki.hasNext()) {
String name = ki.next();
String tableName = dbNames.get(name);
+ if (!RegexUtil.isRegex(tableName)) {
+ log.warn("Skipping tableName: '" + tableName
+ + "' because there was an error parsing it as a regex: "
+ + RegexUtil.regexError(tableName));
+ return;
+ }
String[] tableList = dbc.findTableName(tableName, start, end);
log.debug("table name: " + tableList[0]);
try {
Modified: incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/datacollection/writer/SocketTeeWriter.java
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/datacollection/writer/SocketTeeWriter.java?rev=1411813&r1=1411812&r2=1411813&view=diff
==============================================================================
--- incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/datacollection/writer/SocketTeeWriter.java (original)
+++ incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/datacollection/writer/SocketTeeWriter.java Tue Nov 20 19:39:03 2012
@@ -23,6 +23,8 @@ import java.util.concurrent.ArrayBlockin
import java.util.regex.PatternSyntaxException;
import org.apache.hadoop.chukwa.Chunk;
import org.apache.hadoop.chukwa.util.Filter;
+import org.apache.hadoop.chukwa.util.RegexUtil;
+import org.apache.hadoop.chukwa.util.RegexUtil.CheckedPatternSyntaxException;
import org.apache.hadoop.conf.Configuration;
import org.apache.log4j.Logger;
import java.net.ServerSocket;
@@ -167,7 +169,7 @@ public class SocketTeeWriter extends Pip
*/
public void setup() {
try { //outer try catches IOExceptions
- try { //inner try catches Pattern Syntax errors
+ try { //inner try catches bad command syntax errors
sock.setSoTimeout(timeout);
sock.setKeepAlive(USE_KEEPALIVE);
in = new BufferedReader(new InputStreamReader(sock.getInputStream()));
@@ -175,8 +177,8 @@ public class SocketTeeWriter extends Pip
String cmd = in.readLine();
if(!cmd.contains(" ")) {
- throw new PatternSyntaxException(
- "command should be keyword pattern, but no ' ' seen", cmd, -1);
+ throw new IllegalArgumentException(
+ "command should be keyword pattern, but no ' ' seen: " + cmd);
}
String uppercased = cmd.substring(0, cmd.indexOf(' ')).toUpperCase();
if(RAW.equals(uppercased))
@@ -186,16 +188,27 @@ public class SocketTeeWriter extends Pip
else if(ASCII_HEADER.equals(uppercased))
fmt = DataFormat.Header;
else {
- throw new PatternSyntaxException("bad command '" + uppercased+
+ throw new IllegalArgumentException("bad command '" + uppercased+
"' -- starts with neither '"+ RAW+ "' nor '"+ WRITABLE + " nor "
- + ASCII_HEADER+"'.", cmd, -1);
+ + ASCII_HEADER+"':" + cmd);
}
String cmdAfterSpace = cmd.substring(cmd.indexOf(' ')+1);
if(cmdAfterSpace.toLowerCase().equals("all"))
rules = Filter.ALL;
else
- rules = new Filter(cmdAfterSpace);
+ try {
+ rules = new Filter(cmdAfterSpace);
+ } catch (CheckedPatternSyntaxException pse) {
+ out.write("Error parsing command as a regex: ".getBytes());
+ out.write(pse.getMessage().getBytes());
+ out.writeByte('\n');
+ out.close();
+ in.close();
+ sock.close();
+ log.warn(pse);
+ return;
+ }
//now that we read everything OK we can add ourselves to list, and return.
synchronized(tees) {
@@ -203,7 +216,7 @@ public class SocketTeeWriter extends Pip
}
out.write("OK\n".getBytes());
log.info("tee to " + sock.getInetAddress() + " established");
- } catch(PatternSyntaxException e) {
+ } catch(IllegalArgumentException e) {
out.write(e.toString().getBytes());
out.writeByte('\n');
out.close();
Modified: incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/dataloader/MetricDataLoader.java
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/dataloader/MetricDataLoader.java?rev=1411813&r1=1411812&r2=1411813&view=diff
==============================================================================
--- incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/dataloader/MetricDataLoader.java (original)
+++ incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/dataloader/MetricDataLoader.java Tue Nov 20 19:39:03 2012
@@ -41,6 +41,7 @@ import org.apache.hadoop.chukwa.extracti
import org.apache.hadoop.chukwa.util.ClusterConfig;
import org.apache.hadoop.chukwa.util.DatabaseWriter;
import org.apache.hadoop.chukwa.util.ExceptionUtil;
+import org.apache.hadoop.chukwa.util.RegexUtil;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.SequenceFile;
@@ -220,7 +221,13 @@ public class MetricDataLoader implements
String dbKey = "report.db.name." + recordType;
Matcher m = p.matcher(recordType);
if (dbTables.containsKey(dbKey)) {
- String[] tmp = mdlConfig.findTableName(mdlConfig.get(dbKey), record
+ String tableName = mdlConfig.get(dbKey);
+ if (!RegexUtil.isRegex(tableName)) {
+ log.error("Error parsing 'tableName' as a regex: "
+ + RegexUtil.regexError(tableName));
+ return false;
+ }
+ String[] tmp = mdlConfig.findTableName(tableName, record
.getTime(), record.getTime());
table = tmp[0];
} else if(m.matches()) {
Modified: incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/TsProcessor.java
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/TsProcessor.java?rev=1411813&r1=1411812&r2=1411813&view=diff
==============================================================================
--- incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/TsProcessor.java (original)
+++ incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/TsProcessor.java Tue Nov 20 19:39:03 2012
@@ -31,6 +31,7 @@ import org.apache.hadoop.chukwa.datacoll
import org.apache.hadoop.chukwa.extraction.engine.ChukwaRecord;
import org.apache.hadoop.chukwa.extraction.engine.ChukwaRecordKey;
import org.apache.hadoop.chukwa.extraction.demux.Demux;
+import org.apache.hadoop.chukwa.util.RegexUtil;
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapred.JobConf;
@@ -65,6 +66,8 @@ public class TsProcessor extends Abstrac
static Logger log = Logger.getLogger(TsProcessor.class);
public static final String DEFAULT_DATE_FORMAT = "yyyy-MM-dd HH:mm:ss,SSS";
+ public static final String DEFAULT_TIME_REGEX = "TsProcessor.default.time.regex";
+ public static final String TIME_REGEX = "TsProcessor.time.regex.";
private Map<String, Pattern> datePatternMap;
private Map<String, SimpleDateFormat> dateFormatMap;
@@ -155,14 +158,24 @@ public class TsProcessor extends Abstrac
JobConf jobConf = Demux.jobConf;
String datePattern = null;
+ Pattern pattern = null;
if (jobConf != null) {
- datePattern = jobConf.get("TsProcessor.default.time.regex", null);
- datePattern = jobConf.get("TsProcessor.time.regex." + chunk.getDataType(),
- datePattern);
+ String timeRegexProperty = TIME_REGEX + chunk.getDataType();
+ datePattern = jobConf.get(DEFAULT_TIME_REGEX, null);
+ datePattern = jobConf.get(timeRegexProperty, datePattern);
+ if (datePattern != null) {
+ if (!RegexUtil.isRegex(datePattern, 1)) {
+ log.warn("Error parsing '" + DEFAULT_TIME_REGEX + "' or '"
+ + timeRegexProperty + "' properties as a regex: "
+ + RegexUtil.regexError(datePattern, 1)
+ + ". This date pattern will be skipped.");
+ return null;
+ }
+ pattern = Pattern.compile(datePattern);
+ }
}
- Pattern pattern = datePattern != null ? Pattern.compile(datePattern) : null;
datePatternMap.put(dataType, pattern);
return pattern;
Modified: incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/inputtools/ChukwaInputFormat.java
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/inputtools/ChukwaInputFormat.java?rev=1411813&r1=1411812&r2=1411813&view=diff
==============================================================================
--- incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/inputtools/ChukwaInputFormat.java (original)
+++ incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/inputtools/ChukwaInputFormat.java Tue Nov 20 19:39:03 2012
@@ -22,6 +22,7 @@ package org.apache.hadoop.chukwa.inputto
import java.io.IOException;
import java.util.regex.*;
import org.apache.hadoop.chukwa.*;
+import org.apache.hadoop.chukwa.util.RegexUtil;
import org.apache.hadoop.mapred.*;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.*;
@@ -46,6 +47,9 @@ public class ChukwaInputFormat extends
static Logger LOG = Logger.getLogger(ChukwaInputFormat.class);
+ public static final String DATATYPE_PROPERTY = "chukwa.inputfilter.datatype";
+ public static final String DATATYPE_PROPERTY_DEFAULT = ".*";
+
private SequenceFileRecordReader<ChukwaArchiveKey, Chunk> sfrr;
private long lineInFile = 0;
private Chunk curChunk = null;
@@ -58,8 +62,14 @@ public class ChukwaInputFormat extends
public ChukwaRecordReader(Configuration conf, FileSplit split)
throws IOException {
sfrr = new SequenceFileRecordReader<ChukwaArchiveKey, Chunk>(conf, split);
- dtPattern = Pattern
- .compile(conf.get("chukwa.inputfilter.datatype", ".*"));
+ String datatype = conf.get(DATATYPE_PROPERTY, DATATYPE_PROPERTY_DEFAULT);
+ if (!RegexUtil.isRegex(datatype)) {
+ LOG.warn("Error parsing '" + DATATYPE_PROPERTY
+ + "' property as a regex: " + RegexUtil.regexError(datatype)
+ + ". Using default instead: " + DATATYPE_PROPERTY_DEFAULT);
+ datatype = DATATYPE_PROPERTY_DEFAULT;
+ }
+ dtPattern = Pattern.compile(datatype);
}
@Override
Modified: incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/util/DumpChunks.java
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/util/DumpChunks.java?rev=1411813&r1=1411812&r2=1411813&view=diff
==============================================================================
--- incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/util/DumpChunks.java (original)
+++ incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/util/DumpChunks.java Tue Nov 20 19:39:03 2012
@@ -25,6 +25,7 @@ import java.util.*;
import java.io.*;
import org.apache.hadoop.chukwa.*;
import org.apache.hadoop.chukwa.conf.ChukwaConfiguration;
+import org.apache.hadoop.chukwa.util.RegexUtil.CheckedPatternSyntaxException;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.FileUtil;
@@ -84,13 +85,14 @@ public class DumpChunks {
Filter patterns = null;
if(args[filterArg].toLowerCase().equals("all"))
patterns = Filter.ALL;
- else
+ else {
try {
patterns = new Filter(args[filterArg]);
- } catch (PatternSyntaxException pse) {
+ } catch (CheckedPatternSyntaxException pse) {
System.err.println("Error parsing \"tags\" regular expression: " + pse.getMessage());
- System.exit(-1);
+ return;
}
+ }
System.err.println("Patterns:" + patterns);
ArrayList<Path> filesToSearch = new ArrayList<Path>();
Modified: incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/util/Filter.java
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/util/Filter.java?rev=1411813&r1=1411812&r2=1411813&view=diff
==============================================================================
--- incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/util/Filter.java (original)
+++ incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/util/Filter.java Tue Nov 20 19:39:03 2012
@@ -24,10 +24,14 @@ import java.util.regex.PatternSyntaxExce
import org.apache.commons.lang.ArrayUtils;
import org.apache.hadoop.chukwa.Chunk;
import org.apache.hadoop.chukwa.extraction.engine.RecordUtil;
+import org.apache.hadoop.chukwa.util.RegexUtil.CheckedPatternSyntaxException;
+import org.apache.log4j.Logger;
public class Filter {
-
+
+ static Logger log = Logger.getLogger(Filter.class);
+
private static final String[] SEARCH_TARGS =
{"datatype", "name", "host", "cluster", "content"};
static final String SEPARATOR="&";
@@ -56,6 +60,11 @@ public class Filter {
return p.matcher(content).matches();
} else if(targ.startsWith("tags.")) {
String tagName = targ.substring("tags.".length());
+ if (!RegexUtil.isRegex(tagName)) {
+ log.warn("Error parsing 'tagName' as a regex: "
+ + RegexUtil.regexError(tagName));
+ return false;
+ }
String tagVal = chunk.getTag(tagName);
if(tagVal == null)
return false;
@@ -73,7 +82,7 @@ public class Filter {
List<SearchRule> compiledPatterns;
- public Filter(String listOfPatterns) throws PatternSyntaxException{
+ public Filter(String listOfPatterns) throws CheckedPatternSyntaxException {
compiledPatterns = new ArrayList<SearchRule>();
//FIXME: could escape these
String[] patterns = listOfPatterns.split(SEPARATOR);
@@ -81,17 +90,22 @@ public class Filter {
int equalsPos = p.indexOf('=');
if(equalsPos < 0 || equalsPos > (p.length() -2)) {
- throw new PatternSyntaxException(
+ throw new CheckedPatternSyntaxException(
"pattern must be of form targ=pattern", p, -1);
}
String targ = p.substring(0, equalsPos);
if(!targ.startsWith("tags.") && !ArrayUtils.contains(SEARCH_TARGS, targ)) {
- throw new PatternSyntaxException(
+ throw new CheckedPatternSyntaxException(
"pattern doesn't start with recognized search target", p, -1);
}
- Pattern pat = Pattern.compile(p.substring(equalsPos+1), Pattern.DOTALL);
+ String regex = p.substring(equalsPos+1);
+ if (!RegexUtil.isRegex(regex)) {
+ throw new CheckedPatternSyntaxException(RegexUtil.regexException(regex));
+ }
+
+ Pattern pat = Pattern.compile(regex, Pattern.DOTALL);
compiledPatterns.add(new SearchRule(pat, targ));
}
}
@@ -119,7 +133,7 @@ public class Filter {
}
private static final class MatchAll extends Filter {
- public MatchAll() {
+ public MatchAll() throws CheckedPatternSyntaxException {
super("datatype=.*");
}
@@ -131,6 +145,14 @@ public class Filter {
return "ALL";
}
}
- public static final Filter ALL = new MatchAll();
+
+ public static final Filter ALL = newMatchAll();
+ private static Filter newMatchAll() {
+ try {
+ return new MatchAll();
+ } catch (CheckedPatternSyntaxException e) {
+ throw new RuntimeException("Illegal MatchAll regular expression.", e);
+ }
+ }
}//end class
Modified: incubator/chukwa/trunk/src/test/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/TestTsProcessor.java
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/src/test/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/TestTsProcessor.java?rev=1411813&r1=1411812&r2=1411813&view=diff
==============================================================================
--- incubator/chukwa/trunk/src/test/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/TestTsProcessor.java (original)
+++ incubator/chukwa/trunk/src/test/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/TestTsProcessor.java Tue Nov 20 19:39:03 2012
@@ -165,4 +165,24 @@ public class TestTsProcessor extends Tes
return key;
}
+ public void testParseIllegalRegex() {
+ jobConf.set(TsProcessor.DEFAULT_TIME_REGEX, "(");
+
+ ChunkBuilder cb = new ChunkBuilder();
+ cb.addRecord("2012-10-25 00:18:44,818 some sample record data".getBytes());
+ Chunk chunk = cb.getChunk();
+
+ TsProcessor p = new TsProcessor();
+ p.reset(chunk);
+
+ ChukwaTestOutputCollector<ChukwaRecordKey, ChukwaRecord> output =
+ new ChukwaTestOutputCollector<ChukwaRecordKey, ChukwaRecord>();
+
+ p.process(null, chunk, output, Reporter.NULL);
+
+ assertEquals("Output data size not correct.", 1, output.data.size());
+ ChukwaRecordKey key = output.data.keySet().iterator().next();
+ ChukwaRecord record = output.data.get(key);
+ assertNull("Output should not be error.", record.getValue("cchunkData"));
+ }
}
\ No newline at end of file
Modified: incubator/chukwa/trunk/src/test/java/org/apache/hadoop/chukwa/inputtools/TestInputFormat.java
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/src/test/java/org/apache/hadoop/chukwa/inputtools/TestInputFormat.java?rev=1411813&r1=1411812&r2=1411813&view=diff
==============================================================================
--- incubator/chukwa/trunk/src/test/java/org/apache/hadoop/chukwa/inputtools/TestInputFormat.java (original)
+++ incubator/chukwa/trunk/src/test/java/org/apache/hadoop/chukwa/inputtools/TestInputFormat.java Tue Nov 20 19:39:03 2012
@@ -20,6 +20,8 @@ package org.apache.hadoop.chukwa.inputto
import java.io.IOException;
+import java.util.regex.PatternSyntaxException;
+
import org.apache.hadoop.mapred.Reporter;
import junit.framework.TestCase;
import org.apache.hadoop.chukwa.*;
@@ -82,4 +84,25 @@ public class TestInputFormat extends Tes
}
}
+ public void testInputFormatIllegalRegex() {
+ try {
+ JobConf conf = new JobConf();
+ conf.set("chukwa.inputfilter.datatype", "(");
+ String TMP_DIR = System.getProperty("test.build.data", "/tmp");
+ Path filename = new Path("file:///" + TMP_DIR + "/tmpSeqFile");
+ long len = FileSystem.getLocal(conf).getFileStatus(filename).getLen();
+ InputSplit split = new FileSplit(filename, 0, len, (String[]) null);
+
+ ChukwaInputFormat in = new ChukwaInputFormat();
+ RecordReader<LongWritable, Text> r = in.getRecordReader(split, conf,
+ Reporter.NULL);
+
+ } catch (PatternSyntaxException e) {
+ e.printStackTrace();
+ fail("Illegal regular expression caused PatternSyntaxException: " + e);
+ } catch (IOException e) {
+ e.printStackTrace();
+ fail("IO exception " + e);
+ }
+ }
}
Modified: incubator/chukwa/trunk/src/test/java/org/apache/hadoop/chukwa/util/TestDumpChunks.java
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/src/test/java/org/apache/hadoop/chukwa/util/TestDumpChunks.java?rev=1411813&r1=1411812&r2=1411813&view=diff
==============================================================================
--- incubator/chukwa/trunk/src/test/java/org/apache/hadoop/chukwa/util/TestDumpChunks.java (original)
+++ incubator/chukwa/trunk/src/test/java/org/apache/hadoop/chukwa/util/TestDumpChunks.java Tue Nov 20 19:39:03 2012
@@ -19,6 +19,7 @@ package org.apache.hadoop.chukwa.util;
import junit.framework.TestCase;
import java.util.*;
+import java.util.regex.PatternSyntaxException;
import java.io.*;
import org.apache.hadoop.chukwa.ChukwaArchiveKey;
import org.apache.hadoop.chukwa.ChunkImpl;
@@ -86,4 +87,15 @@ public class TestDumpChunks extends Test
}
+ public void testIllegalRegex() throws Exception {
+ String[] args = { "tags.command=(" };
+ Configuration conf = new Configuration();
+
+ try {
+ DumpChunks.dump(args, conf, System.out);
+ } catch (PatternSyntaxException e) {
+ e.printStackTrace();
+ fail("Illegal regular expression caused PatternSyntaxException: " + e);
+ }
+ }
}
Modified: incubator/chukwa/trunk/src/test/java/org/apache/hadoop/chukwa/util/TestFilter.java
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/src/test/java/org/apache/hadoop/chukwa/util/TestFilter.java?rev=1411813&r1=1411812&r2=1411813&view=diff
==============================================================================
--- incubator/chukwa/trunk/src/test/java/org/apache/hadoop/chukwa/util/TestFilter.java (original)
+++ incubator/chukwa/trunk/src/test/java/org/apache/hadoop/chukwa/util/TestFilter.java Tue Nov 20 19:39:03 2012
@@ -19,6 +19,7 @@ package org.apache.hadoop.chukwa.util;
import junit.framework.TestCase;
import org.apache.hadoop.chukwa.ChunkImpl;
+import org.apache.hadoop.chukwa.util.RegexUtil.CheckedPatternSyntaxException;
public class TestFilter extends TestCase {
@@ -65,7 +66,13 @@ public class TestFilter extends TestCase
ChunkImpl chunk1 = new ChunkImpl("Data", "aname", dat.length, dat, null);
chunk1.setSource("asource");
assertTrue(Filter.ALL.matches(chunk1));
- Filter rule = new Filter("tags.foo=bar");
+ Filter rule = null;
+ try {
+ rule = new Filter("tags.foo=bar");
+ } catch (CheckedPatternSyntaxException e) {
+ e.printStackTrace();
+ fail("Regular expression error: " + e);
+ }
assertFalse(rule.matches(chunk1));
chunk1.addTag("foo=\"bar\"");
@@ -75,4 +82,12 @@ public class TestFilter extends TestCase
assertTrue(Filter.ALL.matches(chunk1));
}
+ public void testIllegalRegex() {
+ try {
+ new Filter("tags.foo=(");
+ } catch (CheckedPatternSyntaxException e) {
+ return;
+ }
+ fail("No CheckedPatternSyntaxException thrown for illegal regular expression.");
+ }
}