You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by rm...@apache.org on 2016/04/26 16:46:03 UTC
[15/51] [partial] incubator-metron git commit: METRON-113 Project
Reorganization (merrimanr) closes apache/incubator-metron#88
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ise/JavaCharStream.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ise/JavaCharStream.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ise/JavaCharStream.java
new file mode 100644
index 0000000..31f3c6f
--- /dev/null
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ise/JavaCharStream.java
@@ -0,0 +1,633 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/* Generated By:JavaCC: Do not edit this line. JavaCharStream.java Version 5.0 */
+/* JavaCCOptions:STATIC=false,SUPPORT_CLASS_VISIBILITY_PUBLIC=false */
+package org.apache.metron.parsers.ise;
+
+/**
+ * An implementation of interface CharStream, where the stream is assumed to
+ * contain only ASCII characters (with java-like unicode escape processing).
+ */
+
+class JavaCharStream
+{
+ /** Whether parser is static. */
+ public static final boolean staticFlag = false;
+
+ static final int hexval(char c) throws java.io.IOException {
+ switch(c)
+ {
+ case '0' :
+ return 0;
+ case '1' :
+ return 1;
+ case '2' :
+ return 2;
+ case '3' :
+ return 3;
+ case '4' :
+ return 4;
+ case '5' :
+ return 5;
+ case '6' :
+ return 6;
+ case '7' :
+ return 7;
+ case '8' :
+ return 8;
+ case '9' :
+ return 9;
+
+ case 'a' :
+ case 'A' :
+ return 10;
+ case 'b' :
+ case 'B' :
+ return 11;
+ case 'c' :
+ case 'C' :
+ return 12;
+ case 'd' :
+ case 'D' :
+ return 13;
+ case 'e' :
+ case 'E' :
+ return 14;
+ case 'f' :
+ case 'F' :
+ return 15;
+ }
+
+ throw new java.io.IOException(); // Should never come here
+ }
+
+/** Position in buffer. */
+ public int bufpos = -1;
+ int bufsize;
+ int available;
+ int tokenBegin;
+ protected int bufline[];
+ protected int bufcolumn[];
+
+ protected int column = 0;
+ protected int line = 1;
+
+ protected boolean prevCharIsCR = false;
+ protected boolean prevCharIsLF = false;
+
+ protected java.io.Reader inputStream;
+
+ protected char[] nextCharBuf;
+ protected char[] buffer;
+ protected int maxNextCharInd = 0;
+ protected int nextCharInd = -1;
+ protected int inBuf = 0;
+ protected int tabSize = 8;
+
+ protected void setTabSize(int i) { tabSize = i; }
+ protected int getTabSize(int i) { return tabSize; }
+
+ protected void ExpandBuff(boolean wrapAround)
+ {
+ char[] newbuffer = new char[bufsize + 2048];
+ int newbufline[] = new int[bufsize + 2048];
+ int newbufcolumn[] = new int[bufsize + 2048];
+
+ try
+ {
+ if (wrapAround)
+ {
+ System.arraycopy(buffer, tokenBegin, newbuffer, 0, bufsize - tokenBegin);
+ System.arraycopy(buffer, 0, newbuffer, bufsize - tokenBegin, bufpos);
+ buffer = newbuffer;
+
+ System.arraycopy(bufline, tokenBegin, newbufline, 0, bufsize - tokenBegin);
+ System.arraycopy(bufline, 0, newbufline, bufsize - tokenBegin, bufpos);
+ bufline = newbufline;
+
+ System.arraycopy(bufcolumn, tokenBegin, newbufcolumn, 0, bufsize - tokenBegin);
+ System.arraycopy(bufcolumn, 0, newbufcolumn, bufsize - tokenBegin, bufpos);
+ bufcolumn = newbufcolumn;
+
+ bufpos += (bufsize - tokenBegin);
+ }
+ else
+ {
+ System.arraycopy(buffer, tokenBegin, newbuffer, 0, bufsize - tokenBegin);
+ buffer = newbuffer;
+
+ System.arraycopy(bufline, tokenBegin, newbufline, 0, bufsize - tokenBegin);
+ bufline = newbufline;
+
+ System.arraycopy(bufcolumn, tokenBegin, newbufcolumn, 0, bufsize - tokenBegin);
+ bufcolumn = newbufcolumn;
+
+ bufpos -= tokenBegin;
+ }
+ }
+ catch (Throwable t)
+ {
+ throw new Error(t.getMessage());
+ }
+
+ available = (bufsize += 2048);
+ tokenBegin = 0;
+ }
+
+ protected void FillBuff() throws java.io.IOException
+ {
+ int i;
+ if (maxNextCharInd == 4096)
+ maxNextCharInd = nextCharInd = 0;
+
+ try {
+ if ((i = inputStream.read(nextCharBuf, maxNextCharInd,
+ 4096 - maxNextCharInd)) == -1)
+ {
+ inputStream.close();
+ throw new java.io.IOException();
+ }
+ else
+ maxNextCharInd += i;
+ return;
+ }
+ catch(java.io.IOException e) {
+ if (bufpos != 0)
+ {
+ --bufpos;
+ backup(0);
+ }
+ else
+ {
+ bufline[bufpos] = line;
+ bufcolumn[bufpos] = column;
+ }
+ throw e;
+ }
+ }
+
+ protected char ReadByte() throws java.io.IOException
+ {
+ if (++nextCharInd >= maxNextCharInd)
+ FillBuff();
+
+ return nextCharBuf[nextCharInd];
+ }
+
+/** @return starting character for token. */
+ public char BeginToken() throws java.io.IOException
+ {
+ if (inBuf > 0)
+ {
+ --inBuf;
+
+ if (++bufpos == bufsize)
+ bufpos = 0;
+
+ tokenBegin = bufpos;
+ return buffer[bufpos];
+ }
+
+ tokenBegin = 0;
+ bufpos = -1;
+
+ return readChar();
+ }
+
+ protected void AdjustBuffSize()
+ {
+ if (available == bufsize)
+ {
+ if (tokenBegin > 2048)
+ {
+ bufpos = 0;
+ available = tokenBegin;
+ }
+ else
+ ExpandBuff(false);
+ }
+ else if (available > tokenBegin)
+ available = bufsize;
+ else if ((tokenBegin - available) < 2048)
+ ExpandBuff(true);
+ else
+ available = tokenBegin;
+ }
+
+ protected void UpdateLineColumn(char c)
+ {
+ column++;
+
+ if (prevCharIsLF)
+ {
+ prevCharIsLF = false;
+ line += (column = 1);
+ }
+ else if (prevCharIsCR)
+ {
+ prevCharIsCR = false;
+ if (c == '\n')
+ {
+ prevCharIsLF = true;
+ }
+ else
+ line += (column = 1);
+ }
+
+ switch (c)
+ {
+ case '\r' :
+ prevCharIsCR = true;
+ break;
+ case '\n' :
+ prevCharIsLF = true;
+ break;
+ case '\t' :
+ column--;
+ column += (tabSize - (column % tabSize));
+ break;
+ default :
+ break;
+ }
+
+ bufline[bufpos] = line;
+ bufcolumn[bufpos] = column;
+ }
+
+/** Read a character. */
+ public char readChar() throws java.io.IOException
+ {
+ if (inBuf > 0)
+ {
+ --inBuf;
+
+ if (++bufpos == bufsize)
+ bufpos = 0;
+
+ return buffer[bufpos];
+ }
+
+ char c;
+
+ if (++bufpos == available)
+ AdjustBuffSize();
+
+ if ((buffer[bufpos] = c = ReadByte()) == '\\')
+ {
+ UpdateLineColumn(c);
+
+ int backSlashCnt = 1;
+
+ for (;;) // Read all the backslashes
+ {
+ if (++bufpos == available)
+ AdjustBuffSize();
+
+ try
+ {
+ if ((buffer[bufpos] = c = ReadByte()) != '\\')
+ {
+ UpdateLineColumn(c);
+ // found a non-backslash char.
+ if ((c == 'u') && ((backSlashCnt & 1) == 1))
+ {
+ if (--bufpos < 0)
+ bufpos = bufsize - 1;
+
+ break;
+ }
+
+ backup(backSlashCnt);
+ return '\\';
+ }
+ }
+ catch(java.io.IOException e)
+ {
+ // We are returning one backslash so we should only backup (count-1)
+ if (backSlashCnt > 1)
+ backup(backSlashCnt-1);
+
+ return '\\';
+ }
+
+ UpdateLineColumn(c);
+ backSlashCnt++;
+ }
+
+ // Here, we have seen an odd number of backslash's followed by a 'u'
+ try
+ {
+ while ((c = ReadByte()) == 'u')
+ ++column;
+
+ buffer[bufpos] = c = (char)(hexval(c) << 12 |
+ hexval(ReadByte()) << 8 |
+ hexval(ReadByte()) << 4 |
+ hexval(ReadByte()));
+
+ column += 4;
+ }
+ catch(java.io.IOException e)
+ {
+ throw new Error("Invalid escape character at line " + line +
+ " column " + column + ".");
+ }
+
+ if (backSlashCnt == 1)
+ return c;
+ else
+ {
+ backup(backSlashCnt - 1);
+ return '\\';
+ }
+ }
+ else
+ {
+ UpdateLineColumn(c);
+ return c;
+ }
+ }
+
+ @Deprecated
+ /**
+ * @deprecated
+ * @see #getEndColumn
+ */
+ public int getColumn() {
+ return bufcolumn[bufpos];
+ }
+
+ @Deprecated
+ /**
+ * @deprecated
+ * @see #getEndLine
+ */
+ public int getLine() {
+ return bufline[bufpos];
+ }
+
+/** Get end column. */
+ public int getEndColumn() {
+ return bufcolumn[bufpos];
+ }
+
+/** Get end line. */
+ public int getEndLine() {
+ return bufline[bufpos];
+ }
+
+/** @return column of token start */
+ public int getBeginColumn() {
+ return bufcolumn[tokenBegin];
+ }
+
+/** @return line number of token start */
+ public int getBeginLine() {
+ return bufline[tokenBegin];
+ }
+
+/** Retreat. */
+ public void backup(int amount) {
+
+ inBuf += amount;
+ if ((bufpos -= amount) < 0)
+ bufpos += bufsize;
+ }
+
+/** Constructor. */
+ public JavaCharStream(java.io.Reader dstream,
+ int startline, int startcolumn, int buffersize)
+ {
+ inputStream = dstream;
+ line = startline;
+ column = startcolumn - 1;
+
+ available = bufsize = buffersize;
+ buffer = new char[buffersize];
+ bufline = new int[buffersize];
+ bufcolumn = new int[buffersize];
+ nextCharBuf = new char[4096];
+ }
+
+/** Constructor. */
+ public JavaCharStream(java.io.Reader dstream,
+ int startline, int startcolumn)
+ {
+ this(dstream, startline, startcolumn, 4096);
+ }
+
+/** Constructor. */
+ public JavaCharStream(java.io.Reader dstream)
+ {
+ this(dstream, 1, 1, 4096);
+ }
+/** Reinitialise. */
+ public void ReInit(java.io.Reader dstream,
+ int startline, int startcolumn, int buffersize)
+ {
+ inputStream = dstream;
+ line = startline;
+ column = startcolumn - 1;
+
+ if (buffer == null || buffersize != buffer.length)
+ {
+ available = bufsize = buffersize;
+ buffer = new char[buffersize];
+ bufline = new int[buffersize];
+ bufcolumn = new int[buffersize];
+ nextCharBuf = new char[4096];
+ }
+ prevCharIsLF = prevCharIsCR = false;
+ tokenBegin = inBuf = maxNextCharInd = 0;
+ nextCharInd = bufpos = -1;
+ }
+
+/** Reinitialise. */
+ public void ReInit(java.io.Reader dstream,
+ int startline, int startcolumn)
+ {
+ ReInit(dstream, startline, startcolumn, 4096);
+ }
+
+/** Reinitialise. */
+ public void ReInit(java.io.Reader dstream)
+ {
+ ReInit(dstream, 1, 1, 4096);
+ }
+/** Constructor. */
+ public JavaCharStream(java.io.InputStream dstream, String encoding, int startline,
+ int startcolumn, int buffersize) throws java.io.UnsupportedEncodingException
+ {
+ this(encoding == null ? new java.io.InputStreamReader(dstream) : new java.io.InputStreamReader(dstream, encoding), startline, startcolumn, buffersize);
+ }
+
+/** Constructor. */
+ public JavaCharStream(java.io.InputStream dstream, int startline,
+ int startcolumn, int buffersize)
+ {
+ this(new java.io.InputStreamReader(dstream), startline, startcolumn, 4096);
+ }
+
+/** Constructor. */
+ public JavaCharStream(java.io.InputStream dstream, String encoding, int startline,
+ int startcolumn) throws java.io.UnsupportedEncodingException
+ {
+ this(dstream, encoding, startline, startcolumn, 4096);
+ }
+
+/** Constructor. */
+ public JavaCharStream(java.io.InputStream dstream, int startline,
+ int startcolumn)
+ {
+ this(dstream, startline, startcolumn, 4096);
+ }
+
+/** Constructor. */
+ public JavaCharStream(java.io.InputStream dstream, String encoding) throws java.io.UnsupportedEncodingException
+ {
+ this(dstream, encoding, 1, 1, 4096);
+ }
+
+/** Constructor. */
+ public JavaCharStream(java.io.InputStream dstream)
+ {
+ this(dstream, 1, 1, 4096);
+ }
+
+/** Reinitialise. */
+ public void ReInit(java.io.InputStream dstream, String encoding, int startline,
+ int startcolumn, int buffersize) throws java.io.UnsupportedEncodingException
+ {
+ ReInit(encoding == null ? new java.io.InputStreamReader(dstream) : new java.io.InputStreamReader(dstream, encoding), startline, startcolumn, buffersize);
+ }
+
+/** Reinitialise. */
+ public void ReInit(java.io.InputStream dstream, int startline,
+ int startcolumn, int buffersize)
+ {
+ ReInit(new java.io.InputStreamReader(dstream), startline, startcolumn, buffersize);
+ }
+/** Reinitialise. */
+ public void ReInit(java.io.InputStream dstream, String encoding, int startline,
+ int startcolumn) throws java.io.UnsupportedEncodingException
+ {
+ ReInit(dstream, encoding, startline, startcolumn, 4096);
+ }
+/** Reinitialise. */
+ public void ReInit(java.io.InputStream dstream, int startline,
+ int startcolumn)
+ {
+ ReInit(dstream, startline, startcolumn, 4096);
+ }
+/** Reinitialise. */
+ public void ReInit(java.io.InputStream dstream, String encoding) throws java.io.UnsupportedEncodingException
+ {
+ ReInit(dstream, encoding, 1, 1, 4096);
+ }
+
+/** Reinitialise. */
+ public void ReInit(java.io.InputStream dstream)
+ {
+ ReInit(dstream, 1, 1, 4096);
+ }
+
+ /** @return token image as String */
+ public String GetImage()
+ {
+ if (bufpos >= tokenBegin)
+ return new String(buffer, tokenBegin, bufpos - tokenBegin + 1);
+ else
+ return new String(buffer, tokenBegin, bufsize - tokenBegin) +
+ new String(buffer, 0, bufpos + 1);
+ }
+
+ /** @return suffix */
+ public char[] GetSuffix(int len)
+ {
+ char[] ret = new char[len];
+
+ if ((bufpos + 1) >= len)
+ System.arraycopy(buffer, bufpos - len + 1, ret, 0, len);
+ else
+ {
+ System.arraycopy(buffer, bufsize - (len - bufpos - 1), ret, 0,
+ len - bufpos - 1);
+ System.arraycopy(buffer, 0, ret, len - bufpos - 1, bufpos + 1);
+ }
+
+ return ret;
+ }
+
+ /** Set buffers back to null when finished. */
+ public void Done()
+ {
+ nextCharBuf = null;
+ buffer = null;
+ bufline = null;
+ bufcolumn = null;
+ }
+
+ /**
+ * Method to adjust line and column numbers for the start of a token.
+ */
+ public void adjustBeginLineColumn(int newLine, int newCol)
+ {
+ int start = tokenBegin;
+ int len;
+
+ if (bufpos >= tokenBegin)
+ {
+ len = bufpos - tokenBegin + inBuf + 1;
+ }
+ else
+ {
+ len = bufsize - tokenBegin + bufpos + 1 + inBuf;
+ }
+
+ int i = 0, j = 0, k = 0;
+ int nextColDiff = 0, columnDiff = 0;
+
+ while (i < len && bufline[j = start % bufsize] == bufline[k = ++start % bufsize])
+ {
+ bufline[j] = newLine;
+ nextColDiff = columnDiff + bufcolumn[k] - bufcolumn[j];
+ bufcolumn[j] = newCol + columnDiff;
+ columnDiff = nextColDiff;
+ i++;
+ }
+
+ if (i < len)
+ {
+ bufline[j] = newLine++;
+ bufcolumn[j] = newCol + columnDiff;
+
+ while (i++ < len)
+ {
+ if (bufline[j = start % bufsize] != bufline[++start % bufsize])
+ bufline[j] = newLine++;
+ else
+ bufline[j] = newLine;
+ }
+ }
+
+ line = bufline[j];
+ column = bufcolumn[j];
+ }
+
+}
+/* JavaCC - OriginalChecksum=96a5b0b0fa09286690f250998f047719 (do not edit this line) */
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ise/ParseException.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ise/ParseException.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ise/ParseException.java
new file mode 100644
index 0000000..fc21aa1
--- /dev/null
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ise/ParseException.java
@@ -0,0 +1,204 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/* Generated By:JavaCC: Do not edit this line. ParseException.java Version 5.0 */
+/* JavaCCOptions:KEEP_LINE_COL=null */
+package org.apache.metron.parsers.ise;
+
+/**
+ * This exception is thrown when parse errors are encountered.
+ * You can explicitly create objects of this exception type by
+ * calling the method generateParseException in the generated
+ * parser.
+ *
+ * You can modify this class to customize your error reporting
+ * mechanisms so long as you retain the public fields.
+ */
+public class ParseException extends Exception {
+
+ /**
+ * The version identifier for this Serializable class.
+ * Increment only if the <i>serialized</i> form of the
+ * class changes.
+ */
+ private static final long serialVersionUID = 1L;
+
+ /**
+ * This constructor is used by the method "generateParseException"
+ * in the generated parser. Calling this constructor generates
+ * a new object of this type with the fields "currentToken",
+ * "expectedTokenSequences", and "tokenImage" set.
+ */
+ public ParseException(Token currentTokenVal,
+ int[][] expectedTokenSequencesVal,
+ String[] tokenImageVal
+ )
+ {
+ super(initialise(currentTokenVal, expectedTokenSequencesVal, tokenImageVal));
+ currentToken = currentTokenVal;
+ expectedTokenSequences = expectedTokenSequencesVal;
+ tokenImage = tokenImageVal;
+ }
+
+ /**
+ * The following constructors are for use by you for whatever
+ * purpose you can think of. Constructing the exception in this
+ * manner makes the exception behave in the normal way - i.e., as
+ * documented in the class "Throwable". The fields "errorToken",
+ * "expectedTokenSequences", and "tokenImage" do not contain
+ * relevant information. The JavaCC generated code does not use
+ * these constructors.
+ */
+
+ public ParseException() {
+ super();
+ }
+
+ /** Constructor with message. */
+ public ParseException(String message) {
+ super(message);
+ }
+
+
+ /**
+ * This is the last token that has been consumed successfully. If
+ * this object has been created due to a parse error, the token
+ * followng this token will (therefore) be the first error token.
+ */
+ public Token currentToken;
+
+ /**
+ * Each entry in this array is an array of integers. Each array
+ * of integers represents a sequence of tokens (by their ordinal
+ * values) that is expected at this point of the parse.
+ */
+ public int[][] expectedTokenSequences;
+
+ /**
+ * This is a reference to the "tokenImage" array of the generated
+ * parser within which the parse error occurred. This array is
+ * defined in the generated ...Constants interface.
+ */
+ public String[] tokenImage;
+
+ /**
+ * It uses "currentToken" and "expectedTokenSequences" to generate a parse
+ * error message and returns it. If this object has been created
+ * due to a parse error, and you do not catch it (it gets thrown
+ * from the parser) the correct error message
+ * gets displayed.
+ */
+ private static String initialise(Token currentToken,
+ int[][] expectedTokenSequences,
+ String[] tokenImage) {
+ String eol = System.getProperty("line.separator", "\n");
+ StringBuffer expected = new StringBuffer();
+ int maxSize = 0;
+ for (int i = 0; i < expectedTokenSequences.length; i++) {
+ if (maxSize < expectedTokenSequences[i].length) {
+ maxSize = expectedTokenSequences[i].length;
+ }
+ for (int j = 0; j < expectedTokenSequences[i].length; j++) {
+ expected.append(tokenImage[expectedTokenSequences[i][j]]).append(' ');
+ }
+ if (expectedTokenSequences[i][expectedTokenSequences[i].length - 1] != 0) {
+ expected.append("...");
+ }
+ expected.append(eol).append(" ");
+ }
+ String retval = "Encountered \"";
+ Token tok = currentToken.next;
+ for (int i = 0; i < maxSize; i++) {
+ if (i != 0) retval += " ";
+ if (tok.kind == 0) {
+ retval += tokenImage[0];
+ break;
+ }
+ retval += " " + tokenImage[tok.kind];
+ retval += " \"";
+ retval += add_escapes(tok.image);
+ retval += " \"";
+ tok = tok.next;
+ }
+ retval += "\" at line " + currentToken.next.beginLine + ", column " + currentToken.next.beginColumn;
+ retval += "." + eol;
+ if (expectedTokenSequences.length == 1) {
+ retval += "Was expecting:" + eol + " ";
+ } else {
+ retval += "Was expecting one of:" + eol + " ";
+ }
+ retval += expected.toString();
+ return retval;
+ }
+
+ /**
+ * The end of line string for this machine.
+ */
+ protected String eol = System.getProperty("line.separator", "\n");
+
+ /**
+ * Used to convert raw characters to their escaped version
+ * when these raw version cannot be used as part of an ASCII
+ * string literal.
+ */
+ static String add_escapes(String str) {
+ StringBuffer retval = new StringBuffer();
+ char ch;
+ for (int i = 0; i < str.length(); i++) {
+ switch (str.charAt(i))
+ {
+ case 0 :
+ continue;
+ case '\b':
+ retval.append("\\b");
+ continue;
+ case '\t':
+ retval.append("\\t");
+ continue;
+ case '\n':
+ retval.append("\\n");
+ continue;
+ case '\f':
+ retval.append("\\f");
+ continue;
+ case '\r':
+ retval.append("\\r");
+ continue;
+ case '\"':
+ retval.append("\\\"");
+ continue;
+ case '\'':
+ retval.append("\\\'");
+ continue;
+ case '\\':
+ retval.append("\\\\");
+ continue;
+ default:
+ if ((ch = str.charAt(i)) < 0x20 || ch > 0x7e) {
+ String s = "0000" + Integer.toString(ch, 16);
+ retval.append("\\u" + s.substring(s.length() - 4, s.length()));
+ } else {
+ retval.append(ch);
+ }
+ continue;
+ }
+ }
+ return retval.toString();
+ }
+
+}
+/* JavaCC - OriginalChecksum=f9f7217056f99de5708d01ebd497dede (do not edit this line) */
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ise/Token.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ise/Token.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ise/Token.java
new file mode 100644
index 0000000..3545ec4
--- /dev/null
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ise/Token.java
@@ -0,0 +1,148 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/* Generated By:JavaCC: Do not edit this line. Token.java Version 5.0 */
+/* JavaCCOptions:TOKEN_EXTENDS=,KEEP_LINE_COL=null,SUPPORT_CLASS_VISIBILITY_PUBLIC=false */
+package org.apache.metron.parsers.ise;
+
+/**
+ * Describes the input token stream.
+ */
+
+class Token implements java.io.Serializable {
+
+ /**
+ * The version identifier for this Serializable class.
+ * Increment only if the <i>serialized</i> form of the
+ * class changes.
+ */
+ private static final long serialVersionUID = 1L;
+
+ /**
+ * An integer that describes the kind of this token. This numbering
+ * system is determined by JavaCCParser, and a table of these numbers is
+ * stored in the file ...Constants.java.
+ */
+ public int kind;
+
+ /** The line number of the first character of this Token. */
+ public int beginLine;
+ /** The column number of the first character of this Token. */
+ public int beginColumn;
+ /** The line number of the last character of this Token. */
+ public int endLine;
+ /** The column number of the last character of this Token. */
+ public int endColumn;
+
+ /**
+ * The string image of the token.
+ */
+ public String image;
+
+ /**
+ * A reference to the next regular (non-special) token from the input
+ * stream. If this is the last token from the input stream, or if the
+ * token manager has not read tokens beyond this one, this field is
+ * set to null. This is true only if this token is also a regular
+ * token. Otherwise, see below for a description of the contents of
+ * this field.
+ */
+ public Token next;
+
+ /**
+ * This field is used to access special tokens that occur prior to this
+ * token, but after the immediately preceding regular (non-special) token.
+ * If there are no such special tokens, this field is set to null.
+ * When there are more than one such special token, this field refers
+ * to the last of these special tokens, which in turn refers to the next
+ * previous special token through its specialToken field, and so on
+ * until the first special token (whose specialToken field is null).
+ * The next fields of special tokens refer to other special tokens that
+ * immediately follow it (without an intervening regular token). If there
+ * is no such token, this field is null.
+ */
+ public Token specialToken;
+
+ /**
+ * An optional attribute value of the Token.
+ * Tokens which are not used as syntactic sugar will often contain
+ * meaningful values that will be used later on by the compiler or
+ * interpreter. This attribute value is often different from the image.
+ * Any subclass of Token that actually wants to return a non-null value can
+ * override this method as appropriate.
+ */
+ public Object getValue() {
+ return null;
+ }
+
+ /**
+ * No-argument constructor
+ */
+ public Token() {}
+
+ /**
+ * Constructs a new token for the specified Image.
+ */
+ public Token(int kind)
+ {
+ this(kind, null);
+ }
+
+ /**
+ * Constructs a new token for the specified Image and Kind.
+ */
+ public Token(int kind, String image)
+ {
+ this.kind = kind;
+ this.image = image;
+ }
+
+ /**
+ * Returns the image.
+ */
+ public String toString()
+ {
+ return image;
+ }
+
+ /**
+ * Returns a new Token object, by default. However, if you want, you
+ * can create and return subclass objects based on the value of ofKind.
+ * Simply add the cases to the switch for all those special cases.
+ * For example, if you have a subclass of Token called IDToken that
+ * you want to create if ofKind is ID, simply add something like :
+ *
+ * case MyParserConstants.ID : return new IDToken(ofKind, image);
+ *
+ * to the following switch statement. Then you can cast matchedToken
+ * variable to the appropriate type and use sit in your lexical actions.
+ */
+ public static Token newToken(int ofKind, String image)
+ {
+ switch(ofKind)
+ {
+ default : return new Token(ofKind, image);
+ }
+ }
+
+ public static Token newToken(int ofKind)
+ {
+ return newToken(ofKind, null);
+ }
+
+}
+/* JavaCC - OriginalChecksum=99daf0baa94b6c270eea5be0575db6aa (do not edit this line) */
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ise/TokenMgrError.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ise/TokenMgrError.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ise/TokenMgrError.java
new file mode 100644
index 0000000..2ccc23a
--- /dev/null
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ise/TokenMgrError.java
@@ -0,0 +1,164 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/* Generated By:JavaCC: Do not edit this line. TokenMgrError.java Version 5.0 */
+/* JavaCCOptions: */
+package org.apache.metron.parsers.ise;
+
+/** Token Manager Error. */
+class TokenMgrError extends Error
+{
+
+ /**
+ * The version identifier for this Serializable class.
+ * Increment only if the <i>serialized</i> form of the
+ * class changes.
+ */
+ private static final long serialVersionUID = 1L;
+
+ /*
+ * Ordinals for various reasons why an Error of this type can be thrown.
+ */
+
+ /**
+ * Lexical error occurred.
+ */
+ static final int LEXICAL_ERROR = 0;
+
+ /**
+ * An attempt was made to create a second instance of a static token manager.
+ */
+ static final int STATIC_LEXER_ERROR = 1;
+
+ /**
+ * Tried to change to an invalid lexical state.
+ */
+ static final int INVALID_LEXICAL_STATE = 2;
+
+ /**
+ * Detected (and bailed out of) an infinite loop in the token manager.
+ */
+ static final int LOOP_DETECTED = 3;
+
+ /**
+ * Indicates the reason why the exception is thrown. It will have
+ * one of the above 4 values.
+ */
+ int errorCode;
+
+ /**
+ * Replaces unprintable characters by their escaped (or unicode escaped)
+ * equivalents in the given string
+ */
+ protected static final String addEscapes(String str) {
+ StringBuffer retval = new StringBuffer();
+ char ch;
+ for (int i = 0; i < str.length(); i++) {
+ switch (str.charAt(i))
+ {
+ case 0 :
+ continue;
+ case '\b':
+ retval.append("\\b");
+ continue;
+ case '\t':
+ retval.append("\\t");
+ continue;
+ case '\n':
+ retval.append("\\n");
+ continue;
+ case '\f':
+ retval.append("\\f");
+ continue;
+ case '\r':
+ retval.append("\\r");
+ continue;
+ case '\"':
+ retval.append("\\\"");
+ continue;
+ case '\'':
+ retval.append("\\\'");
+ continue;
+ case '\\':
+ retval.append("\\\\");
+ continue;
+ default:
+ if ((ch = str.charAt(i)) < 0x20 || ch > 0x7e) {
+ String s = "0000" + Integer.toString(ch, 16);
+ retval.append("\\u" + s.substring(s.length() - 4, s.length()));
+ } else {
+ retval.append(ch);
+ }
+ continue;
+ }
+ }
+ return retval.toString();
+ }
+
+ /**
+ * Returns a detailed message for the Error when it is thrown by the
+ * token manager to indicate a lexical error.
+ * Parameters :
+ * EOFSeen : indicates if EOF caused the lexical error
+ * curLexState : lexical state in which this error occurred
+ * errorLine : line number when the error occurred
+ * errorColumn : column number when the error occurred
+ * errorAfter : prefix that was seen before this error occurred
+ * curchar : the offending character
+ * Note: You can customize the lexical error message by modifying this method.
+ */
+ protected static String LexicalError(boolean EOFSeen, int lexState, int errorLine, int errorColumn, String errorAfter, char curChar) {
+ return("Lexical error at line " +
+ errorLine + ", column " +
+ errorColumn + ". Encountered: " +
+ (EOFSeen ? "<EOF> " : ("\"" + addEscapes(String.valueOf(curChar)) + "\"") + " (" + (int)curChar + "), ") +
+ "after : \"" + addEscapes(errorAfter) + "\"");
+ }
+
+ /**
+ * You can also modify the body of this method to customize your error messages.
+ * For example, cases like LOOP_DETECTED and INVALID_LEXICAL_STATE are not
+ * of end-users concern, so you can return something like :
+ *
+ * "Internal Error : Please file a bug report .... "
+ *
+ * from this method for such cases in the release version of your parser.
+ */
+ public String getMessage() {
+ return super.getMessage();
+ }
+
+ /*
+ * Constructors of various flavors follow.
+ */
+
+ /** No arg constructor. */
+ public TokenMgrError() {
+ }
+
+ /** Constructor with message and reason. */
+ public TokenMgrError(String message, int reason) {
+ super(message);
+ errorCode = reason;
+ }
+
+ /** Full Constructor. */
+ public TokenMgrError(boolean EOFSeen, int lexState, int errorLine, int errorColumn, String errorAfter, char curChar, int reason) {
+ this(LexicalError(EOFSeen, lexState, errorLine, errorColumn, errorAfter, curChar), reason);
+ }
+}
+/* JavaCC - OriginalChecksum=5fbf6813c9d6a1d713f1d4a002af1322 (do not edit this line) */
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/lancope/BasicLancopeParser.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/lancope/BasicLancopeParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/lancope/BasicLancopeParser.java
new file mode 100644
index 0000000..6c25d67
--- /dev/null
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/lancope/BasicLancopeParser.java
@@ -0,0 +1,90 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.parsers.lancope;
+
+import org.apache.metron.parsers.BasicParser;
+import org.json.simple.JSONObject;
+import org.json.simple.JSONValue;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+
+@SuppressWarnings("serial")
+public class BasicLancopeParser extends BasicParser {
+ // Sample Lancope Message
+ // {"message":"<131>Jul 17 15:59:01 smc-01 StealthWatch[12365]: 2014-07-17T15:58:30Z 10.40.10.254 0.0.0.0 Minor High Concern Index The host's concern index has either exceeded the CI threshold or rapidly increased. Observed 36.55M points. Policy maximum allows up to 20M points.","@version":"1","@timestamp":"2014-07-17T15:56:05.992Z","type":"syslog","host":"10.122.196.201"}
+
+ private static final Logger _LOG = LoggerFactory.getLogger(BasicLancopeParser
+ .class);
+
+ @Override
+ public void init() {
+
+ }
+
+ //@SuppressWarnings("unchecked")
+ @Override
+ public List<JSONObject> parse(byte[] msg) {
+
+ JSONObject payload = null;
+ List<JSONObject> messages = new ArrayList<>();
+ try {
+
+ String raw_message = new String(msg, "UTF-8");
+
+ payload = (JSONObject) JSONValue.parse(raw_message);
+
+
+
+ String message = payload.get("message").toString();
+ String[] parts = message.split(" ");
+ payload.put("ip_src_addr", parts[6]);
+ payload.put("ip_dst_addr", parts[7]);
+
+ String fixed_date = parts[5].replace('T', ' ');
+ fixed_date = fixed_date.replace('Z', ' ').trim();
+
+ SimpleDateFormat formatter = new SimpleDateFormat(
+ "yyyy-MM-dd HH:mm:ss");
+
+ Date date;
+
+ date = formatter.parse(fixed_date);
+ long timestamp = date.getTime();
+ payload.put("timestamp", timestamp);
+
+ payload.remove("@timestamp");
+ payload.remove("message");
+ payload.put("original_string", message);
+
+ messages.add(payload);
+ return messages;
+ } catch (Exception e) {
+
+ _LOG.error("Unable to parse message: " + payload.toJSONString());
+ return null;
+ }
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/logstash/BasicLogstashParser.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/logstash/BasicLogstashParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/logstash/BasicLogstashParser.java
new file mode 100644
index 0000000..39177aa
--- /dev/null
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/logstash/BasicLogstashParser.java
@@ -0,0 +1,89 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.parsers.logstash;
+
+import org.apache.metron.parsers.BasicParser;
+import org.json.simple.JSONObject;
+import org.json.simple.parser.JSONParser;
+
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.List;
+
+public class BasicLogstashParser extends BasicParser {
+
+ @Override
+ public void init() {
+
+ }
+
+ @Override
+ public List<JSONObject> parse(byte[] raw_message) {
+ List<JSONObject> messages = new ArrayList<>();
+ try {
+
+ /*
+ * We need to create a new JSONParser each time because its
+ * not serializable and the parser is created on the storm nimbus
+ * node, then transfered to the workers.
+ */
+ JSONParser jsonParser = new JSONParser();
+ String rawString = new String(raw_message, "UTF-8");
+ JSONObject rawJson = (JSONObject) jsonParser.parse(rawString);
+
+ // remove logstash meta fields
+ rawJson.remove("@version");
+ rawJson.remove("type");
+ rawJson.remove("host");
+ rawJson.remove("tags");
+
+ // rename other keys
+ rawJson = mutate(rawJson, "message", "original_string");
+ rawJson = mutate(rawJson, "src_ip", "ip_src_addr");
+ rawJson = mutate(rawJson, "dst_ip", "ip_dst_addr");
+ rawJson = mutate(rawJson, "src_port", "ip_src_port");
+ rawJson = mutate(rawJson, "dst_port", "ip_dst_port");
+ rawJson = mutate(rawJson, "src_ip", "ip_src_addr");
+
+ // convert timestamp to milli since epoch
+ long timestamp = LogstashToEpoch((String) rawJson.remove("@timestamp"));
+ rawJson.put("timestamp", timestamp);
+ messages.add(rawJson);
+ return messages;
+ } catch (Exception e) {
+ e.printStackTrace();
+ return null;
+ }
+ }
+
+ private JSONObject mutate(JSONObject json, String oldKey, String newKey) {
+ if (json.containsKey(oldKey)) {
+ json.put(newKey, json.remove(oldKey));
+ }
+ return json;
+ }
+
+ private long LogstashToEpoch(String timestamp) throws java.text.ParseException {
+ SimpleDateFormat logstashDateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'");
+ return logstashDateFormat.parse(timestamp).getTime();
+
+ }
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParser.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParser.java
new file mode 100644
index 0000000..c67e2b5
--- /dev/null
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParser.java
@@ -0,0 +1,209 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.parsers.paloalto;
+
+
+import org.apache.metron.parsers.BasicParser;
+import org.json.simple.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
+
+public class BasicPaloAltoFirewallParser extends BasicParser {
+
+ private static final Logger _LOG = LoggerFactory.getLogger
+ (BasicPaloAltoFirewallParser.class);
+
+ private static final long serialVersionUID = 3147090149725343999L;
+ public static final String PaloAltoDomain = "palo_alto_domain";
+ public static final String ReceiveTime = "receive_time";
+ public static final String SerialNum = "serial_num";
+ public static final String Type = "type";
+ public static final String ThreatContentType = "threat_content_type";
+ public static final String ConfigVersion = "config_version";
+ public static final String GenerateTime = "generate_time";
+ public static final String SourceAddress = "source_address";
+ public static final String DestinationAddress = "destination_address";
+ public static final String NATSourceIP = "nat_source_ip";
+ public static final String NATDestinationIP = "nat_destination_ip";
+ public static final String Rule = "rule";
+ public static final String SourceUser = "source_user";
+ public static final String DestinationUser = "destination_user";
+ public static final String Application = "application";
+ public static final String VirtualSystem = "virtual_system";
+ public static final String SourceZone = "source_zone";
+ public static final String DestinationZone = "destination_zone";
+ public static final String InboundInterface = "inbound_interface";
+ public static final String OutboundInterface = "outbound_interface";
+ public static final String LogAction = "log_action";
+ public static final String TimeLogged = "time_logged";
+ public static final String SessionID = "session_id";
+ public static final String RepeatCount = "repeat_count";
+ public static final String SourcePort = "source_port";
+ public static final String DestinationPort = "destination_port";
+ public static final String NATSourcePort = "nats_source_port";
+ public static final String NATDestinationPort = "nats_destination_port";
+ public static final String Flags = "flags";
+ public static final String IPProtocol = "ip_protocol";
+ public static final String Action = "action";
+
+ //Threat
+ public static final String URL = "url";
+ public static final String HOST = "host";
+ public static final String ThreatContentName = "threat_content_name";
+ public static final String Category = "category";
+ public static final String Direction = "direction";
+ public static final String Seqno = "seqno";
+ public static final String ActionFlags = "action_flags";
+ public static final String SourceCountry = "source_country";
+ public static final String DestinationCountry = "destination_country";
+ public static final String Cpadding = "cpadding";
+ public static final String ContentType = "content_type";
+
+ //Traffic
+ public static final String Bytes = "content_type";
+ public static final String BytesSent = "content_type";
+ public static final String BytesReceived = "content_type";
+ public static final String Packets = "content_type";
+ public static final String StartTime = "content_type";
+ public static final String ElapsedTimeInSec = "content_type";
+ public static final String Padding = "content_type";
+ public static final String PktsSent = "pkts_sent";
+ public static final String PktsReceived = "pkts_received";
+
+
+ @Override
+ public void init() {
+
+ }
+
+ @SuppressWarnings({"unchecked", "unused"})
+ public List<JSONObject> parse(byte[] msg) {
+
+ JSONObject outputMessage = new JSONObject();
+ String toParse = "";
+ List<JSONObject> messages = new ArrayList<>();
+ try {
+
+ toParse = new String(msg, "UTF-8");
+ _LOG.debug("Received message: " + toParse);
+
+
+ parseMessage(toParse, outputMessage);
+ long timestamp = System.currentTimeMillis();
+ outputMessage.put("timestamp", System.currentTimeMillis());
+ outputMessage.put("ip_src_addr", outputMessage.remove("source_address"));
+ outputMessage.put("ip_src_port", outputMessage.remove("source_port"));
+ outputMessage.put("ip_dst_addr", outputMessage.remove("destination_address"));
+ outputMessage.put("ip_dst_port", outputMessage.remove("destination_port"));
+ outputMessage.put("protocol", outputMessage.remove("ip_protocol"));
+
+ outputMessage.put("original_string", toParse);
+ messages.add(outputMessage);
+ return messages;
+ } catch (Exception e) {
+ e.printStackTrace();
+ _LOG.error("Failed to parse: " + toParse);
+ return null;
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ private void parseMessage(String message, JSONObject outputMessage) {
+
+ String[] tokens = message.split(",");
+
+ String type = tokens[3].trim();
+
+ //populate common objects
+ outputMessage.put(PaloAltoDomain, tokens[0].trim());
+ outputMessage.put(ReceiveTime, tokens[1].trim());
+ outputMessage.put(SerialNum, tokens[2].trim());
+ outputMessage.put(Type, type);
+ outputMessage.put(ThreatContentType, tokens[4].trim());
+ outputMessage.put(ConfigVersion, tokens[5].trim());
+ outputMessage.put(GenerateTime, tokens[6].trim());
+ outputMessage.put(SourceAddress, tokens[7].trim());
+ outputMessage.put(DestinationAddress, tokens[8].trim());
+ outputMessage.put(NATSourceIP, tokens[9].trim());
+ outputMessage.put(NATDestinationIP, tokens[10].trim());
+ outputMessage.put(Rule, tokens[11].trim());
+ outputMessage.put(SourceUser, tokens[12].trim());
+ outputMessage.put(DestinationUser, tokens[13].trim());
+ outputMessage.put(Application, tokens[14].trim());
+ outputMessage.put(VirtualSystem, tokens[15].trim());
+ outputMessage.put(SourceZone, tokens[16].trim());
+ outputMessage.put(DestinationZone, tokens[17].trim());
+ outputMessage.put(InboundInterface, tokens[18].trim());
+ outputMessage.put(OutboundInterface, tokens[19].trim());
+ outputMessage.put(LogAction, tokens[20].trim());
+ outputMessage.put(TimeLogged, tokens[21].trim());
+ outputMessage.put(SessionID, tokens[22].trim());
+ outputMessage.put(RepeatCount, tokens[23].trim());
+ outputMessage.put(SourcePort, tokens[24].trim());
+ outputMessage.put(DestinationPort, tokens[25].trim());
+ outputMessage.put(NATSourcePort, tokens[26].trim());
+ outputMessage.put(NATDestinationPort, tokens[27].trim());
+ outputMessage.put(Flags, tokens[28].trim());
+ outputMessage.put(IPProtocol, tokens[29].trim());
+ outputMessage.put(Action, tokens[30].trim());
+
+
+ if ("THREAT".equals(type.toUpperCase())) {
+ outputMessage.put(URL, tokens[31].trim());
+ try {
+ URL url = new URL(tokens[31].trim());
+ outputMessage.put(HOST, url.getHost());
+ } catch (MalformedURLException e) {
+ }
+ outputMessage.put(ThreatContentName, tokens[32].trim());
+ outputMessage.put(Category, tokens[33].trim());
+ outputMessage.put(Direction, tokens[34].trim());
+ outputMessage.put(Seqno, tokens[35].trim());
+ outputMessage.put(ActionFlags, tokens[36].trim());
+ outputMessage.put(SourceCountry, tokens[37].trim());
+ outputMessage.put(DestinationCountry, tokens[38].trim());
+ outputMessage.put(Cpadding, tokens[39].trim());
+ outputMessage.put(ContentType, tokens[40].trim());
+
+ } else {
+ outputMessage.put(Bytes, tokens[31].trim());
+ outputMessage.put(BytesSent, tokens[32].trim());
+ outputMessage.put(BytesReceived, tokens[33].trim());
+ outputMessage.put(Packets, tokens[34].trim());
+ outputMessage.put(StartTime, tokens[35].trim());
+ outputMessage.put(ElapsedTimeInSec, tokens[36].trim());
+ outputMessage.put(Category, tokens[37].trim());
+ outputMessage.put(Padding, tokens[38].trim());
+ outputMessage.put(Seqno, tokens[39].trim());
+ outputMessage.put(ActionFlags, tokens[40].trim());
+ outputMessage.put(SourceCountry, tokens[41].trim());
+ outputMessage.put(DestinationCountry, tokens[42].trim());
+ outputMessage.put(Cpadding, tokens[43].trim());
+ outputMessage.put(PktsSent, tokens[44].trim());
+ outputMessage.put(PktsReceived, tokens[45].trim());
+ }
+
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/pcap/PcapParser.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/pcap/PcapParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/pcap/PcapParser.java
new file mode 100644
index 0000000..899dbd8
--- /dev/null
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/pcap/PcapParser.java
@@ -0,0 +1,229 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.parsers.pcap;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.log4j.Logger;
+import org.apache.metron.parsers.interfaces.MessageParser;
+import org.apache.metron.pcap.Constants;
+import org.apache.metron.pcap.MetronEthernetDecoder;
+import org.apache.metron.pcap.PacketInfo;
+import org.apache.metron.pcap.PcapByteInputStream;
+import org.json.simple.JSONObject;
+import org.json.simple.JSONValue;
+import org.krakenapps.pcap.decoder.ethernet.EthernetDecoder;
+import org.krakenapps.pcap.decoder.ethernet.EthernetType;
+import org.krakenapps.pcap.decoder.ip.IpDecoder;
+import org.krakenapps.pcap.decoder.ip.Ipv4Packet;
+import org.krakenapps.pcap.decoder.tcp.TcpPacket;
+import org.krakenapps.pcap.decoder.udp.UdpPacket;
+import org.krakenapps.pcap.file.GlobalHeader;
+import org.krakenapps.pcap.packet.PacketHeader;
+import org.krakenapps.pcap.packet.PcapPacket;
+import org.krakenapps.pcap.util.Buffer;
+
+import java.io.EOFException;
+import java.io.File;
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+public class PcapParser implements MessageParser<JSONObject>, Serializable {
+
+ private static final Logger LOG = Logger.getLogger(PcapParser.class);
+
+ private EthernetDecoder ethernetDecoder;
+ private long timePrecisionDivisor = 1L;
+
+ public PcapParser withTsPrecision(String tsPrecision) {
+ if (tsPrecision.equalsIgnoreCase("MILLI")) {
+ //Convert nanos to millis
+ LOG.info("Configured for MILLI, setting timePrecisionDivisor to 1000000L" );
+ timePrecisionDivisor = 1000000L;
+ } else if (tsPrecision.equalsIgnoreCase("MICRO")) {
+ //Convert nanos to micro
+ LOG.info("Configured for MICRO, setting timePrecisionDivisor to 1000L" );
+ timePrecisionDivisor = 1000L;
+ } else if (tsPrecision.equalsIgnoreCase("NANO")) {
+ //Keep nano as is.
+ LOG.info("Configured for NANO, setting timePrecisionDivisor to 1L" );
+ timePrecisionDivisor = 1L;
+ } else {
+ LOG.info("bolt.parser.ts.precision not set. Default to NANO");
+ timePrecisionDivisor = 1L;
+ }
+ return this;
+ }
+
+ @Override
+ public void init() {
+ ethernetDecoder = new MetronEthernetDecoder();
+ IpDecoder ipDecoder = new IpDecoder();
+ ethernetDecoder.register(EthernetType.IPV4, ipDecoder);
+ }
+
+ @Override
+ public List<JSONObject> parse(byte[] pcap) {
+ List<JSONObject> messages = new ArrayList<>();
+ List<PacketInfo> packetInfoList = new ArrayList<>();
+ try {
+ packetInfoList = getPacketInfo(pcap);
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ for (PacketInfo packetInfo : packetInfoList) {
+ JSONObject message = (JSONObject) JSONValue.parse(packetInfo.getJsonIndexDoc());
+ messages.add(message);
+ }
+ return messages;
+ }
+
+ @Override
+ public boolean validate(JSONObject message) {
+ List<String> requiredFields = Arrays.asList("ip_src_addr",
+ "ip_dst_addr",
+ "ip_protocol",
+ "ip_src_port",
+ "ip_dst_port");
+ return message.keySet().containsAll(requiredFields);
+
+ }
+
+ /**
+ * Parses the.
+ *
+ * @param pcap
+ * the pcap
+ * @return the list * @throws IOException Signals that an I/O exception has
+ * occurred. * @throws IOException * @throws IOException * @throws
+ * IOException
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ */
+ public List<PacketInfo> getPacketInfo(byte[] pcap) throws IOException {
+ List<PacketInfo> packetInfoList = new ArrayList<PacketInfo>();
+
+ PcapByteInputStream pcapByteInputStream = new PcapByteInputStream(pcap);
+
+ GlobalHeader globalHeader = pcapByteInputStream.getGlobalHeader();
+ while (true) {
+ try
+
+ {
+ PcapPacket packet = pcapByteInputStream.getPacket();
+ // int packetCounter = 0;
+ // PacketHeader packetHeader = null;
+ // Ipv4Packet ipv4Packet = null;
+ TcpPacket tcpPacket = null;
+ UdpPacket udpPacket = null;
+ // Buffer packetDataBuffer = null;
+ int sourcePort = 0;
+ int destinationPort = 0;
+
+ // LOG.trace("Got packet # " + ++packetCounter);
+
+ // LOG.trace(packet.getPacketData());
+ ethernetDecoder.decode(packet);
+
+ PacketHeader packetHeader = packet.getPacketHeader();
+ Ipv4Packet ipv4Packet = Ipv4Packet.parse(packet.getPacketData());
+
+ if (ipv4Packet.getProtocol() == Constants.PROTOCOL_TCP) {
+ tcpPacket = TcpPacket.parse(ipv4Packet);
+
+ }
+
+ if (ipv4Packet.getProtocol() == Constants.PROTOCOL_UDP) {
+
+ Buffer packetDataBuffer = ipv4Packet.getData();
+ sourcePort = packetDataBuffer.getUnsignedShort();
+ destinationPort = packetDataBuffer.getUnsignedShort();
+
+ udpPacket = new UdpPacket(ipv4Packet, sourcePort, destinationPort);
+
+ udpPacket.setLength(packetDataBuffer.getUnsignedShort());
+ udpPacket.setChecksum(packetDataBuffer.getUnsignedShort());
+ packetDataBuffer.discardReadBytes();
+ udpPacket.setData(packetDataBuffer);
+ }
+
+ packetInfoList.add(new PacketInfo(globalHeader, packetHeader, packet,
+ ipv4Packet, tcpPacket, udpPacket));
+ } catch (NegativeArraySizeException ignored) {
+ LOG.debug("Ignorable exception while parsing packet.", ignored);
+ } catch (EOFException eof) { // $codepro.audit.disable logExceptions
+ // Ignore exception and break
+ break;
+ }
+ }
+ return packetInfoList;
+ }
+
+ /**
+ * The main method.
+ *
+ * @param args
+ * the arguments
+ * @throws IOException
+ * Signals that an I/O exception has occurred.
+ * @throws InterruptedException
+ * the interrupted exception
+ */
+ public static void main(String[] args) throws IOException,
+ InterruptedException {
+
+ double totalIterations = 1000000;
+ double parallelism = 64;
+ double targetEvents = 1000000;
+ PcapParser pcapParser = new PcapParser();
+ File fin = new File("/Users/sheetal/Downloads/bad_packets/bad_packet_1405988125427.pcap");
+ File fout = new File(fin.getAbsolutePath() + ".parsed");
+ byte[] pcapBytes = FileUtils.readFileToByteArray(fin);
+ long startTime = System.currentTimeMillis();
+ for (int i = 0; i < totalIterations; i++) {
+ List<PacketInfo> list = pcapParser.getPacketInfo(pcapBytes);
+
+ for (PacketInfo packetInfo : list) {
+ System.out.println(packetInfo.getJsonIndexDoc());
+ }
+ }
+ long endTime = System.currentTimeMillis();
+
+ System.out.println("Time taken to process " + totalIterations + " events :"
+ + (endTime - startTime) + " milliseconds");
+
+ System.out
+ .println("With parallelism of "
+ + parallelism
+ + " estimated time to process "
+ + targetEvents
+ + " events: "
+ + (((((endTime - startTime) / totalIterations) * targetEvents) / parallelism) / 1000)
+ + " seconds");
+ System.out.println("With parallelism of " + parallelism
+ + " estimated # of events per second: "
+ + ((parallelism * 1000 * totalIterations) / (endTime - startTime))
+ + " events");
+ System.out.println("Expected Parallelism to process " + targetEvents
+ + " events in a second: "
+ + (targetEvents / ((1000 * totalIterations) / (endTime - startTime))));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/snort/BasicSnortParser.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/snort/BasicSnortParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/snort/BasicSnortParser.java
new file mode 100644
index 0000000..4455e57
--- /dev/null
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/snort/BasicSnortParser.java
@@ -0,0 +1,163 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.parsers.snort;
+
+import org.apache.metron.parsers.BasicParser;
+import org.json.simple.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.*;
+
+@SuppressWarnings("serial")
+public class BasicSnortParser extends BasicParser {
+
+ private static final Logger _LOG = LoggerFactory
+ .getLogger(BasicSnortParser.class);
+
+ /**
+ * The default field names for Snort Alerts.
+ */
+ private String[] fieldNames = new String[] {
+ "timestamp",
+ "sig_generator",
+ "sig_id",
+ "sig_rev",
+ "msg",
+ "protocol",
+ "ip_src_addr",
+ "ip_src_port",
+ "ip_dst_addr",
+ "ip_dst_port",
+ "ethsrc",
+ "ethdst",
+ "ethlen",
+ "tcpflags",
+ "tcpseq",
+ "tcpack",
+ "tcplen",
+ "tcpwindow",
+ "ttl",
+ "tos",
+ "id",
+ "dgmlen",
+ "iplen",
+ "icmptype",
+ "icmpcode",
+ "icmpid",
+ "icmpseq"
+ };
+
+
+ /**
+ * Snort alerts are received as CSV records
+ */
+ private String recordDelimiter = ",";
+
+ @Override
+ public void init() {
+
+ }
+
+ @Override
+ public List<JSONObject> parse(byte[] rawMessage) {
+
+ JSONObject jsonMessage = new JSONObject();
+ List<JSONObject> messages = new ArrayList<>();
+ try {
+ // snort alerts expected as csv records
+ String csvMessage = new String(rawMessage, "UTF-8");
+ String[] records = csvMessage.split(recordDelimiter, -1);
+
+ // validate the number of fields
+ if (records.length != fieldNames.length) {
+ throw new IllegalArgumentException("Unexpected number of fields, expected: " + fieldNames.length + " got: " + records.length);
+ }
+ long timestamp = 0L;
+ // build the json record from each field
+ for (int i=0; i<records.length; i++) {
+
+ String field = fieldNames[i];
+ String record = records[i];
+
+ if("timestamp".equals(field)) {
+
+ // convert the timestamp to epoch
+ timestamp = toEpoch(record);
+ jsonMessage.put("timestamp", timestamp);
+
+ } else {
+ jsonMessage.put(field, record);
+ }
+ }
+
+ // add original msg; required by 'checkForSchemaCorrectness'
+ jsonMessage.put("original_string", csvMessage);
+ jsonMessage.put("is_alert", "true");
+ messages.add(jsonMessage);
+ } catch (Exception e) {
+
+ _LOG.error("unable to parse message: " + rawMessage);
+ e.printStackTrace();
+ return null;
+ }
+
+ return messages;
+ }
+
+ /**
+ * Parses Snort's default date-time representation and
+ * converts to epoch.
+ * @param snortDatetime Snort's default date-time as String '01/27-16:01:04.877970'
+ * @return epoch time
+ * @throws java.text.ParseException
+ */
+ private long toEpoch(String snortDatetime) throws ParseException {
+
+ /*
+ * TODO how does Snort not embed the year in their default timestamp?! need to change this in
+ * Snort configuration. for now, just assume current year.
+ */
+ int year = Calendar.getInstance().get(Calendar.YEAR);
+ String withYear = Integer.toString(year) + " " + snortDatetime;
+
+ // convert to epoch time
+ SimpleDateFormat df = new SimpleDateFormat("yyyy MM/dd-HH:mm:ss.S");
+ Date date = df.parse(withYear);
+ return date.getTime();
+ }
+
+ public String getRecordDelimiter() {
+ return this.recordDelimiter;
+ }
+
+ public void setRecordDelimiter(String recordDelimiter) {
+ this.recordDelimiter = recordDelimiter;
+ }
+
+ public String[] getFieldNames() {
+ return this.fieldNames;
+ }
+
+ public void setFieldNames(String[] fieldNames) {
+ this.fieldNames = fieldNames;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/sourcefire/BasicSourcefireParser.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/sourcefire/BasicSourcefireParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/sourcefire/BasicSourcefireParser.java
new file mode 100644
index 0000000..40badcd
--- /dev/null
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/sourcefire/BasicSourcefireParser.java
@@ -0,0 +1,122 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.parsers.sourcefire;
+
+import org.apache.metron.parsers.BasicParser;
+import org.json.simple.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+@SuppressWarnings("serial")
+public class BasicSourcefireParser extends BasicParser {
+
+ private static final Logger _LOG = LoggerFactory
+ .getLogger(BasicSourcefireParser.class);
+
+ public static final String hostkey = "host";
+ String domain_name_regex = "([^\\.]+)\\.([a-z]{2}|[a-z]{3}|([a-z]{2}\\.[a-z]{2}))$";
+ String sidRegex = "(.*)(\\[[0-9]+:[0-9]+:[0-9]\\])(.*)$";
+ //String sidRegex = "(\\[[0-9]+:[0-9]+:[0-9]\\])(.*)$";
+ Pattern sidPattern = Pattern.compile(sidRegex);
+ Pattern pattern = Pattern.compile(domain_name_regex);
+
+ @Override
+ public void init() {
+
+ }
+
+ @SuppressWarnings({ "unchecked", "unused" })
+ public List<JSONObject> parse(byte[] msg) {
+
+ JSONObject payload = new JSONObject();
+ String toParse = "";
+ List<JSONObject> messages = new ArrayList<>();
+ try {
+
+ toParse = new String(msg, "UTF-8");
+ _LOG.debug("Received message: " + toParse);
+
+ String tmp = toParse.substring(toParse.lastIndexOf("{"));
+ payload.put("key", tmp);
+
+ String protocol = tmp.substring(tmp.indexOf("{") + 1,
+ tmp.indexOf("}")).toLowerCase();
+ String source = tmp.substring(tmp.indexOf("}") + 1,
+ tmp.indexOf("->")).trim();
+ String dest = tmp.substring(tmp.indexOf("->") + 2, tmp.length())
+ .trim();
+
+ payload.put("protocol", protocol);
+
+ String source_ip = "";
+ String dest_ip = "";
+
+ if (source.contains(":")) {
+ String parts[] = source.split(":");
+ payload.put("ip_src_addr", parts[0]);
+ payload.put("ip_src_port", parts[1]);
+ source_ip = parts[0];
+ } else {
+ payload.put("ip_src_addr", source);
+ source_ip = source;
+
+ }
+
+ if (dest.contains(":")) {
+ String parts[] = dest.split(":");
+ payload.put("ip_dst_addr", parts[0]);
+ payload.put("ip_dst_port", parts[1]);
+ dest_ip = parts[0];
+ } else {
+ payload.put("ip_dst_addr", dest);
+ dest_ip = dest;
+ }
+ long timestamp = System.currentTimeMillis();
+ payload.put("timestamp", timestamp);
+
+ Matcher sidMatcher = sidPattern.matcher(toParse);
+ String originalString = null;
+ String signatureId = "";
+ if (sidMatcher.find()) {
+ signatureId = sidMatcher.group(2);
+ originalString = sidMatcher.group(1) +" "+ sidMatcher.group(2) + " " + sidMatcher.group(3);
+ } else {
+ _LOG.warn("Unable to find SID in message: " + toParse);
+ originalString = toParse;
+ }
+ payload.put("original_string", originalString);
+ payload.put("signature_id", signatureId);
+ messages.add(payload);
+ return messages;
+ } catch (Exception e) {
+ e.printStackTrace();
+ _LOG.error("Failed to parse: " + toParse);
+ return null;
+ }
+ }
+
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/utils/GrokUtils.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/utils/GrokUtils.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/utils/GrokUtils.java
new file mode 100644
index 0000000..e60e5f8
--- /dev/null
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/utils/GrokUtils.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.parsers.utils;
+import java.io.Serializable;
+
+import com.google.code.regexp.Pattern;
+
+public class GrokUtils implements Serializable {
+
+ private static final long serialVersionUID = 7465176887422419286L;
+ /**
+ * Extract Grok patter like %{FOO} to FOO, Also Grok pattern with semantic.
+ */
+ public static final Pattern GROK_PATTERN = Pattern.compile(
+ "%\\{" +
+ "(?<name>" +
+ "(?<pattern>[A-z0-9]+)" +
+ "(?::(?<subname>[A-z0-9_:;\\/\\s\\.]+))?" +
+ ")" +
+ "(?:=(?<definition>" +
+ "(?:" +
+ "(?:[^{}]+|\\.+)+" +
+ ")+" +
+ ")" +
+ ")?" +
+ "\\}");
+
+ }
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/utils/ParserUtils.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/utils/ParserUtils.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/utils/ParserUtils.java
new file mode 100644
index 0000000..f98f996
--- /dev/null
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/utils/ParserUtils.java
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.parsers.utils;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+
+import org.apache.commons.io.IOUtils;
+import org.json.simple.JSONObject;
+
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.Locale;
+import java.util.TimeZone;
+
+public class ParserUtils {
+
+ public static final String PREFIX = "stream2file";
+ public static final String SUFFIX = ".tmp";
+
+ public static File stream2file(InputStream in) throws IOException {
+ final File tempFile = File.createTempFile(PREFIX, SUFFIX);
+ tempFile.deleteOnExit();
+ try (FileOutputStream out = new FileOutputStream(tempFile)) {
+ IOUtils.copy(in, out);
+ }
+ return tempFile;
+ }
+
+ public static Long convertToEpoch(String m, String d, String ts,
+ boolean adjust_timezone) throws ParseException {
+ d = d.trim();
+ if (d.length() <= 2) {
+ d = "0" + d;
+ }
+ Date date = new SimpleDateFormat("MMM", Locale.ENGLISH).parse(m);
+ Calendar cal = Calendar.getInstance();
+ cal.setTime(date);
+ String month = String.valueOf(cal.get(Calendar.MONTH));
+ int year = Calendar.getInstance().get(Calendar.YEAR);
+ if (month.length() <= 2) {
+ month = "0" + month;
+ }
+ String coglomerated_ts = year + "-" + month + "-" + d + " " + ts;
+ SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ if (adjust_timezone) {
+ sdf.setTimeZone(TimeZone.getTimeZone("GMT"));
+ }
+ date = sdf.parse(coglomerated_ts);
+ long timeInMillisSinceEpoch = date.getTime();
+ return timeInMillisSinceEpoch;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/writer/KafkaWriter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/writer/KafkaWriter.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/writer/KafkaWriter.java
new file mode 100644
index 0000000..d168e29
--- /dev/null
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/writer/KafkaWriter.java
@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.parsers.writer;
+
+import backtype.storm.tuple.Tuple;
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.metron.common.Constants;
+import org.apache.metron.common.configuration.Configurations;
+import org.apache.metron.common.interfaces.MessageWriter;
+import org.json.simple.JSONObject;
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.Map;
+
+public class KafkaWriter implements MessageWriter<JSONObject>, Serializable {
+
+ private String brokerUrl;
+ private String keySerializer = "org.apache.kafka.common.serialization.StringSerializer";
+ private String valueSerializer = "org.apache.kafka.common.serialization.StringSerializer";
+ private int requiredAcks = 1;
+ private KafkaProducer kafkaProducer;
+
+ public KafkaWriter(String brokerUrl) {
+ this.brokerUrl = brokerUrl;
+ }
+
+ public KafkaWriter withKeySerializer(String keySerializer) {
+ this.keySerializer = keySerializer;
+ return this;
+ }
+
+ public KafkaWriter withValueSerializer(String valueSerializer) {
+ this.valueSerializer = valueSerializer;
+ return this;
+ }
+
+ public KafkaWriter withRequiredAcks(int requiredAcks) {
+ this.requiredAcks = requiredAcks;
+ return this;
+ }
+
+ @Override
+ public void init() {
+ Map<String, Object> producerConfig = new HashMap<>();
+ producerConfig.put("bootstrap.servers", brokerUrl);
+ producerConfig.put("key.serializer", keySerializer);
+ producerConfig.put("value.serializer", valueSerializer);
+ producerConfig.put("request.required.acks", requiredAcks);
+ this.kafkaProducer = new KafkaProducer<>(producerConfig);
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public void write(String sourceType, Configurations configurations, Tuple tuple, JSONObject message) throws Exception {
+ kafkaProducer.send(new ProducerRecord<String, String>(Constants.ENRICHMENT_TOPIC, message.toJSONString()));
+ }
+
+ @Override
+ public void close() throws Exception {
+ kafkaProducer.close();
+ }
+}