You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@camel.apache.org by Johan Edstrom <se...@gmail.com> on 2010/10/20 04:34:05 UTC
Design advice. - Syslog DataFormat.
Hey
Need some design advice.
I have an RFC 3164 (BSD Syslog protocol) ready DataFormat and the necessary converters.
I wanted this to work with both Netty UDP and Mina UDP after Willems comment, and it does not have to
be specific whatsoever to the network library, it is just byte[] parsing really.
The test route looks like :
protected RouteBuilder createRouteBuilder() throws Exception {
return new RouteBuilder() {
public void configure() throws Exception {
context.setTracing(true);
DataFormat syslogDataFormat = new Rfc3164SyslogDataFormat();
// we setup a Syslog listener on a random port.
from("netty:udp://127.0.0.1:" + serverPort + "?sync=false")
.unmarshal(syslogDataFormat).process(new Processor() {
public void process(Exchange ex) {
assertTrue(ex.getIn().getBody() instanceof SyslogMessage);
}
}).to("mock:syslogReceiver").
marshal(syslogDataFormat).to("mock:syslogReceiver2");
}
};
And the @Test like
@Test
public void testSendingRawUDP() throws IOException, InterruptedException {
MockEndpoint mock = getMockEndpoint("mock:syslogReceiver");
MockEndpoint mock2 = getMockEndpoint("mock:syslogReceiver2");
mock.expectedMessageCount(1);
mock2.expectedMessageCount(1);
mock2.expectedBodiesReceived(message);
DatagramSocket ds = new DatagramSocket();
DatagramPacket packet = new DatagramPacket(message.getBytes(), message.getBytes().length,
InetAddress.getByName("localhost"), serverPort);
ds.send(packet);
mock2.assertIsSatisfied();
mock.assertIsSatisfied();
mock.reset();
mock2.reset();
}
I think I also would want this to be able to handle things like syslog-ng and the syslog protocol in the new RFC nobody implements correctly.
Right now I have this.. (this being what is below the rest of the text)
I.e the question will be - what is the cleanest way of writing a "factory like" dataformat?
I'd suspect I'd get this into a bytebuffer, look for indicators, rewind, pick parser and get a message
and then on the marshal side, is there a clean way I could pick a way that would make it into the DSL?
Sorry if this sounds retarded.
/je
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.camel.component.syslog;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.nio.ByteBuffer;
import java.util.Calendar;
import java.util.Date;
import java.util.GregorianCalendar;
import java.util.HashMap;
import java.util.Map;
import org.apache.camel.Converter;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
public class Rfc3164SyslogConverter {
private static final transient Log LOG = LogFactory.getLog(Rfc3164SyslogConverter.class);
private static enum MONTHS {
jan, feb, mar, apr, may, jun, jul, aug, sep, oct, nov, dec
}
private final static Map<String, MONTHS> MONTH_VALUE_MAP = new HashMap<String, MONTHS>() {
{
put("jan", MONTHS.jan);
put("feb", MONTHS.feb);
put("mar", MONTHS.mar);
put("apr", MONTHS.apr);
put("may", MONTHS.may);
put("jun", MONTHS.jun);
put("jul", MONTHS.jul);
put("aug", MONTHS.aug);
put("sep", MONTHS.sep);
put("oct", MONTHS.oct);
put("nov", MONTHS.nov);
put("dec", MONTHS.dec);
}
};
@Converter
public static String toString(SyslogMessage message) {
StringBuilder sbr = new StringBuilder();
sbr.append("<");
if (message.getFacility() == null) {
message.setFacility(SyslogFacility.USER);
}
if (message.getSeverity() == null) {
message.setSeverity(SyslogSeverity.INFO);
}
if (message.getHostname() == null) {
//This is massively ugly..
try {
message.setHostname(InetAddress.getLocalHost().toString());
} catch (UnknownHostException e) {
message.setHostname("UNKNOWN_HOST");
}
}
sbr.append(message.getFacility().ordinal() * 8 + message.getSeverity().ordinal());
sbr.append(">");
if (message.getTimestamp() == null) {
message.setTimestamp(new Date());
}
//SDF isn't going to help much here.
Calendar cal = GregorianCalendar.getInstance();
cal.setTime(message.getTimestamp());
String firstLetter = MONTHS.values()[cal.get(Calendar.MONTH)].toString().substring(0, 1); // Get first letter
String remainder = MONTHS.values()[cal.get(Calendar.MONTH)].toString()
.substring(1); // Get remainder of word.
String capitalized = firstLetter.toUpperCase() + remainder.toLowerCase();
sbr.append(capitalized);
sbr.append(" ");
if (cal.get(Calendar.DAY_OF_MONTH) < 10) {
sbr.append(" ").append(cal.get(Calendar.DAY_OF_MONTH));
} else {
sbr.append(cal.get(Calendar.DAY_OF_MONTH));
}
sbr.append(" ");
if (cal.get(Calendar.HOUR_OF_DAY) < 10) {
sbr.append("0").append(cal.get(Calendar.HOUR_OF_DAY));
} else {
sbr.append(cal.get(Calendar.HOUR_OF_DAY));
}
sbr.append(":");
if (cal.get(Calendar.MINUTE) < 10) {
sbr.append("0").append(cal.get(Calendar.MINUTE));
} else {
sbr.append(cal.get(Calendar.MINUTE));
}
sbr.append(":");
if (cal.get(Calendar.SECOND) < 10) {
sbr.append("0").append(cal.get(Calendar.SECOND));
} else {
sbr.append(cal.get(Calendar.SECOND));
}
sbr.append(" ");
sbr.append(message.getHostname());
sbr.append(" ");
sbr.append(message.getLogMessage());
return sbr.toString();
}
@Converter
public static SyslogMessage toSyslogMessage(String body) {
return parseMessage(body.getBytes());
}
public final static SyslogMessage parseMessage(byte[] bytes) {
ByteBuffer byteBuffer = ByteBuffer.allocate(bytes.length);
byteBuffer.put(bytes);
byteBuffer.rewind();
SyslogMessage syslogMessage = new SyslogMessage();
Character charFound = (char) byteBuffer.get();
while (charFound != '<') {
//Ignore noise in beginning of message.
charFound = (char) byteBuffer.get();
}
char priChar = 0;
if (charFound == '<') {
int facility = 0;
while (Character.isDigit(priChar = (char) (byteBuffer.get() & 0xff))) {
facility *= 10;
facility += Character.digit(priChar, 10);
}
syslogMessage.setFacility(SyslogFacility.values()[facility >> 3]);
syslogMessage.setSeverity(SyslogSeverity.values()[facility & 0x07]);
}
if (priChar != '>') {
//Invalid character - this is not a well defined syslog message.
LOG.error("Invalid syslog message, missing a > in the Facility/Priority part");
}
//Done parsing severity and facility
//<169>Oct 22 10:52:01 TZ-6 scapegoat.dmz.example.org 10.1.2.3 sched[0]: That's All Folks!
//Need to parse the date.
/**
The TIMESTAMP field is the local time and is in the format of "Mmm dd
hh:mm:ss" (without the quote marks) where:
Mmm is the English language abbreviation for the month of the
year with the first character in uppercase and the other two
characters in lowercase. The following are the only acceptable
values:
Jan, Feb, Mar, Apr, May, Jun, Jul, Aug, Sep, Oct, Nov, Dec
dd is the day of the month. If the day of the month is less
than 10, then it MUST be represented as a space and then the
number. For example, the 7th day of August would be
represented as "Aug 7", with two spaces between the "g" and
the "7".
hh:mm:ss is the local time. The hour (hh) is represented in a
24-hour format. Valid entries are between 00 and 23,
inclusive. The minute (mm) and second (ss) entries are between
00 and 59 inclusive.
*/
char[] month = new char[3];
for (int i = 0; i < 3; i++) {
month[i] = (char) (byteBuffer.get() & 0xff);
}
charFound = (char) byteBuffer.get();
if (charFound != ' ') {
//Invalid Message - missing mandatory space.
LOG.error("Invalid syslog message, missing a mandatory space after month");
}
charFound = (char) (byteBuffer.get() & 0xff);
int day = 0;
if (charFound == ' ') {
//Extra space for the day - this is okay.
//Just ignored per the spec.
} else {
day *= 10;
day += Character.digit(charFound, 10);
}
while (Character.isDigit(charFound = (char) (byteBuffer.get() & 0xff))) {
day *= 10;
day += Character.digit(charFound, 10);
}
int hour = 0;
while (Character.isDigit(charFound = (char) (byteBuffer.get() & 0xff))) {
hour *= 10;
hour += Character.digit(charFound, 10);
}
int minute = 0;
while (Character.isDigit(charFound = (char) (byteBuffer.get() & 0xff))) {
minute *= 10;
minute += Character.digit(charFound, 10);
}
int second = 0;
while (Character.isDigit(charFound = (char) (byteBuffer.get() & 0xff))) {
second *= 10;
second += Character.digit(charFound, 10);
}
//The host is the char sequence until the next ' '
StringBuilder host = new StringBuilder();
while ((charFound = (char) (byteBuffer.get() & 0xff)) != ' ') {
host.append(charFound);
}
syslogMessage.setHostname(host.toString());
StringBuilder msg = new StringBuilder();
while (byteBuffer.hasRemaining()) {
charFound = (char) (byteBuffer.get() & 0xff);
msg.append(charFound);
}
Calendar calendar = new GregorianCalendar();
calendar.set(Calendar.MONTH, MONTH_VALUE_MAP.get((String.valueOf(month).toLowerCase())).ordinal());
calendar.set(Calendar.DAY_OF_MONTH, day);
calendar.set(Calendar.HOUR_OF_DAY, hour);
calendar.set(Calendar.MINUTE, minute);
calendar.set(Calendar.SECOND, second);
syslogMessage.setTimestamp(calendar.getTime());
syslogMessage.setLogMessage(msg.toString());
if (LOG.isTraceEnabled()) {
LOG.trace("Syslog message : " + syslogMessage.toString());
}
return syslogMessage;
}
}
Johan Edstrom
Re: Design advice. - Syslog DataFormat.
Posted by Claus Ibsen <cl...@gmail.com>.
On Wed, Oct 20, 2010 at 4:03 PM, Johan Edstrom <se...@gmail.com> wrote:
> Thanks,
>
> Will hunt down the core bits too.
>
> As for the message Mina does this by default, the UDP codec basically says when a UDP packet is
> "complete" I'm done and in is flipped to out, it also conforms with the BSD syslog spec, i.e it is 1024
> bytes long and that is it.
>
> I'll look into the Netty side and see if I can't make a similar packet codec for it.
>
Ah perfect, thats nice with the spec. Can be a bit tricky to assemble
packets together to gather the data and build the message.
I assume Netty is almost the same approach. The lead on Netty was also
lead on Mina.
> /je
> On Oct 20, 2010, at 3:40 AM, Claus Ibsen wrote:
>
>> We usually add placeholders for popular data formats in the DSL.
>>
>> So you can do marshal().sysLog().
>>
>> For example even HL7 has this, so just check a bit in camel-core how
>> it can be done.
>>
>> But a nice idea to use a data format so we can use both MINA and Netty.
>>
>> But how does Mina/Netty know when the stream of data is completed, so
>> it can emit the packet?
>> eg can you use the default codecs provided by those?
>>
>>
>>
>> On Wed, Oct 20, 2010 at 4:34 AM, Johan Edstrom <se...@gmail.com> wrote:
>>> Hey
>>>
>>> Need some design advice.
>>> I have an RFC 3164 (BSD Syslog protocol) ready DataFormat and the necessary converters.
>>> I wanted this to work with both Netty UDP and Mina UDP after Willems comment, and it does not have to
>>> be specific whatsoever to the network library, it is just byte[] parsing really.
>>>
>>> The test route looks like :
>>>
>>> protected RouteBuilder createRouteBuilder() throws Exception {
>>> return new RouteBuilder() {
>>> public void configure() throws Exception {
>>>
>>> context.setTracing(true);
>>> DataFormat syslogDataFormat = new Rfc3164SyslogDataFormat();
>>>
>>> // we setup a Syslog listener on a random port.
>>> from("netty:udp://127.0.0.1:" + serverPort + "?sync=false")
>>>
>>> .unmarshal(syslogDataFormat).process(new Processor() {
>>> public void process(Exchange ex) {
>>> assertTrue(ex.getIn().getBody() instanceof SyslogMessage);
>>> }
>>> }).to("mock:syslogReceiver").
>>> marshal(syslogDataFormat).to("mock:syslogReceiver2");
>>> }
>>> };
>>>
>>> And the @Test like
>>>
>>> @Test
>>> public void testSendingRawUDP() throws IOException, InterruptedException {
>>>
>>> MockEndpoint mock = getMockEndpoint("mock:syslogReceiver");
>>> MockEndpoint mock2 = getMockEndpoint("mock:syslogReceiver2");
>>> mock.expectedMessageCount(1);
>>> mock2.expectedMessageCount(1);
>>> mock2.expectedBodiesReceived(message);
>>> DatagramSocket ds = new DatagramSocket();
>>>
>>> DatagramPacket packet = new DatagramPacket(message.getBytes(), message.getBytes().length,
>>> InetAddress.getByName("localhost"), serverPort);
>>> ds.send(packet);
>>>
>>> mock2.assertIsSatisfied();
>>> mock.assertIsSatisfied();
>>> mock.reset();
>>> mock2.reset();
>>> }
>>>
>>> I think I also would want this to be able to handle things like syslog-ng and the syslog protocol in the new RFC nobody implements correctly.
>>>
>>> Right now I have this.. (this being what is below the rest of the text)
>>>
>>> I.e the question will be - what is the cleanest way of writing a "factory like" dataformat?
>>> I'd suspect I'd get this into a bytebuffer, look for indicators, rewind, pick parser and get a message
>>> and then on the marshal side, is there a clean way I could pick a way that would make it into the DSL?
>>>
>>> Sorry if this sounds retarded.
>>>
>>> /je
>>>
>>>
>>>
>>> /**
>>> * Licensed to the Apache Software Foundation (ASF) under one or more
>>> * contributor license agreements. See the NOTICE file distributed with
>>> * this work for additional information regarding copyright ownership.
>>> * The ASF licenses this file to You under the Apache License, Version 2.0
>>> * (the "License"); you may not use this file except in compliance with
>>> * the License. You may obtain a copy of the License at
>>> *
>>> * http://www.apache.org/licenses/LICENSE-2.0
>>> *
>>> * Unless required by applicable law or agreed to in writing, software
>>> * distributed under the License is distributed on an "AS IS" BASIS,
>>> * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
>>> * See the License for the specific language governing permissions and
>>> * limitations under the License.
>>> */
>>>
>>> package org.apache.camel.component.syslog;
>>>
>>> import java.net.InetAddress;
>>> import java.net.UnknownHostException;
>>> import java.nio.ByteBuffer;
>>> import java.util.Calendar;
>>> import java.util.Date;
>>> import java.util.GregorianCalendar;
>>> import java.util.HashMap;
>>> import java.util.Map;
>>>
>>> import org.apache.camel.Converter;
>>> import org.apache.commons.logging.Log;
>>> import org.apache.commons.logging.LogFactory;
>>>
>>> public class Rfc3164SyslogConverter {
>>>
>>> private static final transient Log LOG = LogFactory.getLog(Rfc3164SyslogConverter.class);
>>>
>>> private static enum MONTHS {
>>> jan, feb, mar, apr, may, jun, jul, aug, sep, oct, nov, dec
>>> }
>>>
>>> private final static Map<String, MONTHS> MONTH_VALUE_MAP = new HashMap<String, MONTHS>() {
>>> {
>>> put("jan", MONTHS.jan);
>>> put("feb", MONTHS.feb);
>>> put("mar", MONTHS.mar);
>>> put("apr", MONTHS.apr);
>>> put("may", MONTHS.may);
>>> put("jun", MONTHS.jun);
>>> put("jul", MONTHS.jul);
>>> put("aug", MONTHS.aug);
>>> put("sep", MONTHS.sep);
>>> put("oct", MONTHS.oct);
>>> put("nov", MONTHS.nov);
>>> put("dec", MONTHS.dec);
>>> }
>>> };
>>>
>>> @Converter
>>> public static String toString(SyslogMessage message) {
>>> StringBuilder sbr = new StringBuilder();
>>> sbr.append("<");
>>> if (message.getFacility() == null) {
>>> message.setFacility(SyslogFacility.USER);
>>> }
>>> if (message.getSeverity() == null) {
>>> message.setSeverity(SyslogSeverity.INFO);
>>> }
>>> if (message.getHostname() == null) {
>>> //This is massively ugly..
>>> try {
>>> message.setHostname(InetAddress.getLocalHost().toString());
>>> } catch (UnknownHostException e) {
>>> message.setHostname("UNKNOWN_HOST");
>>> }
>>> }
>>> sbr.append(message.getFacility().ordinal() * 8 + message.getSeverity().ordinal());
>>> sbr.append(">");
>>> if (message.getTimestamp() == null) {
>>> message.setTimestamp(new Date());
>>> }
>>>
>>> //SDF isn't going to help much here.
>>>
>>> Calendar cal = GregorianCalendar.getInstance();
>>> cal.setTime(message.getTimestamp());
>>>
>>> String firstLetter = MONTHS.values()[cal.get(Calendar.MONTH)].toString().substring(0, 1); // Get first letter
>>> String remainder = MONTHS.values()[cal.get(Calendar.MONTH)].toString()
>>> .substring(1); // Get remainder of word.
>>> String capitalized = firstLetter.toUpperCase() + remainder.toLowerCase();
>>>
>>> sbr.append(capitalized);
>>> sbr.append(" ");
>>>
>>> if (cal.get(Calendar.DAY_OF_MONTH) < 10) {
>>> sbr.append(" ").append(cal.get(Calendar.DAY_OF_MONTH));
>>> } else {
>>> sbr.append(cal.get(Calendar.DAY_OF_MONTH));
>>> }
>>>
>>> sbr.append(" ");
>>>
>>> if (cal.get(Calendar.HOUR_OF_DAY) < 10) {
>>> sbr.append("0").append(cal.get(Calendar.HOUR_OF_DAY));
>>> } else {
>>> sbr.append(cal.get(Calendar.HOUR_OF_DAY));
>>> }
>>> sbr.append(":");
>>>
>>> if (cal.get(Calendar.MINUTE) < 10) {
>>> sbr.append("0").append(cal.get(Calendar.MINUTE));
>>> } else {
>>> sbr.append(cal.get(Calendar.MINUTE));
>>> }
>>> sbr.append(":");
>>>
>>> if (cal.get(Calendar.SECOND) < 10) {
>>> sbr.append("0").append(cal.get(Calendar.SECOND));
>>> } else {
>>> sbr.append(cal.get(Calendar.SECOND));
>>> }
>>> sbr.append(" ");
>>>
>>> sbr.append(message.getHostname());
>>> sbr.append(" ");
>>> sbr.append(message.getLogMessage());
>>> return sbr.toString();
>>> }
>>>
>>> @Converter
>>> public static SyslogMessage toSyslogMessage(String body) {
>>> return parseMessage(body.getBytes());
>>> }
>>>
>>> public final static SyslogMessage parseMessage(byte[] bytes) {
>>> ByteBuffer byteBuffer = ByteBuffer.allocate(bytes.length);
>>> byteBuffer.put(bytes);
>>> byteBuffer.rewind();
>>>
>>> SyslogMessage syslogMessage = new SyslogMessage();
>>> Character charFound = (char) byteBuffer.get();
>>>
>>> while (charFound != '<') {
>>> //Ignore noise in beginning of message.
>>> charFound = (char) byteBuffer.get();
>>> }
>>> char priChar = 0;
>>> if (charFound == '<') {
>>> int facility = 0;
>>>
>>> while (Character.isDigit(priChar = (char) (byteBuffer.get() & 0xff))) {
>>> facility *= 10;
>>> facility += Character.digit(priChar, 10);
>>> }
>>> syslogMessage.setFacility(SyslogFacility.values()[facility >> 3]);
>>> syslogMessage.setSeverity(SyslogSeverity.values()[facility & 0x07]);
>>> }
>>>
>>> if (priChar != '>') {
>>> //Invalid character - this is not a well defined syslog message.
>>> LOG.error("Invalid syslog message, missing a > in the Facility/Priority part");
>>> }
>>>
>>> //Done parsing severity and facility
>>> //<169>Oct 22 10:52:01 TZ-6 scapegoat.dmz.example.org 10.1.2.3 sched[0]: That's All Folks!
>>> //Need to parse the date.
>>>
>>> /**
>>> The TIMESTAMP field is the local time and is in the format of "Mmm dd
>>> hh:mm:ss" (without the quote marks) where:
>>>
>>> Mmm is the English language abbreviation for the month of the
>>> year with the first character in uppercase and the other two
>>> characters in lowercase. The following are the only acceptable
>>> values:
>>>
>>> Jan, Feb, Mar, Apr, May, Jun, Jul, Aug, Sep, Oct, Nov, Dec
>>>
>>> dd is the day of the month. If the day of the month is less
>>> than 10, then it MUST be represented as a space and then the
>>> number. For example, the 7th day of August would be
>>> represented as "Aug 7", with two spaces between the "g" and
>>> the "7".
>>>
>>> hh:mm:ss is the local time. The hour (hh) is represented in a
>>> 24-hour format. Valid entries are between 00 and 23,
>>> inclusive. The minute (mm) and second (ss) entries are between
>>> 00 and 59 inclusive.
>>>
>>>
>>> */
>>>
>>> char[] month = new char[3];
>>> for (int i = 0; i < 3; i++) {
>>> month[i] = (char) (byteBuffer.get() & 0xff);
>>> }
>>> charFound = (char) byteBuffer.get();
>>> if (charFound != ' ') {
>>> //Invalid Message - missing mandatory space.
>>> LOG.error("Invalid syslog message, missing a mandatory space after month");
>>> }
>>> charFound = (char) (byteBuffer.get() & 0xff);
>>>
>>> int day = 0;
>>> if (charFound == ' ') {
>>> //Extra space for the day - this is okay.
>>> //Just ignored per the spec.
>>> } else {
>>> day *= 10;
>>> day += Character.digit(charFound, 10);
>>> }
>>>
>>> while (Character.isDigit(charFound = (char) (byteBuffer.get() & 0xff))) {
>>> day *= 10;
>>> day += Character.digit(charFound, 10);
>>> }
>>>
>>> int hour = 0;
>>> while (Character.isDigit(charFound = (char) (byteBuffer.get() & 0xff))) {
>>> hour *= 10;
>>> hour += Character.digit(charFound, 10);
>>> }
>>>
>>> int minute = 0;
>>> while (Character.isDigit(charFound = (char) (byteBuffer.get() & 0xff))) {
>>> minute *= 10;
>>> minute += Character.digit(charFound, 10);
>>> }
>>>
>>> int second = 0;
>>> while (Character.isDigit(charFound = (char) (byteBuffer.get() & 0xff))) {
>>> second *= 10;
>>> second += Character.digit(charFound, 10);
>>> }
>>>
>>> //The host is the char sequence until the next ' '
>>>
>>> StringBuilder host = new StringBuilder();
>>> while ((charFound = (char) (byteBuffer.get() & 0xff)) != ' ') {
>>> host.append(charFound);
>>> }
>>>
>>> syslogMessage.setHostname(host.toString());
>>>
>>> StringBuilder msg = new StringBuilder();
>>> while (byteBuffer.hasRemaining()) {
>>> charFound = (char) (byteBuffer.get() & 0xff);
>>> msg.append(charFound);
>>> }
>>>
>>> Calendar calendar = new GregorianCalendar();
>>> calendar.set(Calendar.MONTH, MONTH_VALUE_MAP.get((String.valueOf(month).toLowerCase())).ordinal());
>>> calendar.set(Calendar.DAY_OF_MONTH, day);
>>> calendar.set(Calendar.HOUR_OF_DAY, hour);
>>> calendar.set(Calendar.MINUTE, minute);
>>> calendar.set(Calendar.SECOND, second);
>>>
>>> syslogMessage.setTimestamp(calendar.getTime());
>>>
>>> syslogMessage.setLogMessage(msg.toString());
>>> if (LOG.isTraceEnabled()) {
>>> LOG.trace("Syslog message : " + syslogMessage.toString());
>>> }
>>>
>>> return syslogMessage;
>>> }
>>> }
>>>
>>>
>>>
>>> Johan Edstrom
>>>
>>>
>>>
>>>
>>>
>>>
>>>
>>
>>
>>
>> --
>> Claus Ibsen
>> Apache Camel Committer
>>
>> Author of Camel in Action: http://www.manning.com/ibsen/
>> Open Source Integration: http://fusesource.com
>> Blog: http://davsclaus.blogspot.com/
>> Twitter: http://twitter.com/davsclaus
>
> Johan Edstrom
>
> joed@opennms.org
>
> They that can give up essential liberty to purchase a little temporary safety, deserve neither liberty nor safety.
>
> Benjamin Franklin, Historical Review of Pennsylvania, 1759
>
>
>
>
>
>
--
Claus Ibsen
Apache Camel Committer
Author of Camel in Action: http://www.manning.com/ibsen/
Open Source Integration: http://fusesource.com
Blog: http://davsclaus.blogspot.com/
Twitter: http://twitter.com/davsclaus
Re: Design advice. - Syslog DataFormat.
Posted by Claus Ibsen <cl...@gmail.com>.
On Wed, Oct 20, 2010 at 4:03 PM, Johan Edstrom <se...@gmail.com> wrote:
> Thanks,
>
> Will hunt down the core bits too.
>
Oh btw we can have options in the DSL as well. In Java DSL you may
just want it as parameters
marshal().sysLog("foo", 123)
And in Spring XML we can have it something like:
<marshal>
<sysLog param1="foo" param2="123"/>
</marshal>
And we should in the future maybe support uri parameters on data
format just like endpoint :)
> As for the message Mina does this by default, the UDP codec basically says when a UDP packet is
> "complete" I'm done and in is flipped to out, it also conforms with the BSD syslog spec, i.e it is 1024
> bytes long and that is it.
>
> I'll look into the Netty side and see if I can't make a similar packet codec for it.
>
> /je
> On Oct 20, 2010, at 3:40 AM, Claus Ibsen wrote:
>
>> We usually add placeholders for popular data formats in the DSL.
>>
>> So you can do marshal().sysLog().
>>
>> For example even HL7 has this, so just check a bit in camel-core how
>> it can be done.
>>
>> But a nice idea to use a data format so we can use both MINA and Netty.
>>
>> But how does Mina/Netty know when the stream of data is completed, so
>> it can emit the packet?
>> eg can you use the default codecs provided by those?
>>
>>
>>
>> On Wed, Oct 20, 2010 at 4:34 AM, Johan Edstrom <se...@gmail.com> wrote:
>>> Hey
>>>
>>> Need some design advice.
>>> I have an RFC 3164 (BSD Syslog protocol) ready DataFormat and the necessary converters.
>>> I wanted this to work with both Netty UDP and Mina UDP after Willems comment, and it does not have to
>>> be specific whatsoever to the network library, it is just byte[] parsing really.
>>>
>>> The test route looks like :
>>>
>>> protected RouteBuilder createRouteBuilder() throws Exception {
>>> return new RouteBuilder() {
>>> public void configure() throws Exception {
>>>
>>> context.setTracing(true);
>>> DataFormat syslogDataFormat = new Rfc3164SyslogDataFormat();
>>>
>>> // we setup a Syslog listener on a random port.
>>> from("netty:udp://127.0.0.1:" + serverPort + "?sync=false")
>>>
>>> .unmarshal(syslogDataFormat).process(new Processor() {
>>> public void process(Exchange ex) {
>>> assertTrue(ex.getIn().getBody() instanceof SyslogMessage);
>>> }
>>> }).to("mock:syslogReceiver").
>>> marshal(syslogDataFormat).to("mock:syslogReceiver2");
>>> }
>>> };
>>>
>>> And the @Test like
>>>
>>> @Test
>>> public void testSendingRawUDP() throws IOException, InterruptedException {
>>>
>>> MockEndpoint mock = getMockEndpoint("mock:syslogReceiver");
>>> MockEndpoint mock2 = getMockEndpoint("mock:syslogReceiver2");
>>> mock.expectedMessageCount(1);
>>> mock2.expectedMessageCount(1);
>>> mock2.expectedBodiesReceived(message);
>>> DatagramSocket ds = new DatagramSocket();
>>>
>>> DatagramPacket packet = new DatagramPacket(message.getBytes(), message.getBytes().length,
>>> InetAddress.getByName("localhost"), serverPort);
>>> ds.send(packet);
>>>
>>> mock2.assertIsSatisfied();
>>> mock.assertIsSatisfied();
>>> mock.reset();
>>> mock2.reset();
>>> }
>>>
>>> I think I also would want this to be able to handle things like syslog-ng and the syslog protocol in the new RFC nobody implements correctly.
>>>
>>> Right now I have this.. (this being what is below the rest of the text)
>>>
>>> I.e the question will be - what is the cleanest way of writing a "factory like" dataformat?
>>> I'd suspect I'd get this into a bytebuffer, look for indicators, rewind, pick parser and get a message
>>> and then on the marshal side, is there a clean way I could pick a way that would make it into the DSL?
>>>
>>> Sorry if this sounds retarded.
>>>
>>> /je
>>>
>>>
>>>
>>> /**
>>> * Licensed to the Apache Software Foundation (ASF) under one or more
>>> * contributor license agreements. See the NOTICE file distributed with
>>> * this work for additional information regarding copyright ownership.
>>> * The ASF licenses this file to You under the Apache License, Version 2.0
>>> * (the "License"); you may not use this file except in compliance with
>>> * the License. You may obtain a copy of the License at
>>> *
>>> * http://www.apache.org/licenses/LICENSE-2.0
>>> *
>>> * Unless required by applicable law or agreed to in writing, software
>>> * distributed under the License is distributed on an "AS IS" BASIS,
>>> * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
>>> * See the License for the specific language governing permissions and
>>> * limitations under the License.
>>> */
>>>
>>> package org.apache.camel.component.syslog;
>>>
>>> import java.net.InetAddress;
>>> import java.net.UnknownHostException;
>>> import java.nio.ByteBuffer;
>>> import java.util.Calendar;
>>> import java.util.Date;
>>> import java.util.GregorianCalendar;
>>> import java.util.HashMap;
>>> import java.util.Map;
>>>
>>> import org.apache.camel.Converter;
>>> import org.apache.commons.logging.Log;
>>> import org.apache.commons.logging.LogFactory;
>>>
>>> public class Rfc3164SyslogConverter {
>>>
>>> private static final transient Log LOG = LogFactory.getLog(Rfc3164SyslogConverter.class);
>>>
>>> private static enum MONTHS {
>>> jan, feb, mar, apr, may, jun, jul, aug, sep, oct, nov, dec
>>> }
>>>
>>> private final static Map<String, MONTHS> MONTH_VALUE_MAP = new HashMap<String, MONTHS>() {
>>> {
>>> put("jan", MONTHS.jan);
>>> put("feb", MONTHS.feb);
>>> put("mar", MONTHS.mar);
>>> put("apr", MONTHS.apr);
>>> put("may", MONTHS.may);
>>> put("jun", MONTHS.jun);
>>> put("jul", MONTHS.jul);
>>> put("aug", MONTHS.aug);
>>> put("sep", MONTHS.sep);
>>> put("oct", MONTHS.oct);
>>> put("nov", MONTHS.nov);
>>> put("dec", MONTHS.dec);
>>> }
>>> };
>>>
>>> @Converter
>>> public static String toString(SyslogMessage message) {
>>> StringBuilder sbr = new StringBuilder();
>>> sbr.append("<");
>>> if (message.getFacility() == null) {
>>> message.setFacility(SyslogFacility.USER);
>>> }
>>> if (message.getSeverity() == null) {
>>> message.setSeverity(SyslogSeverity.INFO);
>>> }
>>> if (message.getHostname() == null) {
>>> //This is massively ugly..
>>> try {
>>> message.setHostname(InetAddress.getLocalHost().toString());
>>> } catch (UnknownHostException e) {
>>> message.setHostname("UNKNOWN_HOST");
>>> }
>>> }
>>> sbr.append(message.getFacility().ordinal() * 8 + message.getSeverity().ordinal());
>>> sbr.append(">");
>>> if (message.getTimestamp() == null) {
>>> message.setTimestamp(new Date());
>>> }
>>>
>>> //SDF isn't going to help much here.
>>>
>>> Calendar cal = GregorianCalendar.getInstance();
>>> cal.setTime(message.getTimestamp());
>>>
>>> String firstLetter = MONTHS.values()[cal.get(Calendar.MONTH)].toString().substring(0, 1); // Get first letter
>>> String remainder = MONTHS.values()[cal.get(Calendar.MONTH)].toString()
>>> .substring(1); // Get remainder of word.
>>> String capitalized = firstLetter.toUpperCase() + remainder.toLowerCase();
>>>
>>> sbr.append(capitalized);
>>> sbr.append(" ");
>>>
>>> if (cal.get(Calendar.DAY_OF_MONTH) < 10) {
>>> sbr.append(" ").append(cal.get(Calendar.DAY_OF_MONTH));
>>> } else {
>>> sbr.append(cal.get(Calendar.DAY_OF_MONTH));
>>> }
>>>
>>> sbr.append(" ");
>>>
>>> if (cal.get(Calendar.HOUR_OF_DAY) < 10) {
>>> sbr.append("0").append(cal.get(Calendar.HOUR_OF_DAY));
>>> } else {
>>> sbr.append(cal.get(Calendar.HOUR_OF_DAY));
>>> }
>>> sbr.append(":");
>>>
>>> if (cal.get(Calendar.MINUTE) < 10) {
>>> sbr.append("0").append(cal.get(Calendar.MINUTE));
>>> } else {
>>> sbr.append(cal.get(Calendar.MINUTE));
>>> }
>>> sbr.append(":");
>>>
>>> if (cal.get(Calendar.SECOND) < 10) {
>>> sbr.append("0").append(cal.get(Calendar.SECOND));
>>> } else {
>>> sbr.append(cal.get(Calendar.SECOND));
>>> }
>>> sbr.append(" ");
>>>
>>> sbr.append(message.getHostname());
>>> sbr.append(" ");
>>> sbr.append(message.getLogMessage());
>>> return sbr.toString();
>>> }
>>>
>>> @Converter
>>> public static SyslogMessage toSyslogMessage(String body) {
>>> return parseMessage(body.getBytes());
>>> }
>>>
>>> public final static SyslogMessage parseMessage(byte[] bytes) {
>>> ByteBuffer byteBuffer = ByteBuffer.allocate(bytes.length);
>>> byteBuffer.put(bytes);
>>> byteBuffer.rewind();
>>>
>>> SyslogMessage syslogMessage = new SyslogMessage();
>>> Character charFound = (char) byteBuffer.get();
>>>
>>> while (charFound != '<') {
>>> //Ignore noise in beginning of message.
>>> charFound = (char) byteBuffer.get();
>>> }
>>> char priChar = 0;
>>> if (charFound == '<') {
>>> int facility = 0;
>>>
>>> while (Character.isDigit(priChar = (char) (byteBuffer.get() & 0xff))) {
>>> facility *= 10;
>>> facility += Character.digit(priChar, 10);
>>> }
>>> syslogMessage.setFacility(SyslogFacility.values()[facility >> 3]);
>>> syslogMessage.setSeverity(SyslogSeverity.values()[facility & 0x07]);
>>> }
>>>
>>> if (priChar != '>') {
>>> //Invalid character - this is not a well defined syslog message.
>>> LOG.error("Invalid syslog message, missing a > in the Facility/Priority part");
>>> }
>>>
>>> //Done parsing severity and facility
>>> //<169>Oct 22 10:52:01 TZ-6 scapegoat.dmz.example.org 10.1.2.3 sched[0]: That's All Folks!
>>> //Need to parse the date.
>>>
>>> /**
>>> The TIMESTAMP field is the local time and is in the format of "Mmm dd
>>> hh:mm:ss" (without the quote marks) where:
>>>
>>> Mmm is the English language abbreviation for the month of the
>>> year with the first character in uppercase and the other two
>>> characters in lowercase. The following are the only acceptable
>>> values:
>>>
>>> Jan, Feb, Mar, Apr, May, Jun, Jul, Aug, Sep, Oct, Nov, Dec
>>>
>>> dd is the day of the month. If the day of the month is less
>>> than 10, then it MUST be represented as a space and then the
>>> number. For example, the 7th day of August would be
>>> represented as "Aug 7", with two spaces between the "g" and
>>> the "7".
>>>
>>> hh:mm:ss is the local time. The hour (hh) is represented in a
>>> 24-hour format. Valid entries are between 00 and 23,
>>> inclusive. The minute (mm) and second (ss) entries are between
>>> 00 and 59 inclusive.
>>>
>>>
>>> */
>>>
>>> char[] month = new char[3];
>>> for (int i = 0; i < 3; i++) {
>>> month[i] = (char) (byteBuffer.get() & 0xff);
>>> }
>>> charFound = (char) byteBuffer.get();
>>> if (charFound != ' ') {
>>> //Invalid Message - missing mandatory space.
>>> LOG.error("Invalid syslog message, missing a mandatory space after month");
>>> }
>>> charFound = (char) (byteBuffer.get() & 0xff);
>>>
>>> int day = 0;
>>> if (charFound == ' ') {
>>> //Extra space for the day - this is okay.
>>> //Just ignored per the spec.
>>> } else {
>>> day *= 10;
>>> day += Character.digit(charFound, 10);
>>> }
>>>
>>> while (Character.isDigit(charFound = (char) (byteBuffer.get() & 0xff))) {
>>> day *= 10;
>>> day += Character.digit(charFound, 10);
>>> }
>>>
>>> int hour = 0;
>>> while (Character.isDigit(charFound = (char) (byteBuffer.get() & 0xff))) {
>>> hour *= 10;
>>> hour += Character.digit(charFound, 10);
>>> }
>>>
>>> int minute = 0;
>>> while (Character.isDigit(charFound = (char) (byteBuffer.get() & 0xff))) {
>>> minute *= 10;
>>> minute += Character.digit(charFound, 10);
>>> }
>>>
>>> int second = 0;
>>> while (Character.isDigit(charFound = (char) (byteBuffer.get() & 0xff))) {
>>> second *= 10;
>>> second += Character.digit(charFound, 10);
>>> }
>>>
>>> //The host is the char sequence until the next ' '
>>>
>>> StringBuilder host = new StringBuilder();
>>> while ((charFound = (char) (byteBuffer.get() & 0xff)) != ' ') {
>>> host.append(charFound);
>>> }
>>>
>>> syslogMessage.setHostname(host.toString());
>>>
>>> StringBuilder msg = new StringBuilder();
>>> while (byteBuffer.hasRemaining()) {
>>> charFound = (char) (byteBuffer.get() & 0xff);
>>> msg.append(charFound);
>>> }
>>>
>>> Calendar calendar = new GregorianCalendar();
>>> calendar.set(Calendar.MONTH, MONTH_VALUE_MAP.get((String.valueOf(month).toLowerCase())).ordinal());
>>> calendar.set(Calendar.DAY_OF_MONTH, day);
>>> calendar.set(Calendar.HOUR_OF_DAY, hour);
>>> calendar.set(Calendar.MINUTE, minute);
>>> calendar.set(Calendar.SECOND, second);
>>>
>>> syslogMessage.setTimestamp(calendar.getTime());
>>>
>>> syslogMessage.setLogMessage(msg.toString());
>>> if (LOG.isTraceEnabled()) {
>>> LOG.trace("Syslog message : " + syslogMessage.toString());
>>> }
>>>
>>> return syslogMessage;
>>> }
>>> }
>>>
>>>
>>>
>>> Johan Edstrom
>>>
>>>
>>>
>>>
>>>
>>>
>>>
>>
>>
>>
>> --
>> Claus Ibsen
>> Apache Camel Committer
>>
>> Author of Camel in Action: http://www.manning.com/ibsen/
>> Open Source Integration: http://fusesource.com
>> Blog: http://davsclaus.blogspot.com/
>> Twitter: http://twitter.com/davsclaus
>
> Johan Edstrom
>
> joed@opennms.org
>
> They that can give up essential liberty to purchase a little temporary safety, deserve neither liberty nor safety.
>
> Benjamin Franklin, Historical Review of Pennsylvania, 1759
>
>
>
>
>
>
--
Claus Ibsen
Apache Camel Committer
Author of Camel in Action: http://www.manning.com/ibsen/
Open Source Integration: http://fusesource.com
Blog: http://davsclaus.blogspot.com/
Twitter: http://twitter.com/davsclaus
Re: Design advice. - Syslog DataFormat.
Posted by Johan Edstrom <se...@gmail.com>.
Thanks,
Will hunt down the core bits too.
As for the message Mina does this by default, the UDP codec basically says when a UDP packet is
"complete" I'm done and in is flipped to out, it also conforms with the BSD syslog spec, i.e it is 1024
bytes long and that is it.
I'll look into the Netty side and see if I can't make a similar packet codec for it.
/je
On Oct 20, 2010, at 3:40 AM, Claus Ibsen wrote:
> We usually add placeholders for popular data formats in the DSL.
>
> So you can do marshal().sysLog().
>
> For example even HL7 has this, so just check a bit in camel-core how
> it can be done.
>
> But a nice idea to use a data format so we can use both MINA and Netty.
>
> But how does Mina/Netty know when the stream of data is completed, so
> it can emit the packet?
> eg can you use the default codecs provided by those?
>
>
>
> On Wed, Oct 20, 2010 at 4:34 AM, Johan Edstrom <se...@gmail.com> wrote:
>> Hey
>>
>> Need some design advice.
>> I have an RFC 3164 (BSD Syslog protocol) ready DataFormat and the necessary converters.
>> I wanted this to work with both Netty UDP and Mina UDP after Willems comment, and it does not have to
>> be specific whatsoever to the network library, it is just byte[] parsing really.
>>
>> The test route looks like :
>>
>> protected RouteBuilder createRouteBuilder() throws Exception {
>> return new RouteBuilder() {
>> public void configure() throws Exception {
>>
>> context.setTracing(true);
>> DataFormat syslogDataFormat = new Rfc3164SyslogDataFormat();
>>
>> // we setup a Syslog listener on a random port.
>> from("netty:udp://127.0.0.1:" + serverPort + "?sync=false")
>>
>> .unmarshal(syslogDataFormat).process(new Processor() {
>> public void process(Exchange ex) {
>> assertTrue(ex.getIn().getBody() instanceof SyslogMessage);
>> }
>> }).to("mock:syslogReceiver").
>> marshal(syslogDataFormat).to("mock:syslogReceiver2");
>> }
>> };
>>
>> And the @Test like
>>
>> @Test
>> public void testSendingRawUDP() throws IOException, InterruptedException {
>>
>> MockEndpoint mock = getMockEndpoint("mock:syslogReceiver");
>> MockEndpoint mock2 = getMockEndpoint("mock:syslogReceiver2");
>> mock.expectedMessageCount(1);
>> mock2.expectedMessageCount(1);
>> mock2.expectedBodiesReceived(message);
>> DatagramSocket ds = new DatagramSocket();
>>
>> DatagramPacket packet = new DatagramPacket(message.getBytes(), message.getBytes().length,
>> InetAddress.getByName("localhost"), serverPort);
>> ds.send(packet);
>>
>> mock2.assertIsSatisfied();
>> mock.assertIsSatisfied();
>> mock.reset();
>> mock2.reset();
>> }
>>
>> I think I also would want this to be able to handle things like syslog-ng and the syslog protocol in the new RFC nobody implements correctly.
>>
>> Right now I have this.. (this being what is below the rest of the text)
>>
>> I.e the question will be - what is the cleanest way of writing a "factory like" dataformat?
>> I'd suspect I'd get this into a bytebuffer, look for indicators, rewind, pick parser and get a message
>> and then on the marshal side, is there a clean way I could pick a way that would make it into the DSL?
>>
>> Sorry if this sounds retarded.
>>
>> /je
>>
>>
>>
>> /**
>> * Licensed to the Apache Software Foundation (ASF) under one or more
>> * contributor license agreements. See the NOTICE file distributed with
>> * this work for additional information regarding copyright ownership.
>> * The ASF licenses this file to You under the Apache License, Version 2.0
>> * (the "License"); you may not use this file except in compliance with
>> * the License. You may obtain a copy of the License at
>> *
>> * http://www.apache.org/licenses/LICENSE-2.0
>> *
>> * Unless required by applicable law or agreed to in writing, software
>> * distributed under the License is distributed on an "AS IS" BASIS,
>> * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
>> * See the License for the specific language governing permissions and
>> * limitations under the License.
>> */
>>
>> package org.apache.camel.component.syslog;
>>
>> import java.net.InetAddress;
>> import java.net.UnknownHostException;
>> import java.nio.ByteBuffer;
>> import java.util.Calendar;
>> import java.util.Date;
>> import java.util.GregorianCalendar;
>> import java.util.HashMap;
>> import java.util.Map;
>>
>> import org.apache.camel.Converter;
>> import org.apache.commons.logging.Log;
>> import org.apache.commons.logging.LogFactory;
>>
>> public class Rfc3164SyslogConverter {
>>
>> private static final transient Log LOG = LogFactory.getLog(Rfc3164SyslogConverter.class);
>>
>> private static enum MONTHS {
>> jan, feb, mar, apr, may, jun, jul, aug, sep, oct, nov, dec
>> }
>>
>> private final static Map<String, MONTHS> MONTH_VALUE_MAP = new HashMap<String, MONTHS>() {
>> {
>> put("jan", MONTHS.jan);
>> put("feb", MONTHS.feb);
>> put("mar", MONTHS.mar);
>> put("apr", MONTHS.apr);
>> put("may", MONTHS.may);
>> put("jun", MONTHS.jun);
>> put("jul", MONTHS.jul);
>> put("aug", MONTHS.aug);
>> put("sep", MONTHS.sep);
>> put("oct", MONTHS.oct);
>> put("nov", MONTHS.nov);
>> put("dec", MONTHS.dec);
>> }
>> };
>>
>> @Converter
>> public static String toString(SyslogMessage message) {
>> StringBuilder sbr = new StringBuilder();
>> sbr.append("<");
>> if (message.getFacility() == null) {
>> message.setFacility(SyslogFacility.USER);
>> }
>> if (message.getSeverity() == null) {
>> message.setSeverity(SyslogSeverity.INFO);
>> }
>> if (message.getHostname() == null) {
>> //This is massively ugly..
>> try {
>> message.setHostname(InetAddress.getLocalHost().toString());
>> } catch (UnknownHostException e) {
>> message.setHostname("UNKNOWN_HOST");
>> }
>> }
>> sbr.append(message.getFacility().ordinal() * 8 + message.getSeverity().ordinal());
>> sbr.append(">");
>> if (message.getTimestamp() == null) {
>> message.setTimestamp(new Date());
>> }
>>
>> //SDF isn't going to help much here.
>>
>> Calendar cal = GregorianCalendar.getInstance();
>> cal.setTime(message.getTimestamp());
>>
>> String firstLetter = MONTHS.values()[cal.get(Calendar.MONTH)].toString().substring(0, 1); // Get first letter
>> String remainder = MONTHS.values()[cal.get(Calendar.MONTH)].toString()
>> .substring(1); // Get remainder of word.
>> String capitalized = firstLetter.toUpperCase() + remainder.toLowerCase();
>>
>> sbr.append(capitalized);
>> sbr.append(" ");
>>
>> if (cal.get(Calendar.DAY_OF_MONTH) < 10) {
>> sbr.append(" ").append(cal.get(Calendar.DAY_OF_MONTH));
>> } else {
>> sbr.append(cal.get(Calendar.DAY_OF_MONTH));
>> }
>>
>> sbr.append(" ");
>>
>> if (cal.get(Calendar.HOUR_OF_DAY) < 10) {
>> sbr.append("0").append(cal.get(Calendar.HOUR_OF_DAY));
>> } else {
>> sbr.append(cal.get(Calendar.HOUR_OF_DAY));
>> }
>> sbr.append(":");
>>
>> if (cal.get(Calendar.MINUTE) < 10) {
>> sbr.append("0").append(cal.get(Calendar.MINUTE));
>> } else {
>> sbr.append(cal.get(Calendar.MINUTE));
>> }
>> sbr.append(":");
>>
>> if (cal.get(Calendar.SECOND) < 10) {
>> sbr.append("0").append(cal.get(Calendar.SECOND));
>> } else {
>> sbr.append(cal.get(Calendar.SECOND));
>> }
>> sbr.append(" ");
>>
>> sbr.append(message.getHostname());
>> sbr.append(" ");
>> sbr.append(message.getLogMessage());
>> return sbr.toString();
>> }
>>
>> @Converter
>> public static SyslogMessage toSyslogMessage(String body) {
>> return parseMessage(body.getBytes());
>> }
>>
>> public final static SyslogMessage parseMessage(byte[] bytes) {
>> ByteBuffer byteBuffer = ByteBuffer.allocate(bytes.length);
>> byteBuffer.put(bytes);
>> byteBuffer.rewind();
>>
>> SyslogMessage syslogMessage = new SyslogMessage();
>> Character charFound = (char) byteBuffer.get();
>>
>> while (charFound != '<') {
>> //Ignore noise in beginning of message.
>> charFound = (char) byteBuffer.get();
>> }
>> char priChar = 0;
>> if (charFound == '<') {
>> int facility = 0;
>>
>> while (Character.isDigit(priChar = (char) (byteBuffer.get() & 0xff))) {
>> facility *= 10;
>> facility += Character.digit(priChar, 10);
>> }
>> syslogMessage.setFacility(SyslogFacility.values()[facility >> 3]);
>> syslogMessage.setSeverity(SyslogSeverity.values()[facility & 0x07]);
>> }
>>
>> if (priChar != '>') {
>> //Invalid character - this is not a well defined syslog message.
>> LOG.error("Invalid syslog message, missing a > in the Facility/Priority part");
>> }
>>
>> //Done parsing severity and facility
>> //<169>Oct 22 10:52:01 TZ-6 scapegoat.dmz.example.org 10.1.2.3 sched[0]: That's All Folks!
>> //Need to parse the date.
>>
>> /**
>> The TIMESTAMP field is the local time and is in the format of "Mmm dd
>> hh:mm:ss" (without the quote marks) where:
>>
>> Mmm is the English language abbreviation for the month of the
>> year with the first character in uppercase and the other two
>> characters in lowercase. The following are the only acceptable
>> values:
>>
>> Jan, Feb, Mar, Apr, May, Jun, Jul, Aug, Sep, Oct, Nov, Dec
>>
>> dd is the day of the month. If the day of the month is less
>> than 10, then it MUST be represented as a space and then the
>> number. For example, the 7th day of August would be
>> represented as "Aug 7", with two spaces between the "g" and
>> the "7".
>>
>> hh:mm:ss is the local time. The hour (hh) is represented in a
>> 24-hour format. Valid entries are between 00 and 23,
>> inclusive. The minute (mm) and second (ss) entries are between
>> 00 and 59 inclusive.
>>
>>
>> */
>>
>> char[] month = new char[3];
>> for (int i = 0; i < 3; i++) {
>> month[i] = (char) (byteBuffer.get() & 0xff);
>> }
>> charFound = (char) byteBuffer.get();
>> if (charFound != ' ') {
>> //Invalid Message - missing mandatory space.
>> LOG.error("Invalid syslog message, missing a mandatory space after month");
>> }
>> charFound = (char) (byteBuffer.get() & 0xff);
>>
>> int day = 0;
>> if (charFound == ' ') {
>> //Extra space for the day - this is okay.
>> //Just ignored per the spec.
>> } else {
>> day *= 10;
>> day += Character.digit(charFound, 10);
>> }
>>
>> while (Character.isDigit(charFound = (char) (byteBuffer.get() & 0xff))) {
>> day *= 10;
>> day += Character.digit(charFound, 10);
>> }
>>
>> int hour = 0;
>> while (Character.isDigit(charFound = (char) (byteBuffer.get() & 0xff))) {
>> hour *= 10;
>> hour += Character.digit(charFound, 10);
>> }
>>
>> int minute = 0;
>> while (Character.isDigit(charFound = (char) (byteBuffer.get() & 0xff))) {
>> minute *= 10;
>> minute += Character.digit(charFound, 10);
>> }
>>
>> int second = 0;
>> while (Character.isDigit(charFound = (char) (byteBuffer.get() & 0xff))) {
>> second *= 10;
>> second += Character.digit(charFound, 10);
>> }
>>
>> //The host is the char sequence until the next ' '
>>
>> StringBuilder host = new StringBuilder();
>> while ((charFound = (char) (byteBuffer.get() & 0xff)) != ' ') {
>> host.append(charFound);
>> }
>>
>> syslogMessage.setHostname(host.toString());
>>
>> StringBuilder msg = new StringBuilder();
>> while (byteBuffer.hasRemaining()) {
>> charFound = (char) (byteBuffer.get() & 0xff);
>> msg.append(charFound);
>> }
>>
>> Calendar calendar = new GregorianCalendar();
>> calendar.set(Calendar.MONTH, MONTH_VALUE_MAP.get((String.valueOf(month).toLowerCase())).ordinal());
>> calendar.set(Calendar.DAY_OF_MONTH, day);
>> calendar.set(Calendar.HOUR_OF_DAY, hour);
>> calendar.set(Calendar.MINUTE, minute);
>> calendar.set(Calendar.SECOND, second);
>>
>> syslogMessage.setTimestamp(calendar.getTime());
>>
>> syslogMessage.setLogMessage(msg.toString());
>> if (LOG.isTraceEnabled()) {
>> LOG.trace("Syslog message : " + syslogMessage.toString());
>> }
>>
>> return syslogMessage;
>> }
>> }
>>
>>
>>
>> Johan Edstrom
>>
>>
>>
>>
>>
>>
>>
>
>
>
> --
> Claus Ibsen
> Apache Camel Committer
>
> Author of Camel in Action: http://www.manning.com/ibsen/
> Open Source Integration: http://fusesource.com
> Blog: http://davsclaus.blogspot.com/
> Twitter: http://twitter.com/davsclaus
Johan Edstrom
joed@opennms.org
They that can give up essential liberty to purchase a little temporary safety, deserve neither liberty nor safety.
Benjamin Franklin, Historical Review of Pennsylvania, 1759
Re: Design advice. - Syslog DataFormat.
Posted by Claus Ibsen <cl...@gmail.com>.
We usually add placeholders for popular data formats in the DSL.
So you can do marshal().sysLog().
For example even HL7 has this, so just check a bit in camel-core how
it can be done.
But a nice idea to use a data format so we can use both MINA and Netty.
But how does Mina/Netty know when the stream of data is completed, so
it can emit the packet?
eg can you use the default codecs provided by those?
On Wed, Oct 20, 2010 at 4:34 AM, Johan Edstrom <se...@gmail.com> wrote:
> Hey
>
> Need some design advice.
> I have an RFC 3164 (BSD Syslog protocol) ready DataFormat and the necessary converters.
> I wanted this to work with both Netty UDP and Mina UDP after Willems comment, and it does not have to
> be specific whatsoever to the network library, it is just byte[] parsing really.
>
> The test route looks like :
>
> protected RouteBuilder createRouteBuilder() throws Exception {
> return new RouteBuilder() {
> public void configure() throws Exception {
>
> context.setTracing(true);
> DataFormat syslogDataFormat = new Rfc3164SyslogDataFormat();
>
> // we setup a Syslog listener on a random port.
> from("netty:udp://127.0.0.1:" + serverPort + "?sync=false")
>
> .unmarshal(syslogDataFormat).process(new Processor() {
> public void process(Exchange ex) {
> assertTrue(ex.getIn().getBody() instanceof SyslogMessage);
> }
> }).to("mock:syslogReceiver").
> marshal(syslogDataFormat).to("mock:syslogReceiver2");
> }
> };
>
> And the @Test like
>
> @Test
> public void testSendingRawUDP() throws IOException, InterruptedException {
>
> MockEndpoint mock = getMockEndpoint("mock:syslogReceiver");
> MockEndpoint mock2 = getMockEndpoint("mock:syslogReceiver2");
> mock.expectedMessageCount(1);
> mock2.expectedMessageCount(1);
> mock2.expectedBodiesReceived(message);
> DatagramSocket ds = new DatagramSocket();
>
> DatagramPacket packet = new DatagramPacket(message.getBytes(), message.getBytes().length,
> InetAddress.getByName("localhost"), serverPort);
> ds.send(packet);
>
> mock2.assertIsSatisfied();
> mock.assertIsSatisfied();
> mock.reset();
> mock2.reset();
> }
>
> I think I also would want this to be able to handle things like syslog-ng and the syslog protocol in the new RFC nobody implements correctly.
>
> Right now I have this.. (this being what is below the rest of the text)
>
> I.e the question will be - what is the cleanest way of writing a "factory like" dataformat?
> I'd suspect I'd get this into a bytebuffer, look for indicators, rewind, pick parser and get a message
> and then on the marshal side, is there a clean way I could pick a way that would make it into the DSL?
>
> Sorry if this sounds retarded.
>
> /je
>
>
>
> /**
> * Licensed to the Apache Software Foundation (ASF) under one or more
> * contributor license agreements. See the NOTICE file distributed with
> * this work for additional information regarding copyright ownership.
> * The ASF licenses this file to You under the Apache License, Version 2.0
> * (the "License"); you may not use this file except in compliance with
> * the License. You may obtain a copy of the License at
> *
> * http://www.apache.org/licenses/LICENSE-2.0
> *
> * Unless required by applicable law or agreed to in writing, software
> * distributed under the License is distributed on an "AS IS" BASIS,
> * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> * See the License for the specific language governing permissions and
> * limitations under the License.
> */
>
> package org.apache.camel.component.syslog;
>
> import java.net.InetAddress;
> import java.net.UnknownHostException;
> import java.nio.ByteBuffer;
> import java.util.Calendar;
> import java.util.Date;
> import java.util.GregorianCalendar;
> import java.util.HashMap;
> import java.util.Map;
>
> import org.apache.camel.Converter;
> import org.apache.commons.logging.Log;
> import org.apache.commons.logging.LogFactory;
>
> public class Rfc3164SyslogConverter {
>
> private static final transient Log LOG = LogFactory.getLog(Rfc3164SyslogConverter.class);
>
> private static enum MONTHS {
> jan, feb, mar, apr, may, jun, jul, aug, sep, oct, nov, dec
> }
>
> private final static Map<String, MONTHS> MONTH_VALUE_MAP = new HashMap<String, MONTHS>() {
> {
> put("jan", MONTHS.jan);
> put("feb", MONTHS.feb);
> put("mar", MONTHS.mar);
> put("apr", MONTHS.apr);
> put("may", MONTHS.may);
> put("jun", MONTHS.jun);
> put("jul", MONTHS.jul);
> put("aug", MONTHS.aug);
> put("sep", MONTHS.sep);
> put("oct", MONTHS.oct);
> put("nov", MONTHS.nov);
> put("dec", MONTHS.dec);
> }
> };
>
> @Converter
> public static String toString(SyslogMessage message) {
> StringBuilder sbr = new StringBuilder();
> sbr.append("<");
> if (message.getFacility() == null) {
> message.setFacility(SyslogFacility.USER);
> }
> if (message.getSeverity() == null) {
> message.setSeverity(SyslogSeverity.INFO);
> }
> if (message.getHostname() == null) {
> //This is massively ugly..
> try {
> message.setHostname(InetAddress.getLocalHost().toString());
> } catch (UnknownHostException e) {
> message.setHostname("UNKNOWN_HOST");
> }
> }
> sbr.append(message.getFacility().ordinal() * 8 + message.getSeverity().ordinal());
> sbr.append(">");
> if (message.getTimestamp() == null) {
> message.setTimestamp(new Date());
> }
>
> //SDF isn't going to help much here.
>
> Calendar cal = GregorianCalendar.getInstance();
> cal.setTime(message.getTimestamp());
>
> String firstLetter = MONTHS.values()[cal.get(Calendar.MONTH)].toString().substring(0, 1); // Get first letter
> String remainder = MONTHS.values()[cal.get(Calendar.MONTH)].toString()
> .substring(1); // Get remainder of word.
> String capitalized = firstLetter.toUpperCase() + remainder.toLowerCase();
>
> sbr.append(capitalized);
> sbr.append(" ");
>
> if (cal.get(Calendar.DAY_OF_MONTH) < 10) {
> sbr.append(" ").append(cal.get(Calendar.DAY_OF_MONTH));
> } else {
> sbr.append(cal.get(Calendar.DAY_OF_MONTH));
> }
>
> sbr.append(" ");
>
> if (cal.get(Calendar.HOUR_OF_DAY) < 10) {
> sbr.append("0").append(cal.get(Calendar.HOUR_OF_DAY));
> } else {
> sbr.append(cal.get(Calendar.HOUR_OF_DAY));
> }
> sbr.append(":");
>
> if (cal.get(Calendar.MINUTE) < 10) {
> sbr.append("0").append(cal.get(Calendar.MINUTE));
> } else {
> sbr.append(cal.get(Calendar.MINUTE));
> }
> sbr.append(":");
>
> if (cal.get(Calendar.SECOND) < 10) {
> sbr.append("0").append(cal.get(Calendar.SECOND));
> } else {
> sbr.append(cal.get(Calendar.SECOND));
> }
> sbr.append(" ");
>
> sbr.append(message.getHostname());
> sbr.append(" ");
> sbr.append(message.getLogMessage());
> return sbr.toString();
> }
>
> @Converter
> public static SyslogMessage toSyslogMessage(String body) {
> return parseMessage(body.getBytes());
> }
>
> public final static SyslogMessage parseMessage(byte[] bytes) {
> ByteBuffer byteBuffer = ByteBuffer.allocate(bytes.length);
> byteBuffer.put(bytes);
> byteBuffer.rewind();
>
> SyslogMessage syslogMessage = new SyslogMessage();
> Character charFound = (char) byteBuffer.get();
>
> while (charFound != '<') {
> //Ignore noise in beginning of message.
> charFound = (char) byteBuffer.get();
> }
> char priChar = 0;
> if (charFound == '<') {
> int facility = 0;
>
> while (Character.isDigit(priChar = (char) (byteBuffer.get() & 0xff))) {
> facility *= 10;
> facility += Character.digit(priChar, 10);
> }
> syslogMessage.setFacility(SyslogFacility.values()[facility >> 3]);
> syslogMessage.setSeverity(SyslogSeverity.values()[facility & 0x07]);
> }
>
> if (priChar != '>') {
> //Invalid character - this is not a well defined syslog message.
> LOG.error("Invalid syslog message, missing a > in the Facility/Priority part");
> }
>
> //Done parsing severity and facility
> //<169>Oct 22 10:52:01 TZ-6 scapegoat.dmz.example.org 10.1.2.3 sched[0]: That's All Folks!
> //Need to parse the date.
>
> /**
> The TIMESTAMP field is the local time and is in the format of "Mmm dd
> hh:mm:ss" (without the quote marks) where:
>
> Mmm is the English language abbreviation for the month of the
> year with the first character in uppercase and the other two
> characters in lowercase. The following are the only acceptable
> values:
>
> Jan, Feb, Mar, Apr, May, Jun, Jul, Aug, Sep, Oct, Nov, Dec
>
> dd is the day of the month. If the day of the month is less
> than 10, then it MUST be represented as a space and then the
> number. For example, the 7th day of August would be
> represented as "Aug 7", with two spaces between the "g" and
> the "7".
>
> hh:mm:ss is the local time. The hour (hh) is represented in a
> 24-hour format. Valid entries are between 00 and 23,
> inclusive. The minute (mm) and second (ss) entries are between
> 00 and 59 inclusive.
>
>
> */
>
> char[] month = new char[3];
> for (int i = 0; i < 3; i++) {
> month[i] = (char) (byteBuffer.get() & 0xff);
> }
> charFound = (char) byteBuffer.get();
> if (charFound != ' ') {
> //Invalid Message - missing mandatory space.
> LOG.error("Invalid syslog message, missing a mandatory space after month");
> }
> charFound = (char) (byteBuffer.get() & 0xff);
>
> int day = 0;
> if (charFound == ' ') {
> //Extra space for the day - this is okay.
> //Just ignored per the spec.
> } else {
> day *= 10;
> day += Character.digit(charFound, 10);
> }
>
> while (Character.isDigit(charFound = (char) (byteBuffer.get() & 0xff))) {
> day *= 10;
> day += Character.digit(charFound, 10);
> }
>
> int hour = 0;
> while (Character.isDigit(charFound = (char) (byteBuffer.get() & 0xff))) {
> hour *= 10;
> hour += Character.digit(charFound, 10);
> }
>
> int minute = 0;
> while (Character.isDigit(charFound = (char) (byteBuffer.get() & 0xff))) {
> minute *= 10;
> minute += Character.digit(charFound, 10);
> }
>
> int second = 0;
> while (Character.isDigit(charFound = (char) (byteBuffer.get() & 0xff))) {
> second *= 10;
> second += Character.digit(charFound, 10);
> }
>
> //The host is the char sequence until the next ' '
>
> StringBuilder host = new StringBuilder();
> while ((charFound = (char) (byteBuffer.get() & 0xff)) != ' ') {
> host.append(charFound);
> }
>
> syslogMessage.setHostname(host.toString());
>
> StringBuilder msg = new StringBuilder();
> while (byteBuffer.hasRemaining()) {
> charFound = (char) (byteBuffer.get() & 0xff);
> msg.append(charFound);
> }
>
> Calendar calendar = new GregorianCalendar();
> calendar.set(Calendar.MONTH, MONTH_VALUE_MAP.get((String.valueOf(month).toLowerCase())).ordinal());
> calendar.set(Calendar.DAY_OF_MONTH, day);
> calendar.set(Calendar.HOUR_OF_DAY, hour);
> calendar.set(Calendar.MINUTE, minute);
> calendar.set(Calendar.SECOND, second);
>
> syslogMessage.setTimestamp(calendar.getTime());
>
> syslogMessage.setLogMessage(msg.toString());
> if (LOG.isTraceEnabled()) {
> LOG.trace("Syslog message : " + syslogMessage.toString());
> }
>
> return syslogMessage;
> }
> }
>
>
>
> Johan Edstrom
>
>
>
>
>
>
>
--
Claus Ibsen
Apache Camel Committer
Author of Camel in Action: http://www.manning.com/ibsen/
Open Source Integration: http://fusesource.com
Blog: http://davsclaus.blogspot.com/
Twitter: http://twitter.com/davsclaus