You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@camel.apache.org by cm...@apache.org on 2012/11/10 12:25:22 UTC
svn commit: r1407761 - in
/camel/branches/camel-2.10.x/components/camel-flatpack/src:
main/java/org/apache/camel/component/flatpack/
test/java/org/apache/camel/component/flatpack/
test/resources/org/apache/camel/component/flatpack/
Author: cmueller
Date: Sat Nov 10 11:25:21 2012
New Revision: 1407761
URL: http://svn.apache.org/viewvc?rev=1407761&view=rev
Log:
CAMEL-5785: Add support for handling long/short lines in Fixed Width Files in the Flatpack DataFormat
Thanks to Chris Geer for the patch
Modified:
camel/branches/camel-2.10.x/components/camel-flatpack/src/main/java/org/apache/camel/component/flatpack/FlatpackDataFormat.java
camel/branches/camel-2.10.x/components/camel-flatpack/src/test/java/org/apache/camel/component/flatpack/FixedLengthAllowLongTest.java
camel/branches/camel-2.10.x/components/camel-flatpack/src/test/java/org/apache/camel/component/flatpack/FixedLengthAllowShortAndLongTest.java
camel/branches/camel-2.10.x/components/camel-flatpack/src/test/java/org/apache/camel/component/flatpack/FixedLengthAllowShortTest.java
camel/branches/camel-2.10.x/components/camel-flatpack/src/test/resources/org/apache/camel/component/flatpack/FixedLengthAllowLongTest-context.xml
camel/branches/camel-2.10.x/components/camel-flatpack/src/test/resources/org/apache/camel/component/flatpack/FixedLengthAllowShortAndLongTest-context.xml
camel/branches/camel-2.10.x/components/camel-flatpack/src/test/resources/org/apache/camel/component/flatpack/FixedLengthAllowShortTest-context.xml
Modified: camel/branches/camel-2.10.x/components/camel-flatpack/src/main/java/org/apache/camel/component/flatpack/FlatpackDataFormat.java
URL: http://svn.apache.org/viewvc/camel/branches/camel-2.10.x/components/camel-flatpack/src/main/java/org/apache/camel/component/flatpack/FlatpackDataFormat.java?rev=1407761&r1=1407760&r2=1407761&view=diff
==============================================================================
--- camel/branches/camel-2.10.x/components/camel-flatpack/src/main/java/org/apache/camel/component/flatpack/FlatpackDataFormat.java (original)
+++ camel/branches/camel-2.10.x/components/camel-flatpack/src/main/java/org/apache/camel/component/flatpack/FlatpackDataFormat.java Sat Nov 10 11:25:21 2012
@@ -61,6 +61,8 @@ public class FlatpackDataFormat implemen
private char textQualifier = '"';
private boolean ignoreFirstRecord = true;
private boolean fixed;
+ private boolean allowShortLines;
+ private boolean ignoreExtraColumns;
private String definition;
@SuppressWarnings("unchecked")
@@ -158,6 +160,28 @@ public class FlatpackDataFormat implemen
this.parserFactory = parserFactory;
}
+ public boolean isAllowShortLines() {
+ return this.allowShortLines;
+ }
+
+ /**
+ * Allows for lines to be shorter than expected and ignores the extra characters
+ */
+ public void setAllowShortLines(boolean allowShortLines) {
+ this.allowShortLines = allowShortLines;
+ }
+
+ /**
+ * Allows for lines to be longer than expected and ignores the extra characters
+ */
+ public void setIgnoreExtraColumns(boolean ignoreExtraColumns) {
+ this.ignoreExtraColumns = ignoreExtraColumns;
+ }
+
+ public boolean isIgnoreExtraColumns() {
+ return ignoreExtraColumns;
+ }
+
// Implementation methods
//-------------------------------------------------------------------------
@@ -165,7 +189,16 @@ public class FlatpackDataFormat implemen
if (isFixed()) {
InputStream is = ResourceHelper.resolveMandatoryResourceAsInputStream(exchange.getContext().getClassResolver(), getDefinition());
InputStreamReader reader = new InputStreamReader(is, IOHelper.getCharsetName(exchange));
- return getParserFactory().newFixedLengthParser(reader, bodyReader);
+ Parser parser = getParserFactory().newFixedLengthParser(reader, bodyReader);
+ if (allowShortLines) {
+ parser.setHandlingShortLines(true);
+ parser.setIgnoreParseWarnings(true);
+ }
+ if (ignoreExtraColumns) {
+ parser.setIgnoreExtraColumns(true);
+ parser.setIgnoreParseWarnings(true);
+ }
+ return parser;
} else {
if (ObjectHelper.isEmpty(getDefinition())) {
return getParserFactory().newDelimitedParser(bodyReader, delimiter, textQualifier);
Modified: camel/branches/camel-2.10.x/components/camel-flatpack/src/test/java/org/apache/camel/component/flatpack/FixedLengthAllowLongTest.java
URL: http://svn.apache.org/viewvc/camel/branches/camel-2.10.x/components/camel-flatpack/src/test/java/org/apache/camel/component/flatpack/FixedLengthAllowLongTest.java?rev=1407761&r1=1407760&r2=1407761&view=diff
==============================================================================
--- camel/branches/camel-2.10.x/components/camel-flatpack/src/test/java/org/apache/camel/component/flatpack/FixedLengthAllowLongTest.java (original)
+++ camel/branches/camel-2.10.x/components/camel-flatpack/src/test/java/org/apache/camel/component/flatpack/FixedLengthAllowLongTest.java Sat Nov 10 11:25:21 2012
@@ -44,6 +44,9 @@ public class FixedLengthAllowLongTest ex
@EndpointInject(uri = "mock:results")
protected MockEndpoint results;
+ @EndpointInject(uri = "mock:results-df")
+ protected MockEndpoint resultsdf;
+
protected String[] expectedFirstName = {"JOHN-LONG", "JIMMY-LONG", "JANE-LONG", "FRED-LONG"};
@Test
@@ -63,4 +66,18 @@ public class FixedLengthAllowLongTest ex
counter++;
}
}
+
+ @Test
+ public void testFlatpackDataFormat() throws Exception {
+ resultsdf.expectedMessageCount(1);
+ resultsdf.assertIsSatisfied();
+
+ Exchange exchange = resultsdf.getReceivedExchanges().get(0);
+ DataSetList data = exchange.getIn().getBody(DataSetList.class);
+ int counter = 0;
+ for (Map<String, Object> map : data) {
+ assertEquals("FIRSTNAME", expectedFirstName[counter], map.get("FIRSTNAME"));
+ counter++;
+ }
+ }
}
Modified: camel/branches/camel-2.10.x/components/camel-flatpack/src/test/java/org/apache/camel/component/flatpack/FixedLengthAllowShortAndLongTest.java
URL: http://svn.apache.org/viewvc/camel/branches/camel-2.10.x/components/camel-flatpack/src/test/java/org/apache/camel/component/flatpack/FixedLengthAllowShortAndLongTest.java?rev=1407761&r1=1407760&r2=1407761&view=diff
==============================================================================
--- camel/branches/camel-2.10.x/components/camel-flatpack/src/test/java/org/apache/camel/component/flatpack/FixedLengthAllowShortAndLongTest.java (original)
+++ camel/branches/camel-2.10.x/components/camel-flatpack/src/test/java/org/apache/camel/component/flatpack/FixedLengthAllowShortAndLongTest.java Sat Nov 10 11:25:21 2012
@@ -44,10 +44,13 @@ public class FixedLengthAllowShortAndLon
@EndpointInject(uri = "mock:results")
protected MockEndpoint results;
+ @EndpointInject(uri = "mock:results-df")
+ protected MockEndpoint resultsdf;
+
protected String[] expectedFirstName = {"JOHN-LONG", "JIMMY-SHORT", "JANE-LONG", "FRED-NORMAL"};
@Test
- public void testCamel() throws Exception {
+ public void testFlatpack() throws Exception {
results.expectedMessageCount(4);
results.assertIsSatisfied();
@@ -63,4 +66,18 @@ public class FixedLengthAllowShortAndLon
counter++;
}
}
+
+ @Test
+ public void testFlatpackDataFormat() throws Exception {
+ resultsdf.expectedMessageCount(1);
+ resultsdf.assertIsSatisfied();
+
+ Exchange exchange = resultsdf.getReceivedExchanges().get(0);
+ DataSetList data = exchange.getIn().getBody(DataSetList.class);
+ int counter = 0;
+ for (Map<String, Object> map : data) {
+ assertEquals("FIRSTNAME", expectedFirstName[counter], map.get("FIRSTNAME"));
+ counter++;
+ }
+ }
}
Modified: camel/branches/camel-2.10.x/components/camel-flatpack/src/test/java/org/apache/camel/component/flatpack/FixedLengthAllowShortTest.java
URL: http://svn.apache.org/viewvc/camel/branches/camel-2.10.x/components/camel-flatpack/src/test/java/org/apache/camel/component/flatpack/FixedLengthAllowShortTest.java?rev=1407761&r1=1407760&r2=1407761&view=diff
==============================================================================
--- camel/branches/camel-2.10.x/components/camel-flatpack/src/test/java/org/apache/camel/component/flatpack/FixedLengthAllowShortTest.java (original)
+++ camel/branches/camel-2.10.x/components/camel-flatpack/src/test/java/org/apache/camel/component/flatpack/FixedLengthAllowShortTest.java Sat Nov 10 11:25:21 2012
@@ -44,6 +44,9 @@ public class FixedLengthAllowShortTest e
@EndpointInject(uri = "mock:results")
protected MockEndpoint results;
+ @EndpointInject(uri = "mock:results-df")
+ protected MockEndpoint resultsdf;
+
protected String[] expectedFirstName = {"JOHN-SHORT", "JIMMY-SHORT", "JANE-SHORT", "FRED-SHORT"};
@Test
@@ -63,4 +66,18 @@ public class FixedLengthAllowShortTest e
counter++;
}
}
+
+ @Test
+ public void testFlatpackDataFormat() throws Exception {
+ resultsdf.expectedMessageCount(1);
+ resultsdf.assertIsSatisfied();
+
+ Exchange exchange = resultsdf.getReceivedExchanges().get(0);
+ DataSetList data = exchange.getIn().getBody(DataSetList.class);
+ int counter = 0;
+ for (Map<String, Object> map : data) {
+ assertEquals("FIRSTNAME", expectedFirstName[counter], map.get("FIRSTNAME"));
+ counter++;
+ }
+ }
}
Modified: camel/branches/camel-2.10.x/components/camel-flatpack/src/test/resources/org/apache/camel/component/flatpack/FixedLengthAllowLongTest-context.xml
URL: http://svn.apache.org/viewvc/camel/branches/camel-2.10.x/components/camel-flatpack/src/test/resources/org/apache/camel/component/flatpack/FixedLengthAllowLongTest-context.xml?rev=1407761&r1=1407760&r2=1407761&view=diff
==============================================================================
--- camel/branches/camel-2.10.x/components/camel-flatpack/src/test/resources/org/apache/camel/component/flatpack/FixedLengthAllowLongTest-context.xml (original)
+++ camel/branches/camel-2.10.x/components/camel-flatpack/src/test/resources/org/apache/camel/component/flatpack/FixedLengthAllowLongTest-context.xml Sat Nov 10 11:25:21 2012
@@ -22,11 +22,20 @@
http://camel.apache.org/schema/spring http://camel.apache.org/schema/spring/camel-spring.xsd
">
+ <bean id="df" class="org.apache.camel.component.flatpack.FlatpackDataFormat">
+ <property name="definition" value="PEOPLE-FixedLength.pzmap.xml"/>
+ <property name="fixed" value="true"/>
+ <property name="ignoreExtraColumns" value="true"/>
+ </bean>
+
<!-- START SNIPPET: example -->
<camelContext xmlns="http://camel.apache.org/schema/spring">
<route>
<from uri="file://src/test/data/fixedLong?noop=true"/>
- <to uri="flatpack:fixed:PEOPLE-FixedLength.pzmap.xml?ignoreExtraColumns=true"/>
+ <multicast>
+ <to uri="flatpack:fixed:PEOPLE-FixedLength.pzmap.xml?ignoreExtraColumns=true"/>
+ <to uri="direct:df"/>
+ </multicast>
</route>
<route>
@@ -34,6 +43,13 @@
<convertBodyTo type="java.util.Map"/>
<to uri="mock:results"/>
</route>
+
+ <route>
+ <from uri="direct:df"/>
+ <unmarshal ref="df"/>
+ <to uri="mock:results-df"/>
+ </route>
+
</camelContext>
<!-- END SNIPPET: example -->
Modified: camel/branches/camel-2.10.x/components/camel-flatpack/src/test/resources/org/apache/camel/component/flatpack/FixedLengthAllowShortAndLongTest-context.xml
URL: http://svn.apache.org/viewvc/camel/branches/camel-2.10.x/components/camel-flatpack/src/test/resources/org/apache/camel/component/flatpack/FixedLengthAllowShortAndLongTest-context.xml?rev=1407761&r1=1407760&r2=1407761&view=diff
==============================================================================
--- camel/branches/camel-2.10.x/components/camel-flatpack/src/test/resources/org/apache/camel/component/flatpack/FixedLengthAllowShortAndLongTest-context.xml (original)
+++ camel/branches/camel-2.10.x/components/camel-flatpack/src/test/resources/org/apache/camel/component/flatpack/FixedLengthAllowShortAndLongTest-context.xml Sat Nov 10 11:25:21 2012
@@ -22,11 +22,21 @@
http://camel.apache.org/schema/spring http://camel.apache.org/schema/spring/camel-spring.xsd
">
+ <bean id="df" class="org.apache.camel.component.flatpack.FlatpackDataFormat">
+ <property name="definition" value="PEOPLE-FixedLength.pzmap.xml"/>
+ <property name="fixed" value="true"/>
+ <property name="ignoreExtraColumns" value="true"/>
+ <property name="allowShortLines" value="true"/>
+ </bean>
+
<!-- START SNIPPET: example -->
<camelContext xmlns="http://camel.apache.org/schema/spring">
<route>
<from uri="file://src/test/data/fixedMixed?noop=true"/>
- <to uri="flatpack:fixed:PEOPLE-FixedLength.pzmap.xml?ignoreExtraColumns=true&allowShortLines=true"/>
+ <multicast>
+ <to uri="flatpack:fixed:PEOPLE-FixedLength.pzmap.xml?ignoreExtraColumns=true&allowShortLines=true"/>
+ <to uri="direct:df"/>
+ </multicast>
</route>
<route>
@@ -34,6 +44,12 @@
<convertBodyTo type="java.util.Map"/>
<to uri="mock:results"/>
</route>
+
+ <route>
+ <from uri="direct:df"/>
+ <unmarshal ref="df"/>
+ <to uri="mock:results-df"/>
+ </route>
</camelContext>
<!-- END SNIPPET: example -->
Modified: camel/branches/camel-2.10.x/components/camel-flatpack/src/test/resources/org/apache/camel/component/flatpack/FixedLengthAllowShortTest-context.xml
URL: http://svn.apache.org/viewvc/camel/branches/camel-2.10.x/components/camel-flatpack/src/test/resources/org/apache/camel/component/flatpack/FixedLengthAllowShortTest-context.xml?rev=1407761&r1=1407760&r2=1407761&view=diff
==============================================================================
--- camel/branches/camel-2.10.x/components/camel-flatpack/src/test/resources/org/apache/camel/component/flatpack/FixedLengthAllowShortTest-context.xml (original)
+++ camel/branches/camel-2.10.x/components/camel-flatpack/src/test/resources/org/apache/camel/component/flatpack/FixedLengthAllowShortTest-context.xml Sat Nov 10 11:25:21 2012
@@ -22,11 +22,20 @@
http://camel.apache.org/schema/spring http://camel.apache.org/schema/spring/camel-spring.xsd
">
+ <bean id="df" class="org.apache.camel.component.flatpack.FlatpackDataFormat">
+ <property name="definition" value="PEOPLE-FixedLength.pzmap.xml"/>
+ <property name="fixed" value="true"/>
+ <property name="allowShortLines" value="true"/>
+ </bean>
+
<!-- START SNIPPET: example -->
- <camelContext xmlns="http://camel.apache.org/schema/spring">
+ <camelContext xmlns="http://camel.apache.org/schema/spring" trace="true">
<route>
<from uri="file://src/test/data/fixedShort?noop=true"/>
- <to uri="flatpack:fixed:PEOPLE-FixedLength.pzmap.xml?allowShortLines=true"/>
+ <multicast>
+ <to uri="flatpack:fixed:PEOPLE-FixedLength.pzmap.xml?allowShortLines=true"/>
+ <to uri="direct:df"/>
+ </multicast>
</route>
<route>
@@ -34,6 +43,12 @@
<convertBodyTo type="java.util.Map"/>
<to uri="mock:results"/>
</route>
+
+ <route>
+ <from uri="direct:df"/>
+ <unmarshal ref="df"/>
+ <to uri="mock:results-df"/>
+ </route>
</camelContext>
<!-- END SNIPPET: example -->