You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@camel.apache.org by da...@apache.org on 2011/11/04 13:31:39 UTC
svn commit: r1197534 - in /camel/trunk/camel-core/src:
main/java/org/apache/camel/builder/
main/java/org/apache/camel/language/tokenizer/
main/java/org/apache/camel/model/language/
main/java/org/apache/camel/support/ test/java/org/apache/camel/language/
Author: davsclaus
Date: Fri Nov 4 12:31:38 2011
New Revision: 1197534
URL: http://svn.apache.org/viewvc?rev=1197534&view=rev
Log:
CAMEL-4617: Added includeTokens option tokenize pair.
Modified:
camel/trunk/camel-core/src/main/java/org/apache/camel/builder/ExpressionBuilder.java
camel/trunk/camel-core/src/main/java/org/apache/camel/language/tokenizer/TokenizeLanguage.java
camel/trunk/camel-core/src/main/java/org/apache/camel/model/language/TokenizerExpression.java
camel/trunk/camel-core/src/main/java/org/apache/camel/support/TokenPairExpressionIterator.java
camel/trunk/camel-core/src/main/java/org/apache/camel/support/TokenXMLPairExpressionIterator.java
camel/trunk/camel-core/src/test/java/org/apache/camel/language/TokenizerTest.java
Modified: camel/trunk/camel-core/src/main/java/org/apache/camel/builder/ExpressionBuilder.java
URL: http://svn.apache.org/viewvc/camel/trunk/camel-core/src/main/java/org/apache/camel/builder/ExpressionBuilder.java?rev=1197534&r1=1197533&r2=1197534&view=diff
==============================================================================
--- camel/trunk/camel-core/src/main/java/org/apache/camel/builder/ExpressionBuilder.java (original)
+++ camel/trunk/camel-core/src/main/java/org/apache/camel/builder/ExpressionBuilder.java Fri Nov 4 12:31:38 2011
@@ -1028,8 +1028,8 @@ public final class ExpressionBuilder {
/**
* Returns an {@link TokenPairExpressionIterator} expression
*/
- public static Expression tokenizePairExpression(String startToken, String endToken) {
- return new TokenPairExpressionIterator(startToken, endToken);
+ public static Expression tokenizePairExpression(String startToken, String endToken, boolean includeTokens) {
+ return new TokenPairExpressionIterator(startToken, endToken, includeTokens);
}
/**
Modified: camel/trunk/camel-core/src/main/java/org/apache/camel/language/tokenizer/TokenizeLanguage.java
URL: http://svn.apache.org/viewvc/camel/trunk/camel-core/src/main/java/org/apache/camel/language/tokenizer/TokenizeLanguage.java?rev=1197534&r1=1197533&r2=1197534&view=diff
==============================================================================
--- camel/trunk/camel-core/src/main/java/org/apache/camel/language/tokenizer/TokenizeLanguage.java (original)
+++ camel/trunk/camel-core/src/main/java/org/apache/camel/language/tokenizer/TokenizeLanguage.java Fri Nov 4 12:31:38 2011
@@ -45,6 +45,7 @@ public class TokenizeLanguage implements
private String headerName;
private boolean regex;
private boolean xml;
+ private boolean includeTokens;
public static Expression tokenize(String token) {
return tokenize(token, false);
@@ -69,10 +70,11 @@ public class TokenizeLanguage implements
return language.createExpression(null);
}
- public static Expression tokenizePair(String startToken, String endToken) {
+ public static Expression tokenizePair(String startToken, String endToken, boolean includeTokens) {
TokenizeLanguage language = new TokenizeLanguage();
language.setToken(startToken);
language.setEndToken(endToken);
+ language.setIncludeTokens(includeTokens);
return language.createExpression(null);
}
@@ -81,6 +83,7 @@ public class TokenizeLanguage implements
language.setToken(tagName);
language.setInheritNamespaceTagName(inheritNamespaceTagName);
language.setXml(true);
+ language.setIncludeTokens(true);
return language.createExpression(null);
}
@@ -97,7 +100,7 @@ public class TokenizeLanguage implements
if (isXml()) {
return ExpressionBuilder.tokenizeXMLExpression(token, inheritNamespaceTagName);
} else if (endToken != null) {
- return ExpressionBuilder.tokenizePairExpression(token, endToken);
+ return ExpressionBuilder.tokenizePairExpression(token, endToken, includeTokens);
}
// use the regular tokenizer
@@ -164,6 +167,14 @@ public class TokenizeLanguage implements
this.xml = xml;
}
+ public boolean isIncludeTokens() {
+ return includeTokens;
+ }
+
+ public void setIncludeTokens(boolean includeTokens) {
+ this.includeTokens = includeTokens;
+ }
+
public boolean isSingleton() {
return false;
}
Modified: camel/trunk/camel-core/src/main/java/org/apache/camel/model/language/TokenizerExpression.java
URL: http://svn.apache.org/viewvc/camel/trunk/camel-core/src/main/java/org/apache/camel/model/language/TokenizerExpression.java?rev=1197534&r1=1197533&r2=1197534&view=diff
==============================================================================
--- camel/trunk/camel-core/src/main/java/org/apache/camel/model/language/TokenizerExpression.java (original)
+++ camel/trunk/camel-core/src/main/java/org/apache/camel/model/language/TokenizerExpression.java Fri Nov 4 12:31:38 2011
@@ -47,6 +47,8 @@ public class TokenizerExpression extends
private Boolean regex;
@XmlAttribute
private Boolean xml;
+ @XmlAttribute
+ private Boolean includeTokens;
public TokenizerExpression() {
}
@@ -104,6 +106,14 @@ public class TokenizerExpression extends
this.xml = xml;
}
+ public Boolean getIncludeTokens() {
+ return includeTokens;
+ }
+
+ public void setIncludeTokens(Boolean includeTokens) {
+ this.includeTokens = includeTokens;
+ }
+
@Override
public Expression createExpression(CamelContext camelContext) {
TokenizeLanguage language = new TokenizeLanguage();
@@ -117,6 +127,9 @@ public class TokenizerExpression extends
if (xml != null) {
language.setXml(xml);
}
+ if (includeTokens != null) {
+ language.setIncludeTokens(includeTokens);
+ }
return language.createExpression();
}
Modified: camel/trunk/camel-core/src/main/java/org/apache/camel/support/TokenPairExpressionIterator.java
URL: http://svn.apache.org/viewvc/camel/trunk/camel-core/src/main/java/org/apache/camel/support/TokenPairExpressionIterator.java?rev=1197534&r1=1197533&r2=1197534&view=diff
==============================================================================
--- camel/trunk/camel-core/src/main/java/org/apache/camel/support/TokenPairExpressionIterator.java (original)
+++ camel/trunk/camel-core/src/main/java/org/apache/camel/support/TokenPairExpressionIterator.java Fri Nov 4 12:31:38 2011
@@ -40,12 +40,14 @@ public class TokenPairExpressionIterator
protected final String startToken;
protected final String endToken;
+ protected final boolean includeTokens;
- public TokenPairExpressionIterator(String startToken, String endToken) {
- this.startToken = startToken;
- this.endToken = endToken;
+ public TokenPairExpressionIterator(String startToken, String endToken, boolean includeTokens) {
ObjectHelper.notEmpty(startToken, "startToken");
ObjectHelper.notEmpty(endToken, "endToken");
+ this.startToken = startToken;
+ this.endToken = endToken;
+ this.includeTokens = includeTokens;
}
@Override
@@ -69,7 +71,7 @@ public class TokenPairExpressionIterator
* @return the iterator
*/
protected Iterator createIterator(InputStream in, String charset) {
- TokenPairIterator iterator = new TokenPairIterator(startToken, endToken, in, charset);
+ TokenPairIterator iterator = new TokenPairIterator(startToken, endToken, includeTokens, in, charset);
iterator.init();
return iterator;
}
@@ -85,22 +87,43 @@ public class TokenPairExpressionIterator
static class TokenPairIterator implements Iterator, Closeable {
final String startToken;
+ String scanStartToken;
final String endToken;
+ String scanEndToken;
+ final boolean includeTokens;
final InputStream in;
final String charset;
Scanner scanner;
Object image;
- TokenPairIterator(String startToken, String endToken, InputStream in, String charset) {
+ TokenPairIterator(String startToken, String endToken, boolean includeTokens, InputStream in, String charset) {
this.startToken = startToken;
this.endToken = endToken;
+ this.includeTokens = includeTokens;
this.in = in;
this.charset = charset;
+
+ // make sure [ and ] is escaped as we use scanner which is reg exp based
+ // where [ and ] have special meaning
+ scanStartToken = startToken;
+ if (scanStartToken.startsWith("[")) {
+ scanStartToken = "\\" + scanStartToken;
+ }
+ if (scanStartToken.endsWith("]")) {
+ scanStartToken = scanStartToken.substring(0, startToken.length() - 1) + "\\]";
+ }
+ scanEndToken = endToken;
+ if (scanEndToken.startsWith("[")) {
+ scanEndToken = "\\" + scanEndToken;
+ }
+ if (scanEndToken.endsWith("]")) {
+ scanEndToken = scanEndToken.substring(0, scanEndToken.length() - 1) + "\\]";
+ }
}
void init() {
// use end token as delimiter
- this.scanner = new Scanner(in, charset).useDelimiter(endToken);
+ this.scanner = new Scanner(in, charset).useDelimiter(scanEndToken);
// this iterator will do look ahead as we may have data
// after the last end token, which the scanner would find
// so we need to be one step ahead of the scanner
@@ -141,7 +164,7 @@ public class TokenPairExpressionIterator
next = ObjectHelper.after(next, startToken);
// include tokens in answer
- if (next != null) {
+ if (next != null && includeTokens) {
StringBuilder sb = new StringBuilder();
next = sb.append(startToken).append(next).append(endToken).toString();
}
Modified: camel/trunk/camel-core/src/main/java/org/apache/camel/support/TokenXMLPairExpressionIterator.java
URL: http://svn.apache.org/viewvc/camel/trunk/camel-core/src/main/java/org/apache/camel/support/TokenXMLPairExpressionIterator.java?rev=1197534&r1=1197533&r2=1197534&view=diff
==============================================================================
--- camel/trunk/camel-core/src/main/java/org/apache/camel/support/TokenXMLPairExpressionIterator.java (original)
+++ camel/trunk/camel-core/src/main/java/org/apache/camel/support/TokenXMLPairExpressionIterator.java Fri Nov 4 12:31:38 2011
@@ -44,7 +44,7 @@ public class TokenXMLPairExpressionItera
protected final String inheritNamespaceToken;
public TokenXMLPairExpressionIterator(String startToken, String endToken, String inheritNamespaceToken) {
- super(startToken, endToken);
+ super(startToken, endToken, true);
// namespace token is optional
this.inheritNamespaceToken = inheritNamespaceToken;
@@ -79,7 +79,7 @@ public class TokenXMLPairExpressionItera
private String rootTokenNamespaces;
XMLTokenPairIterator(String startToken, String endToken, String inheritNamespaceToken, InputStream in, String charset) {
- super(startToken, endToken, in, charset);
+ super(startToken, endToken, true, in, charset);
// remove any ending > as we need to support attributes on the tags, so we need to use a reg exp pattern
String token = startToken.substring(0, startToken.length() - 1) + SCAN_TOKEN_REGEX;
Modified: camel/trunk/camel-core/src/test/java/org/apache/camel/language/TokenizerTest.java
URL: http://svn.apache.org/viewvc/camel/trunk/camel-core/src/test/java/org/apache/camel/language/TokenizerTest.java?rev=1197534&r1=1197533&r2=1197534&view=diff
==============================================================================
--- camel/trunk/camel-core/src/test/java/org/apache/camel/language/TokenizerTest.java (original)
+++ camel/trunk/camel-core/src/test/java/org/apache/camel/language/TokenizerTest.java Fri Nov 4 12:31:38 2011
@@ -99,6 +99,42 @@ public class TokenizerTest extends Excha
assertEquals(false, lan.isSingleton());
}
+ public void testTokenizePairSpecial() throws Exception {
+ Expression exp = TokenizeLanguage.tokenizePair("!", "@", false);
+
+ exchange.getIn().setBody("2011-11-11\n!James@!Claus@\n2 records");
+
+ List names = exp.evaluate(exchange, List.class);
+ assertEquals(2, names.size());
+
+ assertEquals("James", names.get(0));
+ assertEquals("Claus", names.get(1));
+ }
+
+ public void testTokenizePair() throws Exception {
+ Expression exp = TokenizeLanguage.tokenizePair("[START]", "[END]", false);
+
+ exchange.getIn().setBody("2011-11-11\n[START]James[END]\n[START]Claus[END]\n2 records");
+
+ List names = exp.evaluate(exchange, List.class);
+ assertEquals(2, names.size());
+
+ assertEquals("James", names.get(0));
+ assertEquals("Claus", names.get(1));
+ }
+
+ public void testTokenizePairIncludeTokens() throws Exception {
+ Expression exp = TokenizeLanguage.tokenizePair("[START]", "[END]", true);
+
+ exchange.getIn().setBody("2011-11-11\n[START]James[END]\n[START]Claus[END]\n2 records");
+
+ List names = exp.evaluate(exchange, List.class);
+ assertEquals(2, names.size());
+
+ assertEquals("[START]James[END]", names.get(0));
+ assertEquals("[START]Claus[END]", names.get(1));
+ }
+
public void testTokenizeXMLPair() throws Exception {
Expression exp = TokenizeLanguage.tokenizeXML("<person>", null);