[camel] branch camel-3.7.x updated: CAMEL-16152: Fix tokenize pair with simple based tokens. Thanks to Michael Stephan for reportig and the example to use as reproducer.

[da...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

davsclaus pushed a commit to branch camel-3.7.x
in repository https://gitbox.apache.org/repos/asf/camel.git


The following commit(s) were added to refs/heads/camel-3.7.x by this push:
     new 292d0ec  CAMEL-16152: Fix tokenize pair with simple based tokens. Thanks to Michael Stephan for reportig and the example to use as reproducer.
292d0ec is described below

commit 292d0ecbcea36299c4848a55933f3ba302572b17
Author: Claus Ibsen <cl...@gmail.com>
AuthorDate: Sun Feb 7 13:44:24 2021 +0100

    CAMEL-16152: Fix tokenize pair with simple based tokens. Thanks to Michael Stephan for reportig and the example to use as reproducer.
---
 .../tokenizer/TokenizePairTokenSimpleTest.java     | 65 ++++++++++++++++++++++
 .../apache/camel/support/GroupTokenIterator.java   |  9 ++-
 .../main/java/org/apache/camel/util/Scanner.java   |  4 ++
 3 files changed, 77 insertions(+), 1 deletion(-)

diff --git a/core/camel-core/src/test/java/org/apache/camel/language/tokenizer/TokenizePairTokenSimpleTest.java b/core/camel-core/src/test/java/org/apache/camel/language/tokenizer/TokenizePairTokenSimpleTest.java
new file mode 100644
index 0000000..982823e
--- /dev/null
+++ b/core/camel-core/src/test/java/org/apache/camel/language/tokenizer/TokenizePairTokenSimpleTest.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.language.tokenizer;
+
+import org.apache.camel.ContextTestSupport;
+import org.apache.camel.builder.RouteBuilder;
+import org.junit.jupiter.api.Test;
+
+public class TokenizePairTokenSimpleTest extends ContextTestSupport {
+
+    @Test
+    public void testTokenizeConstant() throws Exception {
+        context.addRoutes(new RouteBuilder() {
+            @Override
+            public void configure() throws Exception {
+                from("direct:start")
+                        .split().tokenize("B", 2)
+                        .to("mock:line");
+
+            }
+        });
+        context.start();
+
+        getMockEndpoint("mock:line").expectedBodiesReceived("aaBaa", "aa");
+
+        template.sendBody("direct:start", "aaBaaBaaB");
+
+        assertMockEndpointsSatisfied();
+    }
+
+    @Test
+    public void testTokenizeSimple() throws Exception {
+        context.addRoutes(new RouteBuilder() {
+            @Override
+            public void configure() throws Exception {
+                from("direct:start")
+                        .split().tokenize("${header.test}", 2)
+                        .to("mock:line");
+
+            }
+        });
+        context.start();
+
+        getMockEndpoint("mock:line").expectedBodiesReceived("aaBaa", "aa");
+
+        template.sendBodyAndHeader("direct:start", "aaBaaBaaB", "test", "B");
+
+        assertMockEndpointsSatisfied();
+    }
+
+}
diff --git a/core/camel-support/src/main/java/org/apache/camel/support/GroupTokenIterator.java b/core/camel-support/src/main/java/org/apache/camel/support/GroupTokenIterator.java
index 44c65c3..a5ffa1a7 100644
--- a/core/camel-support/src/main/java/org/apache/camel/support/GroupTokenIterator.java
+++ b/core/camel-support/src/main/java/org/apache/camel/support/GroupTokenIterator.java
@@ -28,6 +28,7 @@ import org.apache.camel.Exchange;
 import org.apache.camel.NoTypeConversionAvailableException;
 import org.apache.camel.RuntimeCamelException;
 import org.apache.camel.util.IOHelper;
+import org.apache.camel.util.Scanner;
 
 /**
  * Group based {@link Iterator} which groups the given {@link Iterator} a number of times and then return a combined
@@ -64,7 +65,13 @@ public final class GroupTokenIterator implements Iterator<Object>, Closeable {
         this.exchange = exchange;
         this.camelContext = exchange.getContext();
         this.it = it;
-        this.token = token;
+        // if the iterator is a scanner then it may have a dynamic delimiter
+        // so we need to use the actual evaluated delimiter as token
+        if (LanguageSupport.hasSimpleFunction(token) && it instanceof Scanner) {
+            this.token = ((Scanner) it).getDelim();
+        } else {
+            this.token = token;
+        }
         this.group = group;
         if (group <= 0) {
             throw new IllegalArgumentException("Group must be a positive number, was: " + group);
diff --git a/core/camel-util/src/main/java/org/apache/camel/util/Scanner.java b/core/camel-util/src/main/java/org/apache/camel/util/Scanner.java
index 57d9d53..5f3acfd 100644
--- a/core/camel-util/src/main/java/org/apache/camel/util/Scanner.java
+++ b/core/camel-util/src/main/java/org/apache/camel/util/Scanner.java
@@ -151,6 +151,10 @@ public final class Scanner implements Iterator<String>, Closeable {
         }
     }
 
+    public String getDelim() {
+        return delimPattern.pattern();
+    }
+
     private void saveState() {
         savedPosition = position;
     }