You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@seatunnel.apache.org by co...@apache.org on 2025/07/02 01:46:55 UTC

(seatunnel) branch dev updated: [Bug][connector-http] Fix paging request running infinitely (#9504)

This is an automated email from the ASF dual-hosted git repository.

corgy pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/seatunnel.git


The following commit(s) were added to refs/heads/dev by this push:
     new 1844e04c97 [Bug][connector-http] Fix paging request running infinitely (#9504)
1844e04c97 is described below

commit 1844e04c9778d5caa9d71e66f2fb49941e04adbe
Author: jiachuan.zhu <ch...@gmail.com>
AuthorDate: Wed Jul 2 09:46:49 2025 +0800

    [Bug][connector-http] Fix paging request running infinitely (#9504)
---
 .../seatunnel/http/source/HttpSourceReader.java    |  16 ++-
 .../http/HttpSourceReaderInternalPollNextTest.java | 141 +++++++++++++++++++++
 .../HttpSourceReaderUpdateRequestParamTest.java    |  36 ++++++
 3 files changed, 186 insertions(+), 7 deletions(-)

diff --git a/seatunnel-connectors-v2/connector-http/connector-http-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/http/source/HttpSourceReader.java b/seatunnel-connectors-v2/connector-http/connector-http-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/http/source/HttpSourceReader.java
index a9ed773a29..5dd4e5c365 100644
--- a/seatunnel-connectors-v2/connector-http/connector-http-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/http/source/HttpSourceReader.java
+++ b/seatunnel-connectors-v2/connector-http/connector-http-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/http/source/HttpSourceReader.java
@@ -73,6 +73,11 @@ public class HttpSourceReader extends AbstractSingleSplitReader<SeaTunnelRow> {
             Configuration.defaultConfiguration().addOptions(DEFAULT_OPTIONS);
     private boolean noMoreElementFlag = true;
     private Optional<PageInfo> pageInfoOptional = Optional.empty();
+    /**
+     * Holds the original request body template for placeholder replacement. This ensures that the
+     * state is not unintentionally mutated during pagination.
+     */
+    private String rawBody = null;
 
     public HttpSourceReader(
             HttpParameter httpParameter,
@@ -85,6 +90,7 @@ public class HttpSourceReader extends AbstractSingleSplitReader<SeaTunnelRow> {
         this.deserializationCollector = new DeserializationCollector(deserializationSchema);
         this.jsonField = jsonField;
         this.contentJson = contentJson;
+        this.rawBody = httpParameter.getBody();
     }
 
     public HttpSourceReader(
@@ -100,6 +106,7 @@ public class HttpSourceReader extends AbstractSingleSplitReader<SeaTunnelRow> {
         this.jsonField = jsonField;
         this.contentJson = contentJson;
         this.pageInfoOptional = Optional.ofNullable(pageInfo);
+        this.rawBody = httpParameter.getBody();
     }
 
     @Override
@@ -206,14 +213,10 @@ public class HttpSourceReader extends AbstractSingleSplitReader<SeaTunnelRow> {
         }
 
         // 2. param in body
-        if (!Strings.isNullOrEmpty(this.httpParameter.getBody())) {
+        if (!Strings.isNullOrEmpty(this.rawBody)) {
             String processedBody =
                     processBodyString(
-                            this.httpParameter.getBody(),
-                            pageField,
-                            pageValue,
-                            usePlaceholderReplacement);
-
+                            this.rawBody, pageField, pageValue, usePlaceholderReplacement);
             // Process cursor if available
             if (pageInfo.getPageCursorFieldName() != null && pageInfo.getCursor() != null) {
                 processedBody =
@@ -344,7 +347,6 @@ public class HttpSourceReader extends AbstractSingleSplitReader<SeaTunnelRow> {
                         pollAndCollectData(output);
                         Thread.sleep(10);
                     }
-
                 } else {
                     // default page number pagination
                     Long pageIndex = info.getPageIndex();
diff --git a/seatunnel-connectors-v2/connector-http/connector-http-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/http/HttpSourceReaderInternalPollNextTest.java b/seatunnel-connectors-v2/connector-http/connector-http-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/http/HttpSourceReaderInternalPollNextTest.java
new file mode 100644
index 0000000000..8b04031ef9
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-http/connector-http-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/http/HttpSourceReaderInternalPollNextTest.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.seatunnel.connectors.seatunnel.http;
+
+import org.apache.seatunnel.api.source.Collector;
+import org.apache.seatunnel.api.table.type.BasicType;
+import org.apache.seatunnel.api.table.type.SeaTunnelDataType;
+import org.apache.seatunnel.api.table.type.SeaTunnelRow;
+import org.apache.seatunnel.api.table.type.SeaTunnelRowType;
+import org.apache.seatunnel.common.utils.JsonUtils;
+import org.apache.seatunnel.connectors.seatunnel.common.source.SingleSplitReaderContext;
+import org.apache.seatunnel.connectors.seatunnel.http.client.HttpClientProvider;
+import org.apache.seatunnel.connectors.seatunnel.http.client.HttpResponse;
+import org.apache.seatunnel.connectors.seatunnel.http.config.HttpPaginationType;
+import org.apache.seatunnel.connectors.seatunnel.http.config.HttpParameter;
+import org.apache.seatunnel.connectors.seatunnel.http.config.HttpRequestMethod;
+import org.apache.seatunnel.connectors.seatunnel.http.config.JsonField;
+import org.apache.seatunnel.connectors.seatunnel.http.config.PageInfo;
+import org.apache.seatunnel.connectors.seatunnel.http.source.HttpSourceReader;
+import org.apache.seatunnel.connectors.seatunnel.http.source.SimpleTextDeserializationSchema;
+
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.Mock;
+import org.mockito.MockitoAnnotations;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyBoolean;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.when;
+
+public class HttpSourceReaderInternalPollNextTest {
+    private HttpParameter httpParameter;
+    private JsonField jsonField;
+    private SimpleTextDeserializationSchema deserializationSchema;
+    private HttpSourceReader httpSourceReader;
+    private AutoCloseable mock;
+    @Mock private SingleSplitReaderContext context;
+    @Mock private Collector<SeaTunnelRow> collector;
+    @Mock private HttpClientProvider httpClientProvider;
+    @Mock private HttpResponse httpResponse;
+
+    @BeforeEach
+    public void setup() throws Exception {
+        mock = MockitoAnnotations.openMocks(this);
+        when(httpResponse.getCode()).thenReturn(200);
+        when(collector.getCheckpointLock()).thenReturn(new Object());
+        when(httpClientProvider.execute(
+                        anyString(), anyString(), any(), any(), any(), anyBoolean()))
+                .thenAnswer(
+                        invocation -> {
+                            String requestBody = invocation.getArgument(4);
+                            if (requestBody != null && requestBody.contains("\"page\":\"1\"")) {
+                                when(httpResponse.getContent())
+                                        .thenReturn("[{\"key1\":\"v1\",\"key2\":\"v2\"}]");
+                            } else {
+                                when(httpResponse.getContent()).thenReturn("[]");
+                            }
+                            when(httpResponse.getCode()).thenReturn(200);
+                            return httpResponse;
+                        });
+
+        httpParameter = new HttpParameter();
+        httpParameter.setUrl("http://test-url.com");
+        httpParameter.setMethod(HttpRequestMethod.GET);
+        Map<String, String> fields = new HashMap<>();
+        fields.put("key1", "$[*].key1");
+        fields.put("key2", "$[*].key2");
+        jsonField = JsonField.builder().fields(fields).build();
+
+        // Create the schema with two string fields
+        SeaTunnelRowType rowType =
+                new SeaTunnelRowType(
+                        new String[] {"key1", "key2"},
+                        new SeaTunnelDataType[] {BasicType.STRING_TYPE, BasicType.STRING_TYPE});
+        deserializationSchema = new SimpleTextDeserializationSchema(rowType);
+        collector =
+                new Collector<SeaTunnelRow>() {
+                    @Override
+                    public void collect(SeaTunnelRow record) {}
+
+                    @Override
+                    public Object getCheckpointLock() {
+                        return null;
+                    }
+                };
+    }
+
+    @Test
+    public void testPageNumberPlaceHolderRequestBodyUpdate() throws Exception {
+        String bodyJson = "{\"page\":\"${page}\",\"limit\":10}";
+        httpParameter.setBody(bodyJson);
+
+        PageInfo pageInfo = new PageInfo();
+        pageInfo.setPageField("page");
+        pageInfo.setPageIndex(1L);
+        pageInfo.setBatchSize(1);
+        pageInfo.setPageType(HttpPaginationType.PAGE_NUMBER.getCode());
+        pageInfo.setUsePlaceholderReplacement(true);
+        pageInfo.setTotalPageSize(2L);
+
+        httpSourceReader =
+                new HttpSourceReader(
+                        httpParameter, context, deserializationSchema, jsonField, null, pageInfo);
+        httpSourceReader.open();
+
+        httpSourceReader.internalPollNext(collector);
+
+        // Verify the body was updated correctly
+        Assertions.assertEquals("{\"page\":\"2\",\"limit\":10}", httpParameter.getBody());
+        Map<String, Object> bodyMap =
+                JsonUtils.toMap(JsonUtils.stringToJsonNode(httpParameter.getBody()));
+        Assertions.assertEquals("2", bodyMap.get("page"));
+        Assertions.assertEquals(10, bodyMap.get("limit"));
+        httpSourceReader.close();
+    }
+
+    @AfterEach
+    public void tearDown() throws Exception {
+        mock.close();
+    }
+}
diff --git a/seatunnel-connectors-v2/connector-http/connector-http-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/http/HttpSourceReaderUpdateRequestParamTest.java b/seatunnel-connectors-v2/connector-http/connector-http-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/http/HttpSourceReaderUpdateRequestParamTest.java
index fd207857b5..a567473ba5 100644
--- a/seatunnel-connectors-v2/connector-http/connector-http-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/http/HttpSourceReaderUpdateRequestParamTest.java
+++ b/seatunnel-connectors-v2/connector-http/connector-http-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/http/HttpSourceReaderUpdateRequestParamTest.java
@@ -26,6 +26,7 @@ import org.apache.seatunnel.common.utils.JsonUtils;
 import org.apache.seatunnel.connectors.seatunnel.common.source.SingleSplitReaderContext;
 import org.apache.seatunnel.connectors.seatunnel.http.client.HttpClientProvider;
 import org.apache.seatunnel.connectors.seatunnel.http.client.HttpResponse;
+import org.apache.seatunnel.connectors.seatunnel.http.config.HttpPaginationType;
 import org.apache.seatunnel.connectors.seatunnel.http.config.HttpParameter;
 import org.apache.seatunnel.connectors.seatunnel.http.config.HttpRequestMethod;
 import org.apache.seatunnel.connectors.seatunnel.http.config.JsonField;
@@ -215,6 +216,7 @@ public class HttpSourceReaderUpdateRequestParamTest {
         pageInfo.setUsePlaceholderReplacement(true);
 
         // Call updateRequestParam method directly
+        httpSourceReader.setRawBody(bodyJson);
         httpSourceReader.updateRequestParam(pageInfo, true);
 
         // Verify the body was updated correctly
@@ -241,6 +243,7 @@ public class HttpSourceReaderUpdateRequestParamTest {
         pageInfo.setUsePlaceholderReplacement(true);
 
         // Call updateRequestParam method directly
+        httpSourceReader.setRawBody(bodyJson);
         httpSourceReader.updateRequestParam(pageInfo, true);
 
         // Verify the body was updated correctly
@@ -268,6 +271,7 @@ public class HttpSourceReaderUpdateRequestParamTest {
         pageInfo.setUsePlaceholderReplacement(true);
 
         // Call updateRequestParam method directly
+        httpSourceReader.setRawBody(bodyJson);
         httpSourceReader.updateRequestParam(pageInfo, true);
 
         // Verify the nested body was updated correctly
@@ -319,6 +323,7 @@ public class HttpSourceReaderUpdateRequestParamTest {
         pageInfo.setUsePlaceholderReplacement(false);
 
         // Call updateRequestParam method directly
+        httpSourceReader.setRawBody(bodyJson);
         httpSourceReader.updateRequestParam(pageInfo, false);
 
         // Verify the body was updated correctly using key-based replacement
@@ -345,6 +350,7 @@ public class HttpSourceReaderUpdateRequestParamTest {
         pageInfo.setUsePlaceholderReplacement(false);
 
         // Call updateRequestParam method directly
+        httpSourceReader.setRawBody(bodyJson);
         httpSourceReader.updateRequestParam(pageInfo, false);
 
         // Verify the nested body was updated correctly using key-based replacement
@@ -372,6 +378,7 @@ public class HttpSourceReaderUpdateRequestParamTest {
         pageInfo.setUsePlaceholderReplacement(true);
 
         // Call updateRequestParam method directly
+        httpSourceReader.setRawBody(bodyJson);
         httpSourceReader.updateRequestParam(pageInfo, true);
 
         // Verify the body was updated correctly
@@ -398,6 +405,7 @@ public class HttpSourceReaderUpdateRequestParamTest {
         pageInfo.setUsePlaceholderReplacement(true);
 
         // Call updateRequestParam method directly
+        httpSourceReader.setRawBody(bodyJson);
         httpSourceReader.updateRequestParam(pageInfo, true);
 
         // Verify the body was updated correctly
@@ -425,6 +433,7 @@ public class HttpSourceReaderUpdateRequestParamTest {
         pageInfo.setUsePlaceholderReplacement(true);
 
         // Call updateRequestParam method directly
+        httpSourceReader.setRawBody(bodyJson);
         httpSourceReader.updateRequestParam(pageInfo, true);
 
         // Verify the body was updated correctly
@@ -455,6 +464,7 @@ public class HttpSourceReaderUpdateRequestParamTest {
         pageInfo.setUsePlaceholderReplacement(true);
 
         // Call updateRequestParam method directly
+        httpSourceReader.setRawBody(bodyJson);
         httpSourceReader.updateRequestParam(pageInfo, true);
 
         // Verify the body was updated correctly
@@ -485,6 +495,7 @@ public class HttpSourceReaderUpdateRequestParamTest {
         pageInfo.setUsePlaceholderReplacement(true);
 
         // Call updateRequestParam method directly
+        httpSourceReader.setRawBody(bodyJson);
         httpSourceReader.updateRequestParam(pageInfo, true);
 
         // Verify the body was updated correctly
@@ -505,6 +516,31 @@ public class HttpSourceReaderUpdateRequestParamTest {
         Assertions.assertEquals(10, bodyMap.get("limit"));
     }
 
+    @Test
+    public void testInternalPollNextWithBodyPlaceholderBatchSize() throws Exception {
+        // Setup test data
+        String bodyJson = "{\"page\":\"${page}\",\"limit\":10}";
+        httpParameter.setBody(bodyJson);
+
+        PageInfo pageInfo = new PageInfo();
+        pageInfo.setPageField("page");
+        pageInfo.setPageIndex(5L);
+        pageInfo.setBatchSize(10);
+        pageInfo.setPageType(HttpPaginationType.PAGE_NUMBER.getCode());
+        pageInfo.setUsePlaceholderReplacement(true);
+
+        // Call updateRequestParam method directly
+        httpSourceReader.setRawBody(bodyJson);
+        httpSourceReader.updateRequestParam(pageInfo, true);
+
+        // Verify the body was updated correctly
+        Assertions.assertEquals("{\"page\":\"5\",\"limit\":10}", httpParameter.getBody());
+        Map<String, Object> bodyMap =
+                JsonUtils.toMap(JsonUtils.stringToJsonNode(httpParameter.getBody()));
+        Assertions.assertEquals("5", bodyMap.get("page"));
+        Assertions.assertEquals(10, bodyMap.get("limit"));
+    }
+
     @Test
     public void testUpdateRequestParamWithHeaderOnlyPageNumberOccurNPE() throws Exception {
         // Setup test data