You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by sh...@apache.org on 2026/03/31 17:12:10 UTC

(pinot) branch master updated: Added integration test for multi-topic realtime clusters (#18039)

This is an automated email from the ASF dual-hosted git repository.

shauryachats pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git


The following commit(s) were added to refs/heads/master by this push:
     new 2e9f35c1a7c Added integration test for multi-topic realtime clusters (#18039)
2e9f35c1a7c is described below

commit 2e9f35c1a7cf22cd98ac11db7f0010dd6074ebcf
Author: Shaurya Chaturvedi <sh...@gmail.com>
AuthorDate: Tue Mar 31 10:12:04 2026 -0700

    Added integration test for multi-topic realtime clusters (#18039)
    
    This PR adds MultiTopicRealtimeClusterIntegrationTest, an integration test that validates realtime ingestion from multiple Kafka topics into a single table using the streamConfigMaps multi-stream configuration.
    
    The motivation was that currently there is no E2E integration test for multi-topic realtime tables (I encountered the same bug which was fixed in #17953).
    
    The test is parameterized over N topics (default 3, overridable via getNumTopics()). For each topic, it generates CSV records with a mutually exclusive source column and non-overlapping value ranges, then verifies correctness through targeted queries:
    
    - Total doc count equals the sum across all topics
    - GROUP BY source returns exactly N groups with the expected counts
    - Filter and aggregation queries isolate each topic's data correctly
    - Value ranges don't leak between topics
    - Segments exist for all N stream config indices (verified via partition group IDs)
    - DISTINCT source returns exactly N values
    - No existing tests covered multi-topic ingestion end-to-end.
---
 .../CustomDataQueryClusterIntegrationTest.java     |  14 +
 .../MultiTopicRealtimeClusterIntegrationTest.java  | 409 +++++++++++++++++++++
 2 files changed, 423 insertions(+)

diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/CustomDataQueryClusterIntegrationTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/CustomDataQueryClusterIntegrationTest.java
index e7b9f10d9cf..381c5847287 100644
--- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/CustomDataQueryClusterIntegrationTest.java
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/CustomDataQueryClusterIntegrationTest.java
@@ -323,6 +323,20 @@ public abstract class CustomDataQueryClusterIntegrationTest extends BaseClusterI
     return getSharedControllerStarter().getHelixResourceManager();
   }
 
+  /**
+   * Returns the Kafka broker list from the shared suite instance.
+   */
+  protected String getSharedKafkaBrokerList() {
+    return _sharedClusterTestSuite.getKafkaBrokerList();
+  }
+
+  /**
+   * Creates a Kafka topic on the shared suite's Kafka cluster.
+   */
+  protected void createSharedKafkaTopic(String topic, int numPartitions) {
+    _sharedClusterTestSuite.createKafkaTopic(topic, numPartitions);
+  }
+
   @Override
   public String getHelixClusterName() {
     return "CustomDataQueryClusterIntegrationTest";
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/MultiTopicRealtimeClusterIntegrationTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/MultiTopicRealtimeClusterIntegrationTest.java
new file mode 100644
index 00000000000..d2013f22399
--- /dev/null
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/MultiTopicRealtimeClusterIntegrationTest.java
@@ -0,0 +1,409 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.integration.tests.custom;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import java.io.File;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import org.apache.helix.model.IdealState;
+import org.apache.pinot.common.utils.LLCSegmentName;
+import org.apache.pinot.integration.tests.ClusterIntegrationTestUtils;
+import org.apache.pinot.plugin.inputformat.csv.CSVMessageDecoder;
+import org.apache.pinot.plugin.stream.kafka.KafkaStreamConfigProperties;
+import org.apache.pinot.spi.config.table.TableConfig;
+import org.apache.pinot.spi.config.table.TableType;
+import org.apache.pinot.spi.config.table.ingestion.IngestionConfig;
+import org.apache.pinot.spi.config.table.ingestion.StreamIngestionConfig;
+import org.apache.pinot.spi.data.FieldSpec;
+import org.apache.pinot.spi.data.Schema;
+import org.apache.pinot.spi.stream.StreamConfigProperties;
+import org.apache.pinot.spi.utils.CommonConstants;
+import org.apache.pinot.spi.utils.IngestionConfigUtils;
+import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
+import org.apache.pinot.spi.utils.builder.TableNameBuilder;
+import org.apache.pinot.util.TestUtils;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertTrue;
+
+
+/**
+ * Integration test for a single realtime table that ingests from N Kafka topics using
+ * the {@link StreamIngestionConfig#getStreamConfigMaps()} multi-stream configuration.
+ *
+ * <p>The number of topics is controlled by {@link #getNumTopics()}. For each topic index {@code i},
+ * a Kafka topic named {@code multiTopicTest_topic{i}} is created and populated with
+ * {@link #RECORDS_PER_TOPIC} CSV records whose {@code source} column is set to {@code TOPIC_{i}}
+ * and whose {@code value} column falls in the exclusive range {@code [i*100 + 1, i*100 + 100]}.
+ *
+ * <p>Queries then verify that every topic's data is present and correctly isolated.
+ * Subclasses can override {@link #getNumTopics()} to test with a different number of topics.
+ */
+public class MultiTopicRealtimeClusterIntegrationTest extends CustomDataQueryClusterIntegrationTest {
+  private static final String TABLE_NAME = "multiTopicTable";
+  private static final String TOPIC_PREFIX = "multiTopicTest_topic";
+  private static final int NUM_PARTITIONS_PER_TOPIC = 2;
+  private static final int RECORDS_PER_TOPIC = 500;
+  private static final int VALUE_RANGE_PER_TOPIC = 100;
+  private static final String CSV_DELIMITER = ",";
+  private static final String CSV_HEADER = "id,name,source,value,ts";
+
+  /**
+   * Returns the number of Kafka topics to create. Override in subclasses to test
+   * with a different topic count.
+   */
+  protected int getNumTopics() {
+    return 3;
+  }
+
+  private String topicName(int topicIndex) {
+    return TOPIC_PREFIX + topicIndex;
+  }
+
+  private String sourceName(int topicIndex) {
+    int maxIndex = getNumTopics() - 1;
+    int width = String.valueOf(maxIndex).length();
+    return String.format("TOPIC_%0" + width + "d", topicIndex);
+  }
+
+  private int valueRangeStart(int topicIndex) {
+    return topicIndex * VALUE_RANGE_PER_TOPIC + 1;
+  }
+
+  private int valueRangeEnd(int topicIndex) {
+    return (topicIndex + 1) * VALUE_RANGE_PER_TOPIC;
+  }
+
+  @Override
+  public String getTableName() {
+    return TABLE_NAME;
+  }
+
+  @Override
+  public String getKafkaTopic() {
+    return topicName(0);
+  }
+
+  @Override
+  protected int getNumKafkaPartitions() {
+    return NUM_PARTITIONS_PER_TOPIC;
+  }
+
+  @Override
+  protected long getCountStarResult() {
+    return (long) RECORDS_PER_TOPIC * getNumTopics();
+  }
+
+  @Override
+  public boolean isRealtimeTable() {
+    return true;
+  }
+
+  @Override
+  protected String getSortedColumn() {
+    return null;
+  }
+
+  @Override
+  protected List<String> getInvertedIndexColumns() {
+    return List.of("source");
+  }
+
+  @Override
+  public Schema createSchema() {
+    return new Schema.SchemaBuilder().setSchemaName(TABLE_NAME)
+        .addSingleValueDimension("id", FieldSpec.DataType.INT)
+        .addSingleValueDimension("name", FieldSpec.DataType.STRING)
+        .addSingleValueDimension("source", FieldSpec.DataType.STRING)
+        .addMetric("value", FieldSpec.DataType.INT)
+        .addDateTime("ts", FieldSpec.DataType.LONG, "1:MILLISECONDS:EPOCH", "1:MILLISECONDS")
+        .build();
+  }
+
+  @Override
+  public List<File> createAvroFiles() {
+    return List.of();
+  }
+
+  protected IngestionConfig getIngestionConfig() {
+    int numTopics = getNumTopics();
+    List<Map<String, String>> streamConfigMaps = new ArrayList<>(numTopics);
+    for (int i = 0; i < numTopics; i++) {
+      streamConfigMaps.add(buildStreamConfigMap(topicName(i)));
+    }
+    IngestionConfig ingestionConfig = new IngestionConfig();
+    ingestionConfig.setStreamIngestionConfig(new StreamIngestionConfig(streamConfigMaps));
+    return ingestionConfig;
+  }
+
+  private Map<String, String> buildStreamConfigMap(String topicName) {
+    String streamType = "kafka";
+    Map<String, String> map = new HashMap<>();
+    map.put(StreamConfigProperties.STREAM_TYPE, streamType);
+    map.put(KafkaStreamConfigProperties.constructStreamProperty(
+            KafkaStreamConfigProperties.LowLevelConsumer.KAFKA_BROKER_LIST),
+        getSharedKafkaBrokerList());
+    map.put(StreamConfigProperties.constructStreamProperty(streamType,
+        StreamConfigProperties.STREAM_CONSUMER_FACTORY_CLASS), getStreamConsumerFactoryClassName());
+    map.put(StreamConfigProperties.constructStreamProperty(streamType,
+        StreamConfigProperties.STREAM_TOPIC_NAME), topicName);
+    map.put(StreamConfigProperties.constructStreamProperty(streamType,
+        StreamConfigProperties.STREAM_DECODER_CLASS), CSVMessageDecoder.class.getName());
+    map.put(StreamConfigProperties.constructStreamProperty(streamType, "decoder.prop.delimiter"), CSV_DELIMITER);
+    map.put(StreamConfigProperties.constructStreamProperty(streamType, "decoder.prop.header"), CSV_HEADER);
+    map.put(StreamConfigProperties.SEGMENT_FLUSH_THRESHOLD_ROWS,
+        Integer.toString(getRealtimeSegmentFlushSize()));
+    map.put(StreamConfigProperties.constructStreamProperty(streamType,
+        StreamConfigProperties.STREAM_CONSUMER_OFFSET_CRITERIA), "smallest");
+    return map;
+  }
+
+  private List<String> generateRecords(int topicIndex) {
+    String source = sourceName(topicIndex);
+    int valStart = valueRangeStart(topicIndex);
+    List<String> records = new ArrayList<>(RECORDS_PER_TOPIC);
+    long baseTs = 1700000000000L;
+    for (int i = 0; i < RECORDS_PER_TOPIC; i++) {
+      int id = topicIndex * RECORDS_PER_TOPIC + i;
+      String name = "name_" + topicIndex + "_" + i;
+      int value = valStart + (i % VALUE_RANGE_PER_TOPIC);
+      long ts = baseTs + ((long) topicIndex * RECORDS_PER_TOPIC + i) * 1000L;
+      records.add(String.join(CSV_DELIMITER,
+          String.valueOf(id), name, source, String.valueOf(value), String.valueOf(ts)));
+    }
+    return records;
+  }
+
+  @Override
+  @BeforeClass
+  public void setUp()
+      throws Exception {
+    initControllerRequestURLBuilder();
+    TestUtils.ensureDirectoriesExistAndEmpty(_tempDir);
+
+    int numTopics = getNumTopics();
+    for (int i = 0; i < numTopics; i++) {
+      createSharedKafkaTopic(topicName(i), NUM_PARTITIONS_PER_TOPIC);
+    }
+
+    Schema schema = createSchema();
+    addSchema(schema);
+
+    TableConfig tableConfig = new TableConfigBuilder(TableType.REALTIME)
+        .setTableName(TABLE_NAME)
+        .setTimeColumnName(getTimeColumnName())
+        .setInvertedIndexColumns(getInvertedIndexColumns())
+        .setNumReplicas(getNumReplicas())
+        .setBrokerTenant(getBrokerTenant())
+        .setServerTenant(getServerTenant())
+        .setIngestionConfig(getIngestionConfig())
+        .setNullHandlingEnabled(getNullHandlingEnabled())
+        .build();
+    addTableConfig(tableConfig);
+
+    String realtimeTableName = TableNameBuilder.REALTIME.tableNameWithType(TABLE_NAME);
+    int totalExpectedPartitions = NUM_PARTITIONS_PER_TOPIC * numTopics;
+    TestUtils.waitForCondition(
+        () -> getNumConsumingPartitions(realtimeTableName) == totalExpectedPartitions,
+        200L, 120_000L,
+        "Failed to get CONSUMING segments for all " + totalExpectedPartitions + " partitions",
+        Duration.ofSeconds(12));
+
+    for (int i = 0; i < numTopics; i++) {
+      ClusterIntegrationTestUtils.pushCsvIntoKafka(generateRecords(i),
+          getSharedKafkaBrokerList(), topicName(i), null, injectTombstones());
+    }
+
+    waitForAllDocsLoaded(600_000L);
+  }
+
+  private int getNumConsumingPartitions(String tableNameWithType) {
+    IdealState idealState = getSharedHelixResourceManager().getTableIdealState(tableNameWithType);
+    if (idealState == null) {
+      return 0;
+    }
+    Set<Integer> consumingPartitions = new HashSet<>();
+    for (String segmentName : idealState.getPartitionSet()) {
+      if (!LLCSegmentName.isLLCSegment(segmentName)) {
+        continue;
+      }
+      Map<String, String> stateMap = idealState.getInstanceStateMap(segmentName);
+      if (stateMap != null
+          && stateMap.containsValue(CommonConstants.Helix.StateModel.SegmentStateModel.CONSUMING)) {
+        consumingPartitions.add(new LLCSegmentName(segmentName).getPartitionGroupId());
+      }
+    }
+    return consumingPartitions.size();
+  }
+
+  @Test
+  public void testTotalDocCount()
+      throws Exception {
+    JsonNode response = postQuery("SELECT COUNT(*) FROM " + TABLE_NAME);
+    assertEquals(response.get("resultTable").get("rows").get(0).get(0).asLong(),
+        (long) RECORDS_PER_TOPIC * getNumTopics(),
+        "Total doc count should equal RECORDS_PER_TOPIC * numTopics");
+  }
+
+  @Test
+  public void testGroupBySource()
+      throws Exception {
+    int numTopics = getNumTopics();
+    JsonNode response = postQuery(
+        "SELECT source, COUNT(*) AS cnt FROM " + TABLE_NAME + " GROUP BY source ORDER BY source");
+    JsonNode rows = response.get("resultTable").get("rows");
+    assertEquals(rows.size(), numTopics, "Should have exactly " + numTopics + " groups");
+
+    for (int i = 0; i < numTopics; i++) {
+      assertEquals(rows.get(i).get(0).asText(), sourceName(i),
+          "Group " + i + " source name mismatch");
+      assertEquals(rows.get(i).get(1).asLong(), RECORDS_PER_TOPIC,
+          "Group " + i + " count mismatch");
+    }
+  }
+
+  @Test
+  public void testFilterBySource()
+      throws Exception {
+    for (int i = 0; i < getNumTopics(); i++) {
+      String source = sourceName(i);
+      JsonNode response = postQuery(
+          "SELECT COUNT(*) FROM " + TABLE_NAME + " WHERE source = '" + source + "'");
+      assertEquals(response.get("resultTable").get("rows").get(0).get(0).asLong(),
+          (long) RECORDS_PER_TOPIC,
+          "Filter on " + source + " should return exactly RECORDS_PER_TOPIC rows");
+    }
+  }
+
+  @Test
+  public void testValueRangesBySource()
+      throws Exception {
+    for (int i = 0; i < getNumTopics(); i++) {
+      String source = sourceName(i);
+      int expectedMin = valueRangeStart(i);
+      int expectedMax = valueRangeEnd(i);
+      JsonNode response = postQuery(
+          "SELECT MIN(value), MAX(value) FROM " + TABLE_NAME + " WHERE source = '" + source + "'");
+      JsonNode row = response.get("resultTable").get("rows").get(0);
+      assertEquals(row.get(0).asInt(), expectedMin,
+          source + " min value should be " + expectedMin + " but was " + row.get(0).asInt());
+      assertEquals(row.get(1).asInt(), expectedMax,
+          source + " max value should be " + expectedMax + " but was " + row.get(1).asInt());
+    }
+  }
+
+  @Test
+  public void testAggregationBySource()
+      throws Exception {
+    int numTopics = getNumTopics();
+    JsonNode response = postQuery(
+        "SELECT source, SUM(value) AS total FROM " + TABLE_NAME + " GROUP BY source ORDER BY source");
+    JsonNode rows = response.get("resultTable").get("rows");
+    assertEquals(rows.size(), numTopics);
+
+    long previousSum = Long.MIN_VALUE;
+    for (int i = 0; i < numTopics; i++) {
+      long sum = rows.get(i).get(1).asLong();
+      assertTrue(sum > 0, sourceName(i) + " sum should be positive");
+      assertTrue(sum > previousSum,
+          sourceName(i) + " sum (" + sum + ") should be larger than previous topic sum (" + previousSum + ")");
+      previousSum = sum;
+    }
+  }
+
+  @Test
+  public void testCrossTopicQuery()
+      throws Exception {
+    int numTopics = getNumTopics();
+    String orClauses = java.util.stream.IntStream.range(0, numTopics)
+        .mapToObj(i -> "source = '" + sourceName(i) + "'")
+        .collect(Collectors.joining(" OR "));
+    JsonNode response = postQuery(
+        "SELECT COUNT(*) FROM " + TABLE_NAME + " WHERE " + orClauses);
+    assertEquals(response.get("resultTable").get("rows").get(0).get(0).asLong(),
+        (long) RECORDS_PER_TOPIC * numTopics,
+        "OR filter across all sources should return all records");
+  }
+
+  @Test
+  public void testNoDataLeakBetweenTopics()
+      throws Exception {
+    for (int i = 0; i < getNumTopics(); i++) {
+      String source = sourceName(i);
+      int maxAllowed = valueRangeEnd(i);
+      int minAllowed = valueRangeStart(i);
+
+      JsonNode aboveRange = postQuery(
+          "SELECT COUNT(*) FROM " + TABLE_NAME
+              + " WHERE source = '" + source + "' AND value > " + maxAllowed);
+      assertEquals(aboveRange.get("resultTable").get("rows").get(0).get(0).asLong(), 0L,
+          source + " should have no values > " + maxAllowed);
+
+      JsonNode belowRange = postQuery(
+          "SELECT COUNT(*) FROM " + TABLE_NAME
+              + " WHERE source = '" + source + "' AND value < " + minAllowed);
+      assertEquals(belowRange.get("resultTable").get("rows").get(0).get(0).asLong(), 0L,
+          source + " should have no values < " + minAllowed);
+    }
+  }
+
+  @Test
+  public void testSegmentsFromAllTopics() {
+    int numTopics = getNumTopics();
+    String realtimeTableName = TableNameBuilder.REALTIME.tableNameWithType(TABLE_NAME);
+    IdealState idealState = getSharedHelixResourceManager().getTableIdealState(realtimeTableName);
+    assertNotNull(idealState);
+
+    Set<Integer> topicIndicesSeen = new HashSet<>();
+    for (String segmentName : idealState.getPartitionSet()) {
+      if (LLCSegmentName.isLLCSegment(segmentName)) {
+        int pgId = new LLCSegmentName(segmentName).getPartitionGroupId();
+        topicIndicesSeen.add(IngestionConfigUtils.getStreamConfigIndexFromPinotPartitionId(pgId));
+      }
+    }
+    for (int i = 0; i < numTopics; i++) {
+      assertTrue(topicIndicesSeen.contains(i),
+          "Should have segments from topic " + i + " (stream config index " + i + ")");
+    }
+  }
+
+  @Test
+  public void testDistinctSources()
+      throws Exception {
+    int numTopics = getNumTopics();
+    JsonNode response = postQuery(
+        "SELECT DISTINCT source FROM " + TABLE_NAME + " ORDER BY source LIMIT " + (numTopics + 5));
+    JsonNode rows = response.get("resultTable").get("rows");
+    assertEquals(rows.size(), numTopics, "Should have exactly " + numTopics + " distinct source values");
+
+    for (int i = 0; i < numTopics; i++) {
+      assertEquals(rows.get(i).get(0).asText(), sourceName(i));
+    }
+  }
+}


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org