[pinot] branch master updated: Add support to create realtime segment in local (#10433)

[kh...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

kharekartik pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git


The following commit(s) were added to refs/heads/master by this push:
     new 16cae5d9c6 Add support to create realtime segment in local (#10433)
16cae5d9c6 is described below

commit 16cae5d9c630dd247fb037d4c0e7a18a8374178c
Author: Kartik Khare <kh...@gmail.com>
AuthorDate: Thu Mar 16 15:22:03 2023 +0530

    Add support to create realtime segment in local (#10433)
    
    Co-authored-by: Kartik Khare <kh...@Kartiks-MacBook-Pro.local>
---
 .../pinot/tools/admin/command/CreateSegmentCommand.java  | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/CreateSegmentCommand.java b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/CreateSegmentCommand.java
index 4263f4bbbe..48d27ab3ed 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/CreateSegmentCommand.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/CreateSegmentCommand.java
@@ -26,12 +26,14 @@ import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 import org.apache.commons.io.FileUtils;
+import org.apache.pinot.common.utils.LLCSegmentName;
 import org.apache.pinot.segment.local.indexsegment.immutable.ImmutableSegmentLoader;
 import org.apache.pinot.segment.local.segment.creator.impl.SegmentIndexCreationDriverImpl;
 import org.apache.pinot.segment.spi.ImmutableSegment;
 import org.apache.pinot.segment.spi.creator.SegmentGeneratorConfig;
 import org.apache.pinot.segment.spi.creator.SegmentIndexCreationDriver;
 import org.apache.pinot.spi.config.table.TableConfig;
+import org.apache.pinot.spi.config.table.TableType;
 import org.apache.pinot.spi.data.Schema;
 import org.apache.pinot.spi.data.readers.FileFormat;
 import org.apache.pinot.spi.data.readers.RecordReaderConfig;
@@ -82,6 +84,10 @@ public class CreateSegmentCommand extends AbstractBaseAdminCommand implements Co
       description = "Option to fail the segment creation if output is an empty segment.")
   private boolean _failOnEmptySegment = false;
 
+  @CommandLine.Option(names = {"-realtimePartitionId"},
+      description = "If table is realtime, partition id to be used for segment name generation. Default is 0.")
+  private int _realtimePartitionId = 0;
+
   @CommandLine.Option(names = {"-postCreationVerification"},
       description = "Verify segment data file after segment creation. Please ensure you have enough local disk to"
           + " hold data for verification")
@@ -139,6 +145,11 @@ public class CreateSegmentCommand extends AbstractBaseAdminCommand implements Co
     return this;
   }
 
+  public CreateSegmentCommand setRealtimePartitionId(int realtimePartitionId) {
+    _realtimePartitionId = realtimePartitionId;
+    return this;
+  }
+
   public CreateSegmentCommand setPostCreationVerification(boolean postCreationVerification) {
     _postCreationVerification = postCreationVerification;
     return this;
@@ -209,6 +220,7 @@ public class CreateSegmentCommand extends AbstractBaseAdminCommand implements Co
     }
     LOGGER.info("Using table config: {}", tableConfig.toJsonString());
     String rawTableName = TableNameBuilder.extractRawTableName(tableConfig.getTableName());
+    TableType tableType = tableConfig.getTableType();
 
     Preconditions.checkArgument(_schemaFile != null, "'schemaFile' must be specified");
     Schema schema;
@@ -246,6 +258,10 @@ public class CreateSegmentCommand extends AbstractBaseAdminCommand implements Co
         segmentGeneratorConfig.setReaderConfig(recordReaderConfig);
         segmentGeneratorConfig.setTableName(rawTableName);
         segmentGeneratorConfig.setSequenceId(sequenceId);
+        if (tableType == TableType.REALTIME) {
+          segmentGeneratorConfig.setSegmentName(new LLCSegmentName(rawTableName, _realtimePartitionId, sequenceId,
+              System.currentTimeMillis()).getSegmentName());
+        }
         segmentGeneratorConfig.setFailOnEmptySegment(_failOnEmptySegment);
         for (int j = 0; j <= _retry; j++) {
           try {


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org