You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ja...@apache.org on 2020/03/18 18:04:17 UTC

[incubator-pinot] 02/05: apply pinot core changes

This is an automated email from the ASF dual-hosted git repository.

jamesshao pushed a commit to branch upsert-refactor
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git

commit 8d2c875e1344f0b61960c0879dd9d1e3c34341c4
Author: james Shao <sj...@uber.com>
AuthorDate: Mon Feb 24 15:47:11 2020 -0800

    apply pinot core changes
---
 pinot-core/pom.xml                                 |   9 +
 .../core/data/manager/BaseTableDataManager.java    |   6 +-
 .../core/data/manager/InstanceDataManager.java     |   6 +
 .../manager/UpsertSegmentDataManager.java}         |  35 +--
 .../manager/config/TableDataManagerConfig.java     |  10 +-
 .../offline/ImmutableSegmentDataManager.java       |   2 +-
 .../manager/offline/TableDataManagerProvider.java  |  18 +-
 .../offline/UpsertImmutableSegmentDataManager.java |  67 ++++
 .../AppendLLRealtimeSegmentDataManager.java        |  58 ++++
 .../realtime/AppendRealtimeTableDataManager.java   |  55 ++++
 .../realtime/HLRealtimeSegmentDataManager.java     |  10 +-
 .../realtime/LLRealtimeSegmentDataManager.java     |  50 +--
 .../manager/realtime/RealtimeTableDataManager.java |  36 ++-
 .../UpsertLLRealtimeSegmentDataManager.java        | 153 +++++++++
 .../realtime/UpsertRealtimeTableDataManager.java   |  82 +++++
 .../UpsertSegment.java}                            |  35 +--
 .../immutable/ImmutableSegmentImpl.java            |   8 +-
 .../immutable/ImmutableSegmentLoader.java          |  27 +-
 .../immutable/ImmutableUpsertSegmentImpl.java      | 251 +++++++++++++++
 .../mutable/MutableAppendSegmentImpl.java}         |  31 +-
 .../indexsegment/mutable/MutableSegmentImpl.java   |  55 +++-
 .../mutable/MutableUpsertSegmentImpl.java          | 159 ++++++++++
 .../core/realtime/impl/RealtimeSegmentConfig.java  |  16 +-
 .../core/segment/index/SegmentMetadataImpl.java    |  29 +-
 .../SegmentDeletionListener.java}                  |  27 +-
 .../pinot/core/segment/updater/SegmentUpdater.java | 349 +++++++++++++++++++++
 .../SegmentUpdaterConfig.java}                     |  29 +-
 .../segment/updater/UpsertWaterMarkManager.java    |  90 ++++++
 .../virtualcolumn/BaseVirtualColumnProvider.java   |  18 ++
 .../virtualcolumn/VirtualColumnContext.java        |   8 +-
 .../virtualcolumn/VirtualColumnProvider.java       |   4 +
 .../mutable/BaseLongVirtualColumnProvider.java     |  57 ++++
 .../BaseVirtualColumnSingleValueReaderWriter.java} |  27 +-
 .../ValidFromInMemoryVirtualColumnProvider.java}   |  39 +--
 .../ValidUntilInMemoryVirtualColumnProvider.java}  |  39 +--
 .../VirtualColumnLongValueReaderWriter.java        | 108 +++++++
 .../realtime/LLRealtimeSegmentDataManagerTest.java |   2 +-
 .../immutable/ImmutableUpsertSegmentImplTest.java  | 127 ++++++++
 .../mutable/MutableSegmentImplTestUtils.java       |   2 +-
 ...erSegmentAggregationSingleValueQueriesTest.java |   5 +-
 .../starter/helix/HelixInstanceDataManager.java    |  11 +-
 .../realtime/provisioning/MemoryEstimator.java     |   5 +-
 42 files changed, 1909 insertions(+), 246 deletions(-)

diff --git a/pinot-core/pom.xml b/pinot-core/pom.xml
index c7179cf..74528c9 100644
--- a/pinot-core/pom.xml
+++ b/pinot-core/pom.xml
@@ -69,6 +69,15 @@
       <artifactId>pinot-common</artifactId>
     </dependency>
     <dependency>
+      <groupId>org.apache.pinot</groupId>
+      <artifactId>pinot-grigio-common</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.testng</groupId>
+      <artifactId>testng</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
       <groupId>joda-time</groupId>
       <artifactId>joda-time</artifactId>
     </dependency>
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/BaseTableDataManager.java b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/BaseTableDataManager.java
index e92d85f..a39bdfc 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/BaseTableDataManager.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/BaseTableDataManager.java
@@ -115,7 +115,7 @@ public abstract class BaseTableDataManager implements TableDataManager {
         immutableSegment.getSegmentMetadata().getTotalRawDocs());
     _serverMetrics.addValueToTableGauge(_tableNameWithType, ServerGauge.SEGMENT_COUNT, 1L);
 
-    ImmutableSegmentDataManager newSegmentManager = new ImmutableSegmentDataManager(immutableSegment);
+    ImmutableSegmentDataManager newSegmentManager = getImmutableSegmentDataManager(immutableSegment);
     SegmentDataManager oldSegmentManager = _segmentDataManagerMap.put(segmentName, newSegmentManager);
     if (oldSegmentManager == null) {
       _logger.info("Added new immutable segment: {} to table: {}", segmentName, _tableNameWithType);
@@ -211,4 +211,8 @@ public abstract class BaseTableDataManager implements TableDataManager {
   public String getTableName() {
     return _tableNameWithType;
   }
+
+  protected ImmutableSegmentDataManager getImmutableSegmentDataManager(ImmutableSegment immutableSegment) {
+    return new ImmutableSegmentDataManager(immutableSegment);
+  }
 }
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/InstanceDataManager.java b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/InstanceDataManager.java
index 4da0c07..175f5a7 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/InstanceDataManager.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/InstanceDataManager.java
@@ -20,6 +20,7 @@ package org.apache.pinot.core.data.manager;
 
 import java.io.File;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 import javax.annotation.Nullable;
 import javax.annotation.concurrent.ThreadSafe;
@@ -130,4 +131,9 @@ public interface InstanceDataManager {
    * Returns the Helix property store.
    */
   ZkHelixPropertyStore<ZNRecord> getPropertyStore();
+
+  /**
+   * Return the mappings from partition -> low water marks of all the tables hosted in this server.
+   */
+  Map<String, Map<Integer, Long>> getLowWaterMarks();
 }
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/VirtualColumnContext.java b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/UpsertSegmentDataManager.java
similarity index 53%
copy from pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/VirtualColumnContext.java
copy to pinot-core/src/main/java/org/apache/pinot/core/data/manager/UpsertSegmentDataManager.java
index bb32320..4f589f6 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/VirtualColumnContext.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/UpsertSegmentDataManager.java
@@ -16,29 +16,24 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.pinot.core.segment.virtualcolumn;
+package org.apache.pinot.core.data.manager;
 
-import org.apache.pinot.spi.data.FieldSpec;
+import org.apache.pinot.grigio.common.storageProvider.UpdateLogEntry;
 
+import java.util.List;
 
-/**
- * Miscellaneous context information about the virtual column.
- * It will be used to build various components (dictionary, reader, etc) in the virtual column provider.
- */
-public class VirtualColumnContext {
-  private FieldSpec _fieldSpec;
-  private int _totalDocCount;
-
-  public VirtualColumnContext(FieldSpec fieldSpec, int totalDocCount) {
-    _fieldSpec = fieldSpec;
-    _totalDocCount = totalDocCount;
-  }
+public interface UpsertSegmentDataManager {
 
-  public FieldSpec getFieldSpec() {
-    return _fieldSpec;
-  }
+  /**
+   * update the upsert-related virtual columns with the new values in this list of update logs
+   * @param messages list of updates logs to update the virtual columns
+   */
+  void updateVirtualColumns(List<UpdateLogEntry> messages);
 
-  public int getTotalDocCount() {
-    return _totalDocCount;
-  }
+  /**
+   * get the upsert related virtual column debug info given an offset
+   * @param offset the offset we want to look up the virtual column info from
+   * @return debug info describing the upsert-related virtual column info
+   */
+  String getVirtualColumnInfo(long offset);
 }
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/config/TableDataManagerConfig.java b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/config/TableDataManagerConfig.java
index 1c01ef3..3bf51df 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/config/TableDataManagerConfig.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/config/TableDataManagerConfig.java
@@ -19,13 +19,15 @@
 package org.apache.pinot.core.data.manager.config;
 
 import com.google.common.base.Preconditions;
-import javax.annotation.Nonnull;
 import org.apache.commons.configuration.Configuration;
 import org.apache.commons.configuration.PropertiesConfiguration;
 import org.apache.pinot.common.config.TableConfig;
 import org.apache.pinot.common.config.TableNameBuilder;
+import org.apache.pinot.common.utils.CommonConstants;
 import org.apache.pinot.common.utils.CommonConstants.Helix.TableType;
 
+import javax.annotation.Nonnull;
+
 
 /**
  * The config used for TableDataManager.
@@ -35,6 +37,7 @@ public class TableDataManagerConfig {
   private static final String TABLE_DATA_MANAGER_DATA_DIRECTORY = "directory";
   private static final String TABLE_DATA_MANAGER_CONSUMER_DIRECTORY = "consumerDirectory";
   private static final String TABLE_DATA_MANAGER_NAME = "name";
+  private static final String TABLE_UPDATE_SEMANTIC = "updateSemantic";
 
   private final Configuration _tableDataManagerConfig;
 
@@ -62,6 +65,10 @@ public class TableDataManagerConfig {
     return _tableDataManagerConfig.getString(TABLE_DATA_MANAGER_NAME);
   }
 
+  public CommonConstants.UpdateSemantic getUpdateSemantic() {
+    return CommonConstants.UpdateSemantic.getUpdateSemantic(_tableDataManagerConfig.getString(TABLE_UPDATE_SEMANTIC));
+  }
+
   public static TableDataManagerConfig getDefaultHelixTableDataManagerConfig(
       @Nonnull InstanceDataManagerConfig instanceDataManagerConfig, @Nonnull String tableNameWithType) {
     Configuration defaultConfig = new PropertiesConfiguration();
@@ -83,5 +90,6 @@ public class TableDataManagerConfig {
     // If we wish to override some table level configs using table config, override them here
     // Note: the configs in TableDataManagerConfig is immutable once the table is created, which mean it will not pick
     // up the latest table config
+    _tableDataManagerConfig.setProperty(TABLE_UPDATE_SEMANTIC, tableConfig.getUpdateSemantic().toString());
   }
 }
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/offline/ImmutableSegmentDataManager.java b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/offline/ImmutableSegmentDataManager.java
index 5ffe55f..0401f61 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/offline/ImmutableSegmentDataManager.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/offline/ImmutableSegmentDataManager.java
@@ -27,7 +27,7 @@ import org.apache.pinot.core.indexsegment.immutable.ImmutableSegment;
  */
 public class ImmutableSegmentDataManager extends SegmentDataManager {
 
-  private final ImmutableSegment _immutableSegment;
+  protected final ImmutableSegment _immutableSegment;
 
   public ImmutableSegmentDataManager(ImmutableSegment immutableSegment) {
     _immutableSegment = immutableSegment;
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/offline/TableDataManagerProvider.java b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/offline/TableDataManagerProvider.java
index aa36055..3ec1bbd 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/offline/TableDataManagerProvider.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/offline/TableDataManagerProvider.java
@@ -18,8 +18,6 @@
  */
 package org.apache.pinot.core.data.manager.offline;
 
-import java.util.concurrent.Semaphore;
-import javax.annotation.Nonnull;
 import org.apache.helix.ZNRecord;
 import org.apache.helix.store.zk.ZkHelixPropertyStore;
 import org.apache.pinot.common.metrics.ServerMetrics;
@@ -27,13 +25,21 @@ import org.apache.pinot.common.utils.CommonConstants;
 import org.apache.pinot.core.data.manager.TableDataManager;
 import org.apache.pinot.core.data.manager.config.InstanceDataManagerConfig;
 import org.apache.pinot.core.data.manager.config.TableDataManagerConfig;
-import org.apache.pinot.core.data.manager.realtime.RealtimeTableDataManager;
+import org.apache.pinot.core.data.manager.realtime.AppendRealtimeTableDataManager;
+import org.apache.pinot.core.data.manager.realtime.UpsertRealtimeTableDataManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.Nonnull;
+import java.util.concurrent.Semaphore;
 
 
 /**
  * Factory for {@link TableDataManager}.
  */
 public class TableDataManagerProvider {
+  private static final Logger LOGGER = LoggerFactory.getLogger(TableDataManagerProvider.class);
+
   private static Semaphore _segmentBuildSemaphore;
 
   private TableDataManagerProvider() {
@@ -55,7 +61,11 @@ public class TableDataManagerProvider {
         tableDataManager = new OfflineTableDataManager();
         break;
       case REALTIME:
-        tableDataManager = new RealtimeTableDataManager(_segmentBuildSemaphore);
+        if (tableDataManagerConfig.getUpdateSemantic() == CommonConstants.UpdateSemantic.UPSERT) {
+          tableDataManager = new UpsertRealtimeTableDataManager(_segmentBuildSemaphore);
+        } else {
+          tableDataManager = new AppendRealtimeTableDataManager(_segmentBuildSemaphore);
+        }
         break;
       default:
         throw new IllegalStateException();
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/offline/UpsertImmutableSegmentDataManager.java b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/offline/UpsertImmutableSegmentDataManager.java
new file mode 100644
index 0000000..790e912
--- /dev/null
+++ b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/offline/UpsertImmutableSegmentDataManager.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.core.data.manager.offline;
+
+import org.apache.pinot.common.config.TableNameBuilder;
+import org.apache.pinot.common.utils.CommonConstants;
+import org.apache.pinot.common.utils.LLCSegmentName;
+import org.apache.pinot.core.data.manager.UpsertSegmentDataManager;
+import org.apache.pinot.core.indexsegment.UpsertSegment;
+import org.apache.pinot.core.indexsegment.immutable.ImmutableSegment;
+import org.apache.pinot.core.segment.updater.SegmentUpdater;
+import org.apache.pinot.grigio.common.storageProvider.UpdateLogEntry;
+
+import java.io.IOException;
+import java.util.List;
+
+public class UpsertImmutableSegmentDataManager extends ImmutableSegmentDataManager implements UpsertSegmentDataManager {
+
+  private String _tableNameWithType;
+
+  public UpsertImmutableSegmentDataManager(ImmutableSegment immutableSegment) throws IOException {
+    super(immutableSegment);
+    _tableNameWithType = TableNameBuilder.ensureTableNameWithType(immutableSegment.getSegmentMetadata().getTableName(),
+        CommonConstants.Helix.TableType.REALTIME);
+    initVirtualColumns();
+  }
+
+  @Override
+  public void updateVirtualColumns(List<UpdateLogEntry> messages) {
+    ((UpsertSegment) _immutableSegment).updateVirtualColumn(messages);
+  }
+
+  @Override
+  public String getVirtualColumnInfo(long offset) {
+    return ((UpsertSegment) _immutableSegment).getVirtualColumnInfo(offset);
+  }
+
+  @Override
+  public void destroy() {
+    SegmentUpdater.getInstance().removeSegmentDataManager(_tableNameWithType, getSegmentName(), this);
+    super.destroy();
+  }
+
+  private void initVirtualColumns() throws IOException {
+    // 1. add listener for update events
+    // 2. load all existing messages
+    // ensure the above orders so we can ensure all events are received by this data manager
+    SegmentUpdater.getInstance().addSegmentDataManager(_tableNameWithType, new LLCSegmentName(getSegmentName()), this);
+    ((UpsertSegment) _immutableSegment).initVirtualColumn();
+  }
+}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/AppendLLRealtimeSegmentDataManager.java b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/AppendLLRealtimeSegmentDataManager.java
new file mode 100644
index 0000000..f725cc8
--- /dev/null
+++ b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/AppendLLRealtimeSegmentDataManager.java
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.core.data.manager.realtime;
+
+import org.apache.pinot.common.config.TableConfig;
+import org.apache.pinot.common.metadata.segment.RealtimeSegmentZKMetadata;
+import org.apache.pinot.common.metrics.ServerMetrics;
+import org.apache.pinot.common.utils.LLCSegmentName;
+import org.apache.pinot.core.indexsegment.mutable.MutableAppendSegmentImpl;
+import org.apache.pinot.core.indexsegment.mutable.MutableSegmentImpl;
+import org.apache.pinot.core.realtime.impl.RealtimeSegmentConfig;
+import org.apache.pinot.core.segment.index.loader.IndexLoadingConfig;
+import org.apache.pinot.spi.data.Schema;
+import org.apache.pinot.spi.data.readers.GenericRow;
+
+import java.util.concurrent.Semaphore;
+
+public class AppendLLRealtimeSegmentDataManager extends LLRealtimeSegmentDataManager {
+
+  public AppendLLRealtimeSegmentDataManager(RealtimeSegmentZKMetadata segmentZKMetadata, TableConfig tableConfig,
+      RealtimeTableDataManager realtimeTableDataManager, String resourceDataDir, IndexLoadingConfig indexLoadingConfig,
+      Schema schema, LLCSegmentName llcSegmentName, Semaphore partitionConsumerSemaphore, ServerMetrics serverMetrics) {
+    super(segmentZKMetadata, tableConfig, realtimeTableDataManager, resourceDataDir, indexLoadingConfig, schema,
+            llcSegmentName, partitionConsumerSemaphore, serverMetrics);
+  }
+
+  @Override
+  protected MutableSegmentImpl createMutableSegment(RealtimeSegmentConfig config) {
+    return new MutableAppendSegmentImpl(config);
+  }
+
+  @Override
+  protected void processTransformedRow(GenericRow row, long offset) {
+    // do nothing
+  }
+
+  @Override
+  protected void postIndexProcessing(GenericRow row, long offset) {
+    // do nothing
+  }
+
+}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/AppendRealtimeTableDataManager.java b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/AppendRealtimeTableDataManager.java
new file mode 100644
index 0000000..741fb08
--- /dev/null
+++ b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/AppendRealtimeTableDataManager.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.core.data.manager.realtime;
+
+import org.apache.pinot.common.config.TableConfig;
+import org.apache.pinot.common.metadata.instance.InstanceZKMetadata;
+import org.apache.pinot.common.metadata.segment.RealtimeSegmentZKMetadata;
+import org.apache.pinot.common.metrics.ServerMetrics;
+import org.apache.pinot.common.utils.LLCSegmentName;
+import org.apache.pinot.core.indexsegment.immutable.ImmutableSegment;
+import org.apache.pinot.core.indexsegment.immutable.ImmutableSegmentLoader;
+import org.apache.pinot.core.segment.index.loader.IndexLoadingConfig;
+import org.apache.pinot.spi.data.Schema;
+
+import java.io.File;
+import java.util.concurrent.Semaphore;
+
+public class AppendRealtimeTableDataManager extends RealtimeTableDataManager {
+
+  public AppendRealtimeTableDataManager(Semaphore segmentBuildSemaphore) {
+    super(segmentBuildSemaphore);
+  }
+
+  @Override
+  protected LLRealtimeSegmentDataManager getLLRealtimeSegmentDataManager(
+          RealtimeSegmentZKMetadata realtimeSegmentZKMetadata, TableConfig tableConfig,
+          RealtimeTableDataManager realtimeTableDataManager, String indexDirPath, IndexLoadingConfig indexLoadingConfig,
+          Schema schema, LLCSegmentName llcSegmentName, Semaphore partitionConsumerSemaphore, ServerMetrics serverMetrics)
+      throws Exception {
+    return new AppendLLRealtimeSegmentDataManager(realtimeSegmentZKMetadata, tableConfig, realtimeTableDataManager,
+            indexDirPath, indexLoadingConfig, schema, llcSegmentName, partitionConsumerSemaphore, serverMetrics);
+  }
+
+  @Override
+  protected ImmutableSegment loadImmutableSegment(File indexDir, IndexLoadingConfig indexLoadingConfig, Schema schema)
+      throws Exception {
+    return ImmutableSegmentLoader.load(indexDir, indexLoadingConfig, schema);
+  }
+}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/HLRealtimeSegmentDataManager.java b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/HLRealtimeSegmentDataManager.java
index 421fa3f..086812e 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/HLRealtimeSegmentDataManager.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/HLRealtimeSegmentDataManager.java
@@ -30,6 +30,7 @@ import java.util.concurrent.atomic.AtomicLong;
 import org.apache.commons.io.FileUtils;
 import org.apache.pinot.common.config.IndexingConfig;
 import org.apache.pinot.common.config.TableConfig;
+import org.apache.pinot.core.indexsegment.mutable.MutableAppendSegmentImpl;
 import org.apache.pinot.spi.data.Schema;
 import org.apache.pinot.common.metadata.instance.InstanceZKMetadata;
 import org.apache.pinot.common.metadata.segment.RealtimeSegmentZKMetadata;
@@ -101,9 +102,9 @@ public class HLRealtimeSegmentDataManager extends RealtimeSegmentDataManager {
   // An instance of this class exists only for the duration of the realtime segment that is currently being consumed.
   // Once the segment is committed, the segment is handled by OfflineSegmentDataManager
   public HLRealtimeSegmentDataManager(final RealtimeSegmentZKMetadata realtimeSegmentZKMetadata,
-      final TableConfig tableConfig, InstanceZKMetadata instanceMetadata,
-      final RealtimeTableDataManager realtimeTableDataManager, final String resourceDataDir,
-      final IndexLoadingConfig indexLoadingConfig, final Schema schema, final ServerMetrics serverMetrics)
+                                      final TableConfig tableConfig, InstanceZKMetadata instanceMetadata,
+                                      final RealtimeTableDataManager realtimeTableDataManager, final String resourceDataDir,
+                                      final IndexLoadingConfig indexLoadingConfig, final Schema schema, final ServerMetrics serverMetrics)
       throws Exception {
     super();
     _segmentVersion = indexLoadingConfig.getSegmentVersion();
@@ -194,7 +195,8 @@ public class HLRealtimeSegmentDataManager extends RealtimeSegmentDataManager {
                 indexLoadingConfig.isDirectRealtimeOffheapAllocation(), serverMetrics))
             .setStatsHistory(realtimeTableDataManager.getStatsHistory())
             .setNullHandlingEnabled(indexingConfig.isNullHandlingEnabled()).build();
-    realtimeSegment = new MutableSegmentImpl(realtimeSegmentConfig);
+    realtimeSegment = new MutableAppendSegmentImpl(realtimeSegmentConfig);
+
 
     notifier = realtimeTableDataManager;
 
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/LLRealtimeSegmentDataManager.java b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/LLRealtimeSegmentDataManager.java
index ab04b74..7765e9e 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/LLRealtimeSegmentDataManager.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/LLRealtimeSegmentDataManager.java
@@ -83,7 +83,7 @@ import org.slf4j.LoggerFactory;
 /**
  * Segment data manager for low level consumer realtime segments, which manages consumption and segment completion.
  */
-public class LLRealtimeSegmentDataManager extends RealtimeSegmentDataManager {
+public abstract class LLRealtimeSegmentDataManager extends RealtimeSegmentDataManager {
   protected enum State {
     // The state machine starts off with this state. While in this state we consume stream events
     // and index them in memory. We continue to be in this state until the end criteria is satisfied
@@ -202,7 +202,7 @@ public class LLRealtimeSegmentDataManager extends RealtimeSegmentDataManager {
   private final int _segmentMaxRowCount;
   private final String _resourceDataDir;
   private final IndexLoadingConfig _indexLoadingConfig;
-  private final Schema _schema;
+  protected final Schema _schema;
   // Semaphore for each partitionId only, which is to prevent two different Kafka consumers
   // from consuming with the same partitionId in parallel in the same host.
   // See the comments in {@link RealtimeTableDataManager}.
@@ -212,21 +212,22 @@ public class LLRealtimeSegmentDataManager extends RealtimeSegmentDataManager {
   // modify the permit. This boolean make sure the semaphore gets released only once when the partition stops consuming.
   private final AtomicBoolean _acquiredConsumerSemaphore;
   private final String _metricKeyName;
-  private final ServerMetrics _serverMetrics;
-  private final MutableSegmentImpl _realtimeSegment;
+  protected final ServerMetrics _serverMetrics;
+  protected final MutableSegmentImpl _realtimeSegment;
   private volatile long _currentOffset;
-  private volatile State _state;
+  protected volatile State _state;
   private volatile int _numRowsConsumed = 0;
   private volatile int _numRowsIndexed = 0; // Can be different from _numRowsConsumed when metrics update is enabled.
   private volatile int _numRowsErrored = 0;
   private volatile int consecutiveErrorCount = 0;
   private long _startTimeMs = 0;
-  private final String _segmentNameStr;
   private final SegmentVersion _segmentVersion;
   private final SegmentBuildTimeLeaseExtender _leaseExtender;
   private SegmentBuildDescriptor _segmentBuildDescriptor;
   private StreamConsumerFactory _streamConsumerFactory;
 
+  protected final String _segmentNameStr;
+
   // Segment end criteria
   private volatile long _consumeEndTime = 0;
   private volatile long _finalOffset = -1;
@@ -238,14 +239,14 @@ public class LLRealtimeSegmentDataManager extends RealtimeSegmentDataManager {
 
   private Thread _consumerThread;
   private final String _streamTopic;
-  private final int _streamPartitionId;
+  protected final int _streamPartitionId;
   final String _clientId;
-  private final LLCSegmentName _llcSegmentName;
+  protected final LLCSegmentName _llcSegmentName;
   private final RecordTransformer _recordTransformer;
   private PartitionLevelConsumer _partitionLevelConsumer = null;
   private StreamMetadataProvider _streamMetadataProvider = null;
   private final File _resourceTmpDir;
-  private final String _tableNameWithType;
+  protected final String _tableNameWithType;
   private final String _timeColumnName;
   private final List<String> _invertedIndexColumns;
   private final List<String> _textIndexColumns;
@@ -253,7 +254,7 @@ public class LLRealtimeSegmentDataManager extends RealtimeSegmentDataManager {
   private final List<String> _varLengthDictionaryColumns;
   private final StarTreeIndexSpec _starTreeIndexSpec;
   private final String _sortedColumn;
-  private Logger segmentLogger;
+  protected Logger segmentLogger;
   private final String _tableStreamName;
   private final PinotDataBufferMemoryManager _memoryManager;
   private AtomicLong _lastUpdatedRowsIndexed = new AtomicLong(0);
@@ -466,16 +467,19 @@ public class LLRealtimeSegmentDataManager extends RealtimeSegmentDataManager {
           GenericRow transformedRow = _recordTransformer.transform(decodedRow);
 
           if (transformedRow != null) {
-            realtimeRowsConsumedMeter = _serverMetrics
-                .addMeteredTableValue(_metricKeyName, ServerMeter.REALTIME_ROWS_CONSUMED, 1, realtimeRowsConsumedMeter);
+            processTransformedRow(transformedRow, _currentOffset);
+            realtimeRowsConsumedMeter =
+                _serverMetrics.addMeteredTableValue(_metricKeyName, ServerMeter.REALTIME_ROWS_CONSUMED, 1,
+                    realtimeRowsConsumedMeter);
             indexedMessageCount++;
+
+            canTakeMore = _realtimeSegment.index(transformedRow, msgMetadata);
+            postIndexProcessing(transformedRow, _currentOffset);
           } else {
             realtimeRowsDroppedMeter = _serverMetrics
                 .addMeteredTableValue(_metricKeyName, ServerMeter.INVALID_REALTIME_ROWS_DROPPED, 1,
                     realtimeRowsDroppedMeter);
           }
-
-          canTakeMore = _realtimeSegment.index(transformedRow, msgMetadata);
         } catch (Exception e) {
           segmentLogger.error("Caught exception while transforming the record: {}", decodedRow, e);
           _numRowsErrored++;
@@ -501,6 +505,11 @@ public class LLRealtimeSegmentDataManager extends RealtimeSegmentDataManager {
     }
   }
 
+  protected abstract void processTransformedRow(GenericRow row, long offset);
+
+
+  protected abstract void postIndexProcessing(GenericRow row, long offset);
+
   public class PartitionConsumer implements Runnable {
     public void run() {
       long initialConsumptionEnd = 0L;
@@ -795,7 +804,7 @@ public class LLRealtimeSegmentDataManager extends RealtimeSegmentDataManager {
       return false;
     }
 
-    _realtimeTableDataManager.replaceLLSegment(_segmentNameStr, _indexLoadingConfig);
+    _realtimeTableDataManager.replaceLLSegment(_segmentNameStr, _indexLoadingConfig, _schema);
     removeSegmentFile();
     return true;
   }
@@ -828,7 +837,7 @@ public class LLRealtimeSegmentDataManager extends RealtimeSegmentDataManager {
       return false;
     }
 
-    _realtimeTableDataManager.replaceLLSegment(_segmentNameStr, _indexLoadingConfig);
+    _realtimeTableDataManager.replaceLLSegment(_segmentNameStr, _indexLoadingConfig, _schema);
     return true;
   }
 
@@ -977,7 +986,7 @@ public class LLRealtimeSegmentDataManager extends RealtimeSegmentDataManager {
 
   protected void downloadSegmentAndReplace(LLCRealtimeSegmentZKMetadata metadata) {
     closeKafkaConsumers();
-    _realtimeTableDataManager.downloadAndReplaceSegment(_segmentNameStr, metadata, _indexLoadingConfig);
+    _realtimeTableDataManager.downloadAndReplaceSegment(_segmentNameStr, metadata, _indexLoadingConfig, _schema);
   }
 
   protected long now() {
@@ -1132,7 +1141,8 @@ public class LLRealtimeSegmentDataManager extends RealtimeSegmentDataManager {
     // Start new realtime segment
     String consumerDir = realtimeTableDataManager.getConsumerDir();
     RealtimeSegmentConfig.Builder realtimeSegmentConfigBuilder =
-        new RealtimeSegmentConfig.Builder().setSegmentName(_segmentNameStr).setStreamName(_streamTopic)
+        new RealtimeSegmentConfig.Builder().setTableName(_tableNameWithType)
+            .setSegmentName(_segmentNameStr).setStreamName(_streamTopic)
             .setSchema(_schema).setCapacity(_segmentMaxRowCount)
             .setAvgNumMultiValues(indexLoadingConfig.getRealtimeAvgMultiValueCount())
             .setNoDictionaryColumns(indexLoadingConfig.getNoDictionaryColumns())
@@ -1175,7 +1185,7 @@ public class LLRealtimeSegmentDataManager extends RealtimeSegmentDataManager {
       }
     }
 
-    _realtimeSegment = new MutableSegmentImpl(realtimeSegmentConfigBuilder.build());
+    _realtimeSegment = createMutableSegment(realtimeSegmentConfigBuilder.build());
     _startOffset = _segmentZKMetadata.getStartOffset();
     _currentOffset = _startOffset;
     _resourceTmpDir = new File(resourceDataDir, "_tmp");
@@ -1211,6 +1221,8 @@ public class LLRealtimeSegmentDataManager extends RealtimeSegmentDataManager {
     start();
   }
 
+  protected abstract MutableSegmentImpl createMutableSegment(RealtimeSegmentConfig config);
+
   /**
    * Creates a new stream consumer
    * @param reason
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/RealtimeTableDataManager.java b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/RealtimeTableDataManager.java
index 94c9614..2be3ca1 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/RealtimeTableDataManager.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/RealtimeTableDataManager.java
@@ -39,6 +39,7 @@ import org.apache.pinot.common.metadata.ZKMetadataProvider;
 import org.apache.pinot.common.metadata.instance.InstanceZKMetadata;
 import org.apache.pinot.common.metadata.segment.LLCRealtimeSegmentZKMetadata;
 import org.apache.pinot.common.metadata.segment.RealtimeSegmentZKMetadata;
+import org.apache.pinot.common.metrics.ServerMetrics;
 import org.apache.pinot.common.utils.CommonConstants.Segment.Realtime.Status;
 import org.apache.pinot.common.utils.LLCSegmentName;
 import org.apache.pinot.common.utils.NamedThreadFactory;
@@ -54,10 +55,11 @@ import org.apache.pinot.core.segment.index.loader.LoaderUtils;
 import org.apache.pinot.core.segment.virtualcolumn.VirtualColumnProviderFactory;
 import org.apache.pinot.spi.data.FieldSpec;
 import org.apache.pinot.spi.data.Schema;
+import org.apache.pinot.core.indexsegment.immutable.ImmutableSegment;
 
 
 @ThreadSafe
-public class RealtimeTableDataManager extends BaseTableDataManager {
+public abstract class RealtimeTableDataManager extends BaseTableDataManager {
   private final ExecutorService _segmentAsyncExecutorService =
       Executors.newSingleThreadExecutor(new NamedThreadFactory("SegmentAsyncExecutorService"));
   private SegmentBuildTimeLeaseExtender _leaseExtender;
@@ -225,7 +227,7 @@ public class RealtimeTableDataManager extends BaseTableDataManager {
     if (indexDir.exists() && (realtimeSegmentZKMetadata.getStatus() == Status.DONE)) {
       // Segment already exists on disk, and metadata has been committed. Treat it like an offline segment
 
-      addSegment(ImmutableSegmentLoader.load(indexDir, indexLoadingConfig, schema));
+      addSegment(loadImmutableSegment(indexDir, indexLoadingConfig, schema));
     } else {
       // Either we don't have the segment on disk or we have not committed in ZK. We should be starting the consumer
       // for realtime segment here. If we wrote it on disk but could not get to commit to zk yet, we should replace the
@@ -247,7 +249,7 @@ public class RealtimeTableDataManager extends BaseTableDataManager {
         LLCRealtimeSegmentZKMetadata llcSegmentMetadata = (LLCRealtimeSegmentZKMetadata) realtimeSegmentZKMetadata;
         if (realtimeSegmentZKMetadata.getStatus().equals(Status.DONE)) {
           // TODO Remove code duplication here and in LLRealtimeSegmentDataManager
-          downloadAndReplaceSegment(segmentName, llcSegmentMetadata, indexLoadingConfig);
+          downloadAndReplaceSegment(segmentName, llcSegmentMetadata, indexLoadingConfig, schema);
           return;
         }
 
@@ -255,10 +257,9 @@ public class RealtimeTableDataManager extends BaseTableDataManager {
         LLCSegmentName llcSegmentName = new LLCSegmentName(segmentName);
         int streamPartitionId = llcSegmentName.getPartitionId();
         _partitionIdToSemaphoreMap.putIfAbsent(streamPartitionId, new Semaphore(1));
-        manager =
-            new LLRealtimeSegmentDataManager(realtimeSegmentZKMetadata, tableConfig, this, _indexDir.getAbsolutePath(),
-                indexLoadingConfig, schema, llcSegmentName, _partitionIdToSemaphoreMap.get(streamPartitionId),
-                _serverMetrics);
+        manager = getLLRealtimeSegmentDataManager(realtimeSegmentZKMetadata, tableConfig, this,
+                _indexDir.getAbsolutePath(), indexLoadingConfig, schema, llcSegmentName,
+                _partitionIdToSemaphoreMap.get(streamPartitionId), _serverMetrics);
       }
       _logger.info("Initialize RealtimeSegmentDataManager - " + segmentName);
       _segmentDataManagerMap.put(segmentName, manager);
@@ -266,7 +267,7 @@ public class RealtimeTableDataManager extends BaseTableDataManager {
   }
 
   public void downloadAndReplaceSegment(String segmentName, LLCRealtimeSegmentZKMetadata llcSegmentMetadata,
-      IndexLoadingConfig indexLoadingConfig) {
+      IndexLoadingConfig indexLoadingConfig, Schema schema) {
     final String uri = llcSegmentMetadata.getDownloadUrl();
     File tempSegmentFolder = new File(_indexDir, "tmp-" + segmentName + "." + System.currentTimeMillis());
     File tempFile = new File(_indexDir, segmentName + ".tar.gz");
@@ -279,7 +280,7 @@ public class RealtimeTableDataManager extends BaseTableDataManager {
       _logger.info("Uncompressed file {} into tmp dir {}", tempFile, tempSegmentFolder);
       FileUtils.moveDirectory(tempSegmentFolder.listFiles()[0], segmentFolder);
       _logger.info("Replacing LLC Segment {}", segmentName);
-      replaceLLSegment(segmentName, indexLoadingConfig);
+      replaceLLSegment(segmentName, indexLoadingConfig, schema);
     } catch (Exception e) {
       throw new RuntimeException(e);
     } finally {
@@ -302,11 +303,10 @@ public class RealtimeTableDataManager extends BaseTableDataManager {
   /**
    * Replaces a committed LLC REALTIME segment.
    */
-  public void replaceLLSegment(String segmentName, IndexLoadingConfig indexLoadingConfig) {
+  public void replaceLLSegment(String segmentName, IndexLoadingConfig indexLoadingConfig, Schema schema) {
     try {
       File indexDir = new File(_indexDir, segmentName);
-      Schema schema = ZKMetadataProvider.getTableSchema(_propertyStore, _tableNameWithType);
-      addSegment(ImmutableSegmentLoader.load(indexDir, indexLoadingConfig, schema));
+      addSegment(loadImmutableSegment(indexDir, indexLoadingConfig, schema));
     } catch (Exception e) {
       throw new RuntimeException(e);
     }
@@ -317,6 +317,18 @@ public class RealtimeTableDataManager extends BaseTableDataManager {
   }
 
   /**
+   *  allow {@link UpsertRealtimeTableDataManager} to override this method
+   */
+  protected abstract LLRealtimeSegmentDataManager getLLRealtimeSegmentDataManager(
+          RealtimeSegmentZKMetadata realtimeSegmentZKMetadata, TableConfig tableConfig,
+          RealtimeTableDataManager realtimeTableDataManager, String indexDirPath, IndexLoadingConfig indexLoadingConfig,
+          Schema schema, LLCSegmentName llcSegmentName, Semaphore partitionConsumerSemaphore, ServerMetrics serverMetrics)
+      throws Exception;
+
+  protected abstract ImmutableSegment loadImmutableSegment(File indexDir, IndexLoadingConfig indexLoadingConfig, Schema schema)
+      throws Exception;
+
+  /**
    * Validate a schema against the table config for real-time record consumption.
    * Ideally, we should validate these things when schema is added or table is created, but either of these
    * may be changed while the table is already provisioned. For the change to take effect, we need to restart the
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/UpsertLLRealtimeSegmentDataManager.java b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/UpsertLLRealtimeSegmentDataManager.java
new file mode 100644
index 0000000..f08037a
--- /dev/null
+++ b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/UpsertLLRealtimeSegmentDataManager.java
@@ -0,0 +1,153 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.core.data.manager.realtime;
+
+import com.google.common.base.Preconditions;
+import org.apache.pinot.common.config.TableConfig;
+import org.apache.pinot.common.metadata.segment.RealtimeSegmentZKMetadata;
+import org.apache.pinot.common.metrics.ServerMeter;
+import org.apache.pinot.common.metrics.ServerMetrics;
+import org.apache.pinot.common.utils.LLCSegmentName;
+import org.apache.pinot.core.data.manager.UpsertSegmentDataManager;
+import org.apache.pinot.core.indexsegment.UpsertSegment;
+import org.apache.pinot.core.indexsegment.mutable.MutableSegmentImpl;
+import org.apache.pinot.core.indexsegment.mutable.MutableUpsertSegmentImpl;
+import org.apache.pinot.core.realtime.impl.RealtimeSegmentConfig;
+import org.apache.pinot.core.segment.index.loader.IndexLoadingConfig;
+import org.apache.pinot.core.segment.updater.SegmentUpdater;
+import org.apache.pinot.grigio.common.messages.KeyCoordinatorQueueMsg;
+import org.apache.pinot.grigio.common.rpcQueue.ProduceTask;
+import org.apache.pinot.grigio.common.rpcQueue.QueueProducer;
+import org.apache.pinot.grigio.common.storageProvider.UpdateLogEntry;
+import org.apache.pinot.grigio.servers.KeyCoordinatorProvider;
+import org.apache.pinot.spi.data.DimensionFieldSpec;
+import org.apache.pinot.spi.data.Schema;
+import org.apache.pinot.spi.data.TimeFieldSpec;
+import org.apache.pinot.spi.data.readers.GenericRow;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.concurrent.Semaphore;
+
+/**
+ * class design is pretty bad right now, need to rework inheritance to abstract the base class or use composition instead
+ */
+public class UpsertLLRealtimeSegmentDataManager extends LLRealtimeSegmentDataManager implements UpsertSegmentDataManager {
+
+  private final QueueProducer _keyCoordinatorQueueProducer;
+
+  public UpsertLLRealtimeSegmentDataManager(RealtimeSegmentZKMetadata segmentZKMetadata, TableConfig tableConfig,
+      RealtimeTableDataManager realtimeTableDataManager, String resourceDataDir, IndexLoadingConfig indexLoadingConfig,
+      Schema schema, LLCSegmentName llcSegmentName, Semaphore partitionConsumerSemaphore, ServerMetrics serverMetrics) throws Exception {
+    super(segmentZKMetadata, tableConfig, realtimeTableDataManager, resourceDataDir, indexLoadingConfig, schema,
+            llcSegmentName, partitionConsumerSemaphore, serverMetrics);
+    Preconditions.checkState(_schema.getPrimaryKeyFieldSpec() != null, "primary key not found");
+    Preconditions.checkState(_schema.getOffsetKeyFieldSpec() != null, "offset key not found");
+    _keyCoordinatorQueueProducer = KeyCoordinatorProvider.getInstance().getCachedProducer(_tableNameWithType);
+    initVirtualColumns();
+  }
+
+  public String getSegmentName() {
+    return _segmentNameStr;
+  }
+
+  @Override
+  public void updateVirtualColumns(List<UpdateLogEntry> messages) {
+    ((UpsertSegment) _realtimeSegment).updateVirtualColumn(messages);
+  }
+
+  @Override
+  public String getVirtualColumnInfo(long offset) {
+    return ((UpsertSegment) _realtimeSegment).getVirtualColumnInfo(offset);
+  }
+
+  @Override
+  public void destroy() {
+    SegmentUpdater.getInstance().removeSegmentDataManager(_tableNameWithType, _llcSegmentName.getSegmentName(), this);
+    super.destroy();
+  }
+
+  @Override
+  protected MutableSegmentImpl createMutableSegment(RealtimeSegmentConfig config) {
+    return new MutableUpsertSegmentImpl(config);
+  }
+
+  @Override
+  protected void processTransformedRow(GenericRow row, long offset) {
+    row.putField(_schema.getOffsetKey(), offset);
+  }
+
+  @Override
+  protected void postIndexProcessing(GenericRow row, long offset) {
+    emitEventToKeyCoordinator(row, offset);
+  }
+
+  @Override
+  protected boolean consumeLoop() throws Exception {
+    boolean result = super.consumeLoop();
+    segmentLogger.info("flushing kafka producer");
+    _keyCoordinatorQueueProducer.flush();
+    segmentLogger.info("done flushing kafka producer");
+    return result;
+  }
+
+  /**
+   * not handling error right now
+   * @param row
+   * @param offset
+   */
+  private void emitEventToKeyCoordinator(GenericRow row, long offset) {
+    final byte[] primaryKeyBytes = getPrimaryKeyBytesFromRow(row);
+    final long timestampMillis = getTimestampFromRow(row);
+    ProduceTask<byte[], KeyCoordinatorQueueMsg> task = new ProduceTask<>(primaryKeyBytes,
+        new KeyCoordinatorQueueMsg(primaryKeyBytes, _segmentNameStr, timestampMillis, offset));
+    task.setCallback(new ProduceTask.Callback() {
+      @Override
+      public void onSuccess() {
+        // do nothing on success
+      }
+
+      @Override
+      public void onFailure(Exception ex) {
+        // right now we just log the error and not really doing anything
+        // TODO: retries/record logics
+        _serverMetrics.addMeteredGlobalValue(ServerMeter.MESSAGE_PRODUCE_FAILED_COUNT, 1);
+      }
+    });
+    _keyCoordinatorQueueProducer.produce(task);
+  }
+
+  private byte[] getPrimaryKeyBytesFromRow(GenericRow row) {
+    DimensionFieldSpec primaryKeyDimension = _schema.getPrimaryKeyFieldSpec();
+    return _schema.getByteArrayFromField(row.getValue(primaryKeyDimension.getName()), primaryKeyDimension);
+  }
+
+  private long getTimestampFromRow(GenericRow row) {
+    TimeFieldSpec spec = _schema.getTimeFieldSpec();
+    return spec.getIncomingGranularitySpec().toMillis(row.getValue(spec.getIncomingTimeColumnName()));
+  }
+
+
+  private void initVirtualColumns() throws IOException {
+    // 1. ensure the data manager can capture all update events
+    // 2. load all existing messages
+    SegmentUpdater.getInstance().addSegmentDataManager(_tableNameWithType, _llcSegmentName, this);
+    ((UpsertSegment) _realtimeSegment).initVirtualColumn();
+  }
+}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/UpsertRealtimeTableDataManager.java b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/UpsertRealtimeTableDataManager.java
new file mode 100644
index 0000000..12569db
--- /dev/null
+++ b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/UpsertRealtimeTableDataManager.java
@@ -0,0 +1,82 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.core.data.manager.realtime;
+
+import org.apache.pinot.common.utils.LLCSegmentName;
+import org.apache.pinot.core.data.manager.offline.UpsertImmutableSegmentDataManager;
+import org.apache.pinot.common.config.TableConfig;
+import org.apache.pinot.common.metadata.instance.InstanceZKMetadata;
+import org.apache.pinot.common.metadata.segment.RealtimeSegmentZKMetadata;
+import org.apache.pinot.common.metrics.ServerMetrics;
+import org.apache.pinot.core.data.manager.offline.ImmutableSegmentDataManager;
+import org.apache.pinot.core.indexsegment.immutable.ImmutableSegment;
+import org.apache.pinot.core.indexsegment.immutable.ImmutableSegmentLoader;
+import org.apache.pinot.core.segment.index.loader.IndexLoadingConfig;
+import org.apache.pinot.grigio.common.storageProvider.UpdateLogStorageProvider;
+import org.apache.pinot.spi.data.Schema;
+
+import javax.annotation.Nonnull;
+import java.io.File;
+import java.io.IOException;
+import java.util.concurrent.Semaphore;
+
+public class UpsertRealtimeTableDataManager extends RealtimeTableDataManager {
+  private UpdateLogStorageProvider _updateLogStorageProvider;
+
+  public UpsertRealtimeTableDataManager(Semaphore segmentBuildSemaphore) {
+    super(segmentBuildSemaphore);
+    _updateLogStorageProvider = UpdateLogStorageProvider.getInstance();
+  }
+
+  @Override
+  public void addSegment(@Nonnull String segmentName, @Nonnull TableConfig tableConfig,
+                         @Nonnull IndexLoadingConfig indexLoadingConfig) throws Exception {
+    _updateLogStorageProvider.addSegment(_tableNameWithType, segmentName);
+    super.addSegment(segmentName, tableConfig, indexLoadingConfig);
+  }
+
+  @Override
+  protected LLRealtimeSegmentDataManager getLLRealtimeSegmentDataManager(
+          RealtimeSegmentZKMetadata realtimeSegmentZKMetadata, TableConfig tableConfig,
+          RealtimeTableDataManager realtimeTableDataManager, String indexDirPath, IndexLoadingConfig indexLoadingConfig,
+          Schema schema, LLCSegmentName llcSegmentName, Semaphore partitionConsumerSemaphore, ServerMetrics serverMetrics)
+      throws Exception {
+    return new UpsertLLRealtimeSegmentDataManager(realtimeSegmentZKMetadata, tableConfig, realtimeTableDataManager,
+            indexDirPath, indexLoadingConfig, schema, llcSegmentName, partitionConsumerSemaphore, serverMetrics);
+  }
+
+  @Override
+  protected ImmutableSegmentDataManager getImmutableSegmentDataManager(ImmutableSegment immutableSegment) {
+    try {
+      return new UpsertImmutableSegmentDataManager(immutableSegment);
+    } catch (IOException e) {
+      throw new RuntimeException("failed to init the upsert immutable segment", e);
+    }
+  }
+
+  @Override
+  protected ImmutableSegment loadImmutableSegment(File indexDir, IndexLoadingConfig indexLoadingConfig, Schema schema) {
+    try {
+      return ImmutableSegmentLoader.loadUpsertSegment(indexDir, indexLoadingConfig, schema);
+    } catch (Exception e) {
+      throw new RuntimeException("failed to load immutable segment", e);
+    }
+  }
+
+}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/VirtualColumnContext.java b/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/UpsertSegment.java
similarity index 52%
copy from pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/VirtualColumnContext.java
copy to pinot-core/src/main/java/org/apache/pinot/core/indexsegment/UpsertSegment.java
index bb32320..125cf06 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/VirtualColumnContext.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/UpsertSegment.java
@@ -16,29 +16,26 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.pinot.core.segment.virtualcolumn;
+package org.apache.pinot.core.indexsegment;
 
-import org.apache.pinot.spi.data.FieldSpec;
+import org.apache.pinot.grigio.common.storageProvider.UpdateLogEntry;
 
+import java.io.IOException;
 
-/**
- * Miscellaneous context information about the virtual column.
- * It will be used to build various components (dictionary, reader, etc) in the virtual column provider.
- */
-public class VirtualColumnContext {
-  private FieldSpec _fieldSpec;
-  private int _totalDocCount;
+public interface UpsertSegment {
 
-  public VirtualColumnContext(FieldSpec fieldSpec, int totalDocCount) {
-    _fieldSpec = fieldSpec;
-    _totalDocCount = totalDocCount;
-  }
+  /**
+   * update the upsert-related virtual columns with the new values in this list of update logs
+   * @param messages list of updates logs to update the virtual columns
+   */
+  void updateVirtualColumn(Iterable<UpdateLogEntry> messages);
 
-  public FieldSpec getFieldSpec() {
-    return _fieldSpec;
-  }
+  /**
+   * get the upsert related virtual column debug info given an offset
+   * @param offset the offset we want to look up the virtual column info from
+   * @return debug info describing the upsert-related virtual column info
+   */
+  String getVirtualColumnInfo(long offset);
 
-  public int getTotalDocCount() {
-    return _totalDocCount;
-  }
+  void initVirtualColumn() throws IOException;
 }
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/immutable/ImmutableSegmentImpl.java b/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/immutable/ImmutableSegmentImpl.java
index 600660a..21f056c 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/immutable/ImmutableSegmentImpl.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/immutable/ImmutableSegmentImpl.java
@@ -43,10 +43,10 @@ import org.slf4j.LoggerFactory;
 public class ImmutableSegmentImpl implements ImmutableSegment {
   private static final Logger LOGGER = LoggerFactory.getLogger(ImmutableSegmentImpl.class);
 
-  private final SegmentDirectory _segmentDirectory;
-  private final SegmentMetadataImpl _segmentMetadata;
-  private final Map<String, ColumnIndexContainer> _indexContainerMap;
-  private final StarTreeIndexContainer _starTreeIndexContainer;
+  protected final SegmentDirectory _segmentDirectory;
+  protected final SegmentMetadataImpl _segmentMetadata;
+  protected final Map<String, ColumnIndexContainer> _indexContainerMap;
+  protected final StarTreeIndexContainer _starTreeIndexContainer;
 
   public ImmutableSegmentImpl(SegmentDirectory segmentDirectory, SegmentMetadataImpl segmentMetadata,
       Map<String, ColumnIndexContainer> columnIndexContainerMap,
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/immutable/ImmutableSegmentLoader.java b/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/immutable/ImmutableSegmentLoader.java
index 90eb839..49d7729 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/immutable/ImmutableSegmentLoader.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/immutable/ImmutableSegmentLoader.java
@@ -44,6 +44,8 @@ import org.apache.pinot.spi.data.Schema;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import javax.annotation.Nonnull;
+
 
 public class ImmutableSegmentLoader {
   private ImmutableSegmentLoader() {
@@ -70,9 +72,23 @@ public class ImmutableSegmentLoader {
   }
 
   public static ImmutableSegment load(File indexDir, IndexLoadingConfig indexLoadingConfig, @Nullable Schema schema)
-      throws Exception {
-    Preconditions
-        .checkArgument(indexDir.isDirectory(), "Index directory: {} does not exist or is not a directory", indexDir);
+          throws Exception {
+    return loadHelper(indexDir, indexLoadingConfig, schema);
+  }
+
+  /**
+   * to load upsert related segment
+   */
+  public static ImmutableUpsertSegmentImpl loadUpsertSegment(File indexDir, IndexLoadingConfig indexLoadingConfig,
+                                                             Schema schema) throws Exception {
+    ImmutableSegmentImpl segment = loadHelper(indexDir, indexLoadingConfig, schema);
+    return ImmutableUpsertSegmentImpl.copyOf(segment);
+  }
+
+  private static ImmutableSegmentImpl loadHelper(File indexDir, IndexLoadingConfig indexLoadingConfig,
+                                                 @Nullable Schema schema) throws Exception {
+    Preconditions.checkArgument(indexDir.isDirectory(), "Index directory: {} does not exist or is not a directory",
+        indexDir);
 
     // Convert segment version if necessary
     // NOTE: this step may modify the segment metadata
@@ -100,7 +116,7 @@ public class ImmutableSegmentLoader {
     }
 
     // Load the metadata again since converter and pre-processor may have changed it
-    SegmentMetadataImpl segmentMetadata = new SegmentMetadataImpl(indexDir);
+    SegmentMetadataImpl segmentMetadata = new SegmentMetadataImpl(indexDir, schema);
 
     // Load the segment
     ReadMode readMode = indexLoadingConfig.getReadMode();
@@ -125,7 +141,7 @@ public class ImmutableSegmentLoader {
         String columnName = fieldSpec.getName();
         VirtualColumnProvider provider =
             VirtualColumnProviderFactory.buildProvider(fieldSpec.getVirtualColumnProvider());
-        VirtualColumnContext context = new VirtualColumnContext(fieldSpec, segmentMetadata.getTotalDocs());
+        VirtualColumnContext context = new VirtualColumnContext(fieldSpec, segmentMetadata.getTotalDocs(), false);
         indexContainerMap.put(columnName, provider.buildColumnIndexContainer(context));
         segmentMetadata.getColumnMetadataMap().put(columnName, provider.buildMetadata(context));
       }
@@ -141,4 +157,5 @@ public class ImmutableSegmentLoader {
 
     return new ImmutableSegmentImpl(segmentDirectory, segmentMetadata, indexContainerMap, starTreeIndexContainer);
   }
+
 }
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/immutable/ImmutableUpsertSegmentImpl.java b/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/immutable/ImmutableUpsertSegmentImpl.java
new file mode 100644
index 0000000..5ab42f5
--- /dev/null
+++ b/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/immutable/ImmutableUpsertSegmentImpl.java
@@ -0,0 +1,251 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.core.indexsegment.immutable;
+
+import com.clearspring.analytics.util.Preconditions;
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.pinot.common.Utils;
+import org.apache.pinot.common.config.TableNameBuilder;
+import org.apache.pinot.common.utils.CommonConstants;
+import org.apache.pinot.core.indexsegment.UpsertSegment;
+import org.apache.pinot.core.io.reader.BaseSingleColumnSingleValueReader;
+import org.apache.pinot.core.io.reader.DataFileReader;
+import org.apache.pinot.core.segment.index.SegmentMetadataImpl;
+import org.apache.pinot.core.segment.index.column.ColumnIndexContainer;
+import org.apache.pinot.core.segment.index.readers.Dictionary;
+import org.apache.pinot.core.segment.store.SegmentDirectory;
+import org.apache.pinot.core.segment.updater.UpsertWaterMarkManager;
+import org.apache.pinot.core.segment.virtualcolumn.mutable.VirtualColumnLongValueReaderWriter;
+import org.apache.pinot.core.startree.v2.store.StarTreeIndexContainer;
+import org.apache.pinot.grigio.common.storageProvider.UpdateLogEntry;
+import org.apache.pinot.grigio.common.storageProvider.UpdateLogEntrySet;
+import org.apache.pinot.grigio.common.storageProvider.UpdateLogStorageProvider;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.Nullable;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class ImmutableUpsertSegmentImpl extends ImmutableSegmentImpl implements UpsertSegment {
+
+  private static final Logger LOGGER = LoggerFactory.getLogger(ImmutableUpsertSegmentImpl.class);
+
+  private final List<VirtualColumnLongValueReaderWriter> _virtualColumnsReaderWriter;
+  private final String _tableNameWithType;
+  private final String _segmentName;
+  private final int _totalDoc;
+  private long _minSourceOffset;
+  private final UpsertWaterMarkManager _upsertWaterMarkManager;
+  private final UpdateLogStorageProvider _updateLogStorageProvider;
+  // use array for mapping bewteen offset to docId, where actual offset = min_offset + array_index
+  // use 4 bytes per record
+  private int[] _sourceOffsetToDocIdArray;
+
+  private static final int DEFAULT_DOC_ID_FOR_MISSING_ENTRY = -1;
+
+  public ImmutableUpsertSegmentImpl(SegmentDirectory segmentDirectory,
+                                    SegmentMetadataImpl segmentMetadata,
+                                    Map<String, ColumnIndexContainer> columnIndexContainerMap,
+                                    @Nullable StarTreeIndexContainer starTreeIndexContainer) {
+    super(segmentDirectory, segmentMetadata, columnIndexContainerMap, starTreeIndexContainer);
+    Preconditions.checkState(segmentMetadata.getSchema().isTableForUpsert(), "table should be upsert but it is not");
+    _tableNameWithType = TableNameBuilder.ensureTableNameWithType(segmentMetadata.getTableName(),
+        CommonConstants.Helix.TableType.REALTIME);
+    _segmentName = segmentMetadata.getName();
+    _totalDoc = segmentMetadata.getTotalDocs();
+    _upsertWaterMarkManager = UpsertWaterMarkManager.getInstance();
+    _updateLogStorageProvider = UpdateLogStorageProvider.getInstance();
+    _virtualColumnsReaderWriter = new ArrayList<>();
+    for (Map.Entry<String, ColumnIndexContainer> entry: columnIndexContainerMap.entrySet()) {
+      String columnName = entry.getKey();
+      ColumnIndexContainer container = entry.getValue();
+      if (segmentMetadata.getSchema().isVirtualColumn(columnName) && (container.getForwardIndex() instanceof VirtualColumnLongValueReaderWriter)) {
+        _virtualColumnsReaderWriter.add((VirtualColumnLongValueReaderWriter) container.getForwardIndex());
+      }
+    }
+     buildOffsetToDocIdMap(columnIndexContainerMap.get(segmentMetadata.getSchema().getOffsetKey()));
+  }
+
+  /** constructor used for creating instance in test cases
+   * should not be used for creating regular segment
+   */
+  @VisibleForTesting
+  protected ImmutableUpsertSegmentImpl(List<VirtualColumnLongValueReaderWriter> readerWriters,
+                                       int totalDoc, UpsertWaterMarkManager manager,
+                                       UpdateLogStorageProvider updateLogStorageProvider,
+                                       long minSourceOffset, int[] offsetToDocId) {
+    super(null, null, null, null);
+    _tableNameWithType = "testTable";
+    _segmentName = "testSegment";
+    _virtualColumnsReaderWriter = readerWriters;
+    _totalDoc = totalDoc;
+    _upsertWaterMarkManager = manager;
+    _updateLogStorageProvider = updateLogStorageProvider;
+    _minSourceOffset = minSourceOffset;
+    _sourceOffsetToDocIdArray = offsetToDocId;
+
+  }
+
+  private void buildOffsetToDocIdMap(ColumnIndexContainer offsetColumnIndexContainer) {
+    long start = System.currentTimeMillis();
+    final DataFileReader reader = offsetColumnIndexContainer.getForwardIndex();
+    final Dictionary dictionary = offsetColumnIndexContainer.getDictionary();
+    Map<Long, Integer> kafkaOffsetToDocIdMap = new HashMap<>();
+    long minOffset = Long.MAX_VALUE;
+    long maxOffset = 0;
+    if (reader instanceof BaseSingleColumnSingleValueReader) {
+      BaseSingleColumnSingleValueReader scsvReader = (BaseSingleColumnSingleValueReader) reader;
+      for (int docId = 0; docId < _totalDoc; docId++) {
+        final Long offset;
+        if (dictionary == null) {
+          offset = scsvReader.getLong(docId);
+        } else {
+          offset = (Long) dictionary.get(scsvReader.getInt(docId));
+        }
+        if (offset == null) {
+          LOGGER.error("kafka offset is null at docID {}", docId);
+        } else {
+          minOffset = Math.min(offset, minOffset);
+          maxOffset = Math.max(offset, maxOffset);
+          kafkaOffsetToDocIdMap.put(offset, docId);
+        }
+      }
+      _minSourceOffset = minOffset;
+      int size = Math.toIntExact(maxOffset - minOffset + 1);
+      _sourceOffsetToDocIdArray = new int[size];
+      for (int i = 0; i < size; i++) {
+        _sourceOffsetToDocIdArray[i] = kafkaOffsetToDocIdMap.getOrDefault(i + minOffset,
+            DEFAULT_DOC_ID_FOR_MISSING_ENTRY);
+      }
+    } else {
+      throw new RuntimeException("unexpected forward reader type for kafka offset column " + reader.getClass());
+    }
+    LOGGER.info("built offset to DocId map for segment {} with {} documents in {} ms", _segmentName, _totalDoc, System.currentTimeMillis() - start);
+  }
+
+  public static ImmutableUpsertSegmentImpl copyOf(ImmutableSegmentImpl immutableSegment) {
+
+    return new ImmutableUpsertSegmentImpl(immutableSegment._segmentDirectory, immutableSegment._segmentMetadata,
+        immutableSegment._indexContainerMap, immutableSegment._starTreeIndexContainer);
+  }
+
+  @Override
+  public void updateVirtualColumn(Iterable<UpdateLogEntry> logEntries) {
+    for (UpdateLogEntry logEntry: logEntries) {
+      boolean updated = false;
+      int docId = getDocIdFromSourceOffset(logEntry.getOffset());
+      for (VirtualColumnLongValueReaderWriter readerWriter : _virtualColumnsReaderWriter) {
+        updated = readerWriter.update(docId, logEntry.getValue(), logEntry.getType()) || updated;
+      }
+      if (updated) {
+        _upsertWaterMarkManager.processMessage(_tableNameWithType, _segmentName, logEntry);
+      }
+    }
+  }
+
+  @Override
+  public String getVirtualColumnInfo(long offset) {
+    int docId = getDocIdFromSourceOffset(offset);
+    StringBuilder result = new StringBuilder("matched: ");
+    for (VirtualColumnLongValueReaderWriter readerWriter : _virtualColumnsReaderWriter) {
+      result.append(readerWriter.getInt(docId)).append("; ");
+    }
+    return result.toString();
+  }
+
+  /**
+   * this method will fetch all updates from update logs in local disk and apply those updates to the current virtual columns
+   * it traverses through all records in the current segment and match existing updates log to its kafka offsets
+   * @throws IOException
+   */
+  @Override
+  public void initVirtualColumn() throws IOException {
+    long start = System.currentTimeMillis();
+    final UpdateLogEntrySet updateLogEntries = _updateLogStorageProvider.getAllMessages(_tableNameWithType, _segmentName);
+    LOGGER.info("load {} update log entry from update log storage provider for segment {} in {} ms",
+        updateLogEntries.size(), _segmentName, System.currentTimeMillis() - start);
+
+    start = System.currentTimeMillis();
+    final long maxOffset = _totalDoc + _minSourceOffset;
+    int unmatchedLogEntryCount = 0;
+    try {
+      Map<Integer, Long> partitionToHighestWatermark = new HashMap<>();
+      int readerWriteCount = _virtualColumnsReaderWriter.size();
+      for (UpdateLogEntry logEntry: updateLogEntries) {
+        final int partition = logEntry.getPartition();
+        final long offset = logEntry.getOffset();
+        if (offset >= _minSourceOffset && offset < maxOffset) {
+          final int docId = _sourceOffsetToDocIdArray[Math.toIntExact(offset - _minSourceOffset)];
+          if (docId != DEFAULT_DOC_ID_FOR_MISSING_ENTRY) {
+            // use traditional for loop over int instead of foreach to give hints for JIT to do loop unroll in byte code
+            for (int i = 0; i < readerWriteCount; i++) {
+              _virtualColumnsReaderWriter.get(i).update(docId, logEntry.getValue(), logEntry.getType());
+            }
+            if (logEntry.getValue() > partitionToHighestWatermark.getOrDefault(partition, -1L)) {
+              partitionToHighestWatermark.put(partition, logEntry.getValue());
+            }
+          } else {
+            LOGGER.error("segment {} got in-range update log at offset {} but no matching docId", _segmentName, offset);
+          }
+        } else {
+          unmatchedLogEntryCount++;
+        }
+      }
+      if (unmatchedLogEntryCount > 0) {
+        LOGGER.info("segment {} encountered {} update logs that are outside of its range", _segmentName,
+            unmatchedLogEntryCount);
+      }
+      partitionToHighestWatermark.forEach((partition, value) ->
+          _upsertWaterMarkManager.processVersionUpdate(_tableNameWithType, partition, value));
+
+    } catch (Exception e) {
+      LOGGER.error("failed to load the offset with thread pool");
+      Utils.rethrowException(e);
+    }
+    LOGGER.info("populated all log entries to virtual columns for current immutable segment {} in {} ms",
+        _segmentName, System.currentTimeMillis() - start);
+  }
+
+  /**
+   * given a offset from source kafka topic, return the docId associated with it
+   * throw exception if there is no docId for the associated kafka offset (because kafka offset might not be continuous)
+   * @param offset offset in the source topic
+   * @return the corresponding docId
+   */
+  private int getDocIdFromSourceOffset(long offset) throws RuntimeException {
+    if (offset < _minSourceOffset || offset - _minSourceOffset >= _sourceOffsetToDocIdArray.length) {
+      LOGGER.error("offset {} is outside range for current segment {} start offset {} size {}",
+          offset, _segmentName, _minSourceOffset, _sourceOffsetToDocIdArray.length);
+      throw new RuntimeException("offset outside range");
+    } else {
+      int position = Math.toIntExact(offset - _minSourceOffset);
+      if (_sourceOffsetToDocIdArray[position] == DEFAULT_DOC_ID_FOR_MISSING_ENTRY) {
+        LOGGER.error("no docId associated with offset {} for segment {}", offset, _segmentName);
+        throw new RuntimeException("docId not found");
+      } else {
+        return _sourceOffsetToDocIdArray[position];
+      }
+    }
+
+  }
+}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/VirtualColumnContext.java b/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/mutable/MutableAppendSegmentImpl.java
similarity index 56%
copy from pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/VirtualColumnContext.java
copy to pinot-core/src/main/java/org/apache/pinot/core/indexsegment/mutable/MutableAppendSegmentImpl.java
index bb32320..1c9effc 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/VirtualColumnContext.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/mutable/MutableAppendSegmentImpl.java
@@ -16,29 +16,22 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.pinot.core.segment.virtualcolumn;
+package org.apache.pinot.core.indexsegment.mutable;
 
-import org.apache.pinot.spi.data.FieldSpec;
+import org.apache.pinot.core.realtime.impl.RealtimeSegmentConfig;
+import org.apache.pinot.spi.data.readers.GenericRow;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
+public class MutableAppendSegmentImpl extends MutableSegmentImpl {
+  private static final Logger LOGGER = LoggerFactory.getLogger(MutableUpsertSegmentImpl.class);
 
-/**
- * Miscellaneous context information about the virtual column.
- * It will be used to build various components (dictionary, reader, etc) in the virtual column provider.
- */
-public class VirtualColumnContext {
-  private FieldSpec _fieldSpec;
-  private int _totalDocCount;
-
-  public VirtualColumnContext(FieldSpec fieldSpec, int totalDocCount) {
-    _fieldSpec = fieldSpec;
-    _totalDocCount = totalDocCount;
-  }
-
-  public FieldSpec getFieldSpec() {
-    return _fieldSpec;
+  public MutableAppendSegmentImpl(RealtimeSegmentConfig config) {
+    super(config);
   }
 
-  public int getTotalDocCount() {
-    return _totalDocCount;
+  @Override
+  protected void postProcessRecords(GenericRow row, int docId) {
+    // nothing
   }
 }
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/mutable/MutableSegmentImpl.java b/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/mutable/MutableSegmentImpl.java
index f800d95..b80ac64 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/mutable/MutableSegmentImpl.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/mutable/MutableSegmentImpl.java
@@ -31,6 +31,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import javax.annotation.Nullable;
+
 import org.apache.pinot.common.config.SegmentPartitionConfig;
 import org.apache.pinot.common.segment.SegmentMetadata;
 import org.apache.pinot.core.indexsegment.IndexSegmentUtils;
@@ -56,6 +57,7 @@ import org.apache.pinot.core.segment.index.data.source.ColumnDataSource;
 import org.apache.pinot.core.segment.index.readers.BloomFilterReader;
 import org.apache.pinot.core.segment.index.readers.InvertedIndexReader;
 import org.apache.pinot.core.segment.index.readers.NullValueVectorReader;
+import org.apache.pinot.core.segment.index.readers.Dictionary;
 import org.apache.pinot.core.segment.virtualcolumn.VirtualColumnContext;
 import org.apache.pinot.core.segment.virtualcolumn.VirtualColumnProvider;
 import org.apache.pinot.core.segment.virtualcolumn.VirtualColumnProviderFactory;
@@ -74,7 +76,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 
-public class MutableSegmentImpl implements MutableSegment {
+public abstract class MutableSegmentImpl implements MutableSegment {
   // For multi-valued column, forward-index.
   // Maximum number of multi-values per row. We assert on this.
   private static final int MAX_MULTI_VALUES_PER_ROW = 1000;
@@ -86,11 +88,12 @@ public class MutableSegmentImpl implements MutableSegment {
   private static final int NODICT_VARIABLE_WIDTH_ESTIMATED_AVERAGE_VALUE_LENGTH_DEFAULT = 100;
   private static final int NODICT_VARIABLE_WIDTH_ESTIMATED_NUMBER_OF_VALUES_DEFAULT = 100_000;
 
-  private final Logger _logger;
+  protected final Logger _logger;
   private final long _startTimeMillis = System.currentTimeMillis();
 
-  private final String _segmentName;
-  private final Schema _schema;
+  protected final String _tableName;
+  protected final String _segmentName;
+  protected final Schema _schema;
   private final int _capacity;
   private final SegmentMetadata _segmentMetadata;
   private final boolean _offHeap;
@@ -108,18 +111,23 @@ public class MutableSegmentImpl implements MutableSegment {
   private final IdMap<FixedIntArray> _recordIdMap;
   private boolean _aggregateMetrics;
 
-  private volatile int _numDocsIndexed = 0;
+  protected volatile int _numDocsIndexed = 0;
 
   // to compute the rolling interval
-  private volatile long _minTime = Long.MAX_VALUE;
-  private volatile long _maxTime = Long.MIN_VALUE;
-  private final int _numKeyColumns;
+  protected volatile long _minTime = Long.MAX_VALUE;
+  protected volatile long _maxTime = Long.MIN_VALUE;
+  protected final int _numKeyColumns;
 
   // Cache the physical (non-virtual) field specs
   private final Collection<FieldSpec> _physicalFieldSpecs;
   private final Collection<DimensionFieldSpec> _physicalDimensionFieldSpecs;
   private final Collection<MetricFieldSpec> _physicalMetricFieldSpecs;
 
+  // Cache some virtual columns
+  protected final Map<String, VirtualColumnProvider> _virtualColumnProviderMap = new HashMap<>();
+  protected final Map<String, Dictionary> _virtualColumnDictionary = new HashMap<>();
+  protected final Map<String, DataFileReader> _virtualColumnIndexReader = new HashMap<>();
+
   // default message metadata
   private volatile long _lastIndexedTimeMs = Long.MIN_VALUE;
   private volatile long _latestIngestionTimeMs = Long.MIN_VALUE;
@@ -127,6 +135,7 @@ public class MutableSegmentImpl implements MutableSegment {
   private RealtimeLuceneReaders _realtimeLuceneReaders;
 
   public MutableSegmentImpl(RealtimeSegmentConfig config) {
+    _tableName = config.getTableName();
     _segmentName = config.getSegmentName();
     _schema = config.getSchema();
     _capacity = config.getCapacity();
@@ -161,6 +170,7 @@ public class MutableSegmentImpl implements MutableSegment {
 
     Collection<FieldSpec> allFieldSpecs = _schema.getAllFieldSpecs();
     List<FieldSpec> physicalFieldSpecs = new ArrayList<>(allFieldSpecs.size());
+    List<FieldSpec> virtualFieldSpecs = new ArrayList<>(allFieldSpecs.size());
     List<DimensionFieldSpec> physicalDimensionFieldSpecs = new ArrayList<>(_schema.getDimensionNames().size());
     List<MetricFieldSpec> physicalMetricFieldSpecs = new ArrayList<>(_schema.getMetricNames().size());
 
@@ -174,6 +184,9 @@ public class MutableSegmentImpl implements MutableSegment {
         } else if (fieldType == FieldSpec.FieldType.METRIC) {
           physicalMetricFieldSpecs.add((MetricFieldSpec) fieldSpec);
         }
+      } else {
+        virtualFieldSpecs.add(fieldSpec);
+
       }
     }
     _physicalFieldSpecs = Collections.unmodifiableCollection(physicalFieldSpecs);
@@ -191,6 +204,7 @@ public class MutableSegmentImpl implements MutableSegment {
 
     int avgNumMultiValues = config.getAvgNumMultiValues();
 
+
     // Initialize for each column
     for (FieldSpec fieldSpec : _physicalFieldSpecs) {
       String column = fieldSpec.getName();
@@ -269,6 +283,7 @@ public class MutableSegmentImpl implements MutableSegment {
 
       _indexReaderWriterMap.put(column, indexReaderWriter);
 
+
       if (invertedIndexColumns.contains(column)) {
         _invertedIndexMap.put(column, new RealtimeInvertedIndexReader());
       }
@@ -287,6 +302,21 @@ public class MutableSegmentImpl implements MutableSegment {
       }
     }
 
+    for (FieldSpec fieldSpec : _physicalFieldSpecs) {
+      String column = fieldSpec.getName();
+      VirtualColumnContext virtualColumnContext = new VirtualColumnContext(fieldSpec, _capacity, true);
+      final VirtualColumnProvider provider =
+              VirtualColumnProviderFactory.buildProvider(fieldSpec.getVirtualColumnProvider());
+      if (provider == null) {
+        throw new RuntimeException(String.format("failed to create virtual segment provider for field %s", fieldSpec.getName()));
+      }
+      DataFileReader reader = provider.buildReader(virtualColumnContext);
+      Dictionary dictionary = provider.buildDictionary(virtualColumnContext);
+      _virtualColumnProviderMap.put(column, provider);
+      _virtualColumnIndexReader.put(column, reader);
+      _virtualColumnDictionary.put(column, dictionary);
+    }
+
     if (_realtimeLuceneReaders != null) {
       // add the realtime lucene index readers to the global queue for refresh task to pick up
       RealtimeLuceneIndexRefreshState realtimeLuceneIndexRefreshState = RealtimeLuceneIndexRefreshState.getInstance();
@@ -347,6 +377,7 @@ public class MutableSegmentImpl implements MutableSegment {
 
       // Update number of document indexed at last to make the latest record queryable
       canTakeMore = _numDocsIndexed++ < _capacity;
+      postProcessRecords(row, docId);
     } else {
       Preconditions
           .checkState(_aggregateMetrics, "Invalid document-id during indexing: " + docId + " expected: " + numDocs);
@@ -491,6 +522,8 @@ public class MutableSegmentImpl implements MutableSegment {
     }
   }
 
+  protected abstract void postProcessRecords(GenericRow row, int docId);
+
   private boolean aggregateMetrics(GenericRow row, int docId) {
     for (MetricFieldSpec metricFieldSpec : _physicalMetricFieldSpecs) {
       String column = metricFieldSpec.getName();
@@ -556,9 +589,9 @@ public class MutableSegmentImpl implements MutableSegment {
   public ColumnDataSource getDataSource(String columnName) {
     FieldSpec fieldSpec = _schema.getFieldSpecFor(columnName);
     if (fieldSpec.isVirtualColumn()) {
-      VirtualColumnContext virtualColumnContext = new VirtualColumnContext(fieldSpec, _numDocsIndexed);
-      VirtualColumnProvider virtualColumnProvider =
-          VirtualColumnProviderFactory.buildProvider(_schema.getFieldSpecFor(columnName).getVirtualColumnProvider());
+      // FIXME
+      VirtualColumnContext virtualColumnContext = new VirtualColumnContext(fieldSpec, _numDocsIndexed, true);
+      VirtualColumnProvider virtualColumnProvider = _virtualColumnProviderMap.get(columnName);
       return new ColumnDataSource(virtualColumnProvider.buildColumnIndexContainer(virtualColumnContext),
           virtualColumnProvider.buildMetadata(virtualColumnContext));
     } else {
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/mutable/MutableUpsertSegmentImpl.java b/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/mutable/MutableUpsertSegmentImpl.java
new file mode 100644
index 0000000..a6e7728
--- /dev/null
+++ b/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/mutable/MutableUpsertSegmentImpl.java
@@ -0,0 +1,159 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.core.indexsegment.mutable;
+
+import com.google.common.base.Preconditions;
+import org.apache.pinot.core.indexsegment.UpsertSegment;
+import org.apache.pinot.core.io.reader.DataFileReader;
+import org.apache.pinot.core.realtime.impl.RealtimeSegmentConfig;
+import org.apache.pinot.core.segment.updater.UpsertWaterMarkManager;
+import org.apache.pinot.core.segment.virtualcolumn.mutable.VirtualColumnLongValueReaderWriter;
+import org.apache.pinot.grigio.common.messages.LogEventType;
+import org.apache.pinot.grigio.common.storageProvider.UpdateLogEntry;
+import org.apache.pinot.grigio.common.storageProvider.UpdateLogEntrySet;
+import org.apache.pinot.grigio.common.storageProvider.UpdateLogStorageProvider;
+import org.apache.pinot.spi.data.readers.GenericRow;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+public class MutableUpsertSegmentImpl extends MutableSegmentImpl implements UpsertSegment {
+
+  private static final Logger LOGGER = LoggerFactory.getLogger(MutableUpsertSegmentImpl.class);
+  private final UpsertWaterMarkManager upsertWaterMarkManager;
+
+  private final String _kafkaOffsetColumnName;
+
+  private final List<VirtualColumnLongValueReaderWriter> _mutableSegmentReaderWriters = new ArrayList<>();
+  // use map for mapping between kafka offset and docId because we at-most have 1 mutable segment per consumer
+  // will use more memory, but we can update this later
+  private final Map<Long, Integer> _sourceOffsetToDocId = new ConcurrentHashMap<>();
+  // to store the update event that arrive before my current record
+  // TODO remove this in the later version of design if necessary
+  private final Map<Long, UpdateLogEntry> _unmatchedInsertRecords = new ConcurrentHashMap<>();
+  private final Map<Long, UpdateLogEntry> _unmatchedDeleteRecords = new ConcurrentHashMap<>();
+
+  public MutableUpsertSegmentImpl(RealtimeSegmentConfig config) {
+    super(config);
+    _kafkaOffsetColumnName = _schema.getOffsetKey();
+    Preconditions.checkState(_schema.isTableForUpsert(), "table should be upsert");
+    for (Map.Entry<String, DataFileReader> entry: _virtualColumnIndexReader.entrySet()) {
+      String column = entry.getKey();
+      DataFileReader reader = entry.getValue();
+      if (reader instanceof VirtualColumnLongValueReaderWriter) {
+        _logger.info("adding virtual column {} to updatable reader writer list", column);
+        _mutableSegmentReaderWriters.add((VirtualColumnLongValueReaderWriter) reader);
+      }
+    }
+    upsertWaterMarkManager = UpsertWaterMarkManager.getInstance();
+    LOGGER.info("starting upsert segment with {} reader writer", _mutableSegmentReaderWriters.size());
+  }
+
+  @Override
+  public synchronized void updateVirtualColumn(Iterable<UpdateLogEntry> logEntries) {
+    for (UpdateLogEntry logEntry: logEntries) {
+      boolean updated = false;
+      boolean offsetFound = false;
+      Integer docId = _sourceOffsetToDocId.get(logEntry.getOffset());
+      if (docId != null) {
+        offsetFound = true;
+        for (VirtualColumnLongValueReaderWriter readerWriter : _mutableSegmentReaderWriters) {
+          updated = readerWriter.update(docId, logEntry.getValue(), logEntry.getType()) || updated;
+        }
+        if (updated) {
+          // only update high water mark if it indeed updated something
+          upsertWaterMarkManager.processMessage(_tableName, _segmentName, logEntry);
+        }
+      }
+      if (!offsetFound) {
+        putEntryToUnmatchMap(logEntry);
+      }
+    }
+  }
+
+  @Override
+  public String getVirtualColumnInfo(long offset) {
+    Integer docId = _sourceOffsetToDocId.get(offset);
+    StringBuilder result = new StringBuilder("matched: ");
+    if (docId == null) {
+      result = new StringBuilder("no doc id found ");
+    } else {
+      for (VirtualColumnLongValueReaderWriter readerWriter : _mutableSegmentReaderWriters) {
+        result.append(readerWriter.getLong(docId)).append("; ");
+      }
+    }
+    if (_unmatchedInsertRecords.containsKey(offset)) {
+      result.append(" unmatched insert: ").append(_unmatchedInsertRecords.get(offset).getValue());
+    }
+    if (_unmatchedDeleteRecords.containsKey(offset)) {
+      result.append(" unmatched delete: ").append(_unmatchedDeleteRecords.get(offset).getValue());
+    }
+    return result.toString();
+  }
+
+  @Override
+  protected synchronized void postProcessRecords(GenericRow row, int docId) {
+    final Long offset = (Long) row.getValue(_kafkaOffsetColumnName);
+    for (VirtualColumnLongValueReaderWriter readerWriter: _mutableSegmentReaderWriters) {
+      readerWriter.addNewRecord(docId);
+    }
+    _sourceOffsetToDocId.put(offset, docId);
+    checkForOutstandingRecords(_unmatchedDeleteRecords, offset, docId);
+    checkForOutstandingRecords(_unmatchedInsertRecords, offset, docId);
+  }
+
+  @Override
+  public void initVirtualColumn() throws IOException {
+    Preconditions.checkState(_numDocsIndexed == 0, "should init virtual column before ingestion");
+    UpdateLogEntrySet updateLogEntries = UpdateLogStorageProvider.getInstance().getAllMessages(_tableName, _segmentName);
+    LOGGER.info("got {} update log entries for current segment {}", updateLogEntries.size(), _segmentName);
+    // some physical data might have been ingested when we init virtual column, we will go through the normal update
+    // flow to ensure we wont miss records
+    updateVirtualColumn(updateLogEntries);
+  }
+
+  private void checkForOutstandingRecords(Map<Long, UpdateLogEntry> unmatchRecordsMap, Long offset, int docId) {
+    UpdateLogEntry unmatchedEntry = unmatchRecordsMap.remove(offset);
+    if (unmatchedEntry != null) {
+      boolean updated = false;
+      for (VirtualColumnLongValueReaderWriter readerWriter: _mutableSegmentReaderWriters) {
+        updated = readerWriter.update(docId, unmatchedEntry.getValue(), unmatchedEntry.getType()) || updated;
+      }
+      if (updated) {
+        upsertWaterMarkManager.processMessage(_tableName, _segmentName, unmatchedEntry);
+      }
+    }
+  }
+
+  private void putEntryToUnmatchMap(UpdateLogEntry logEntry) {
+    final Map<Long, UpdateLogEntry> unmatchedRecords;
+    if (logEntry.getType() == LogEventType.INSERT) {
+      unmatchedRecords = _unmatchedInsertRecords;
+    } else {
+      unmatchedRecords = _unmatchedDeleteRecords;
+    }
+    unmatchedRecords.put(logEntry.getOffset(), logEntry);
+  }
+
+}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/realtime/impl/RealtimeSegmentConfig.java b/pinot-core/src/main/java/org/apache/pinot/core/realtime/impl/RealtimeSegmentConfig.java
index 9aef2a2..827b0f7 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/realtime/impl/RealtimeSegmentConfig.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/realtime/impl/RealtimeSegmentConfig.java
@@ -27,6 +27,7 @@ import org.apache.pinot.core.io.readerwriter.PinotDataBufferMemoryManager;
 
 
 public class RealtimeSegmentConfig {
+  private final String _tableName;
   private final String _segmentName;
   private final String _streamName;
   private final Schema _schema;
@@ -45,12 +46,13 @@ public class RealtimeSegmentConfig {
   private final boolean _nullHandlingEnabled;
   private final String _consumerDir;
 
-  private RealtimeSegmentConfig(String segmentName, String streamName, Schema schema, int capacity,
+  private RealtimeSegmentConfig(String tableName, String segmentName, String streamName, Schema schema, int capacity,
       int avgNumMultiValues, Set<String> noDictionaryColumns, Set<String> varLengthDictionaryColumns,
       Set<String> invertedIndexColumns, Set<String> textIndexColumns, RealtimeSegmentZKMetadata realtimeSegmentZKMetadata,
       boolean offHeap, PinotDataBufferMemoryManager memoryManager, RealtimeSegmentStatsHistory statsHistory,
       SegmentPartitionConfig segmentPartitionConfig, boolean aggregateMetrics, boolean nullHandlingEnabled,
       String consumerDir) {
+    _tableName = tableName;
     _segmentName = segmentName;
     _streamName = streamName;
     _schema = schema;
@@ -70,6 +72,10 @@ public class RealtimeSegmentConfig {
     _consumerDir = consumerDir;
   }
 
+  public String getTableName() {
+    return _tableName;
+  }
+
   public String getSegmentName() {
     return _segmentName;
   }
@@ -144,6 +150,7 @@ public class RealtimeSegmentConfig {
   }
 
   public static class Builder {
+    private String _tableName;
     private String _segmentName;
     private String _streamName;
     private Schema _schema;
@@ -165,6 +172,11 @@ public class RealtimeSegmentConfig {
     public Builder() {
     }
 
+    public Builder setTableName(String tableName) {
+      _tableName = tableName;
+      return this;
+    }
+
     public Builder setSegmentName(String segmentName) {
       _segmentName = segmentName;
       return this;
@@ -259,7 +271,7 @@ public class RealtimeSegmentConfig {
     }
 
     public RealtimeSegmentConfig build() {
-      return new RealtimeSegmentConfig(_segmentName, _streamName, _schema, _capacity, _avgNumMultiValues,
+      return new RealtimeSegmentConfig(_tableName, _segmentName, _streamName, _schema, _capacity, _avgNumMultiValues,
           _noDictionaryColumns, _varLengthDictionaryColumns, _invertedIndexColumns, _textIndexColumns,
           _realtimeSegmentZKMetadata, _offHeap, _memoryManager, _statsHistory, _segmentPartitionConfig,
           _aggregateMetrics, _nullHandlingEnabled, _consumerDir);
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/segment/index/SegmentMetadataImpl.java b/pinot-core/src/main/java/org/apache/pinot/core/segment/index/SegmentMetadataImpl.java
index 7a3dad3..43d28e5 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/segment/index/SegmentMetadataImpl.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/segment/index/SegmentMetadataImpl.java
@@ -22,6 +22,15 @@ import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.node.ArrayNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.google.common.base.Preconditions;
+import org.apache.commons.configuration.ConfigurationException;
+import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.commons.lang.StringEscapeUtils;
+import org.joda.time.Duration;
+import org.joda.time.Interval;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.Nullable;
 import java.io.DataInputStream;
 import java.io.File;
 import java.io.FileInputStream;
@@ -39,10 +48,6 @@ import java.util.Map;
 import java.util.Set;
 import java.util.TimeZone;
 import java.util.concurrent.TimeUnit;
-import javax.annotation.Nullable;
-import org.apache.commons.configuration.ConfigurationException;
-import org.apache.commons.configuration.PropertiesConfiguration;
-import org.apache.commons.lang.StringEscapeUtils;
 import org.apache.pinot.spi.data.MetricFieldSpec;
 import org.apache.pinot.spi.data.Schema;
 import org.apache.pinot.common.metadata.segment.RealtimeSegmentZKMetadata;
@@ -58,10 +63,6 @@ import org.apache.pinot.core.startree.v2.StarTreeV2Constants;
 import org.apache.pinot.core.startree.v2.StarTreeV2Metadata;
 import org.apache.pinot.startree.hll.HllConstants;
 import org.joda.time.DateTimeZone;
-import org.joda.time.Duration;
-import org.joda.time.Interval;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 import static org.apache.pinot.core.segment.creator.impl.V1Constants.MetadataKeys.Segment.*;
 import static org.apache.pinot.core.segment.creator.impl.V1Constants.MetadataKeys.StarTree.*;
@@ -108,6 +109,15 @@ public class SegmentMetadataImpl implements SegmentMetadata {
    */
   public SegmentMetadataImpl(File indexDir)
       throws IOException {
+    this(indexDir, null);
+  }
+
+   /**
+   * For segments on disk and segment is upsert, will use the sechema hint to init virtual column
+   * <p>Index directory passed in should be top level segment directory.
+   * <p>If segment metadata file exists in multiple segment version, load the one in highest segment version.
+   */
+  public SegmentMetadataImpl(File indexDir, Schema schemaHint) throws IOException {
     _indexDir = indexDir;
     PropertiesConfiguration segmentMetadataPropertiesConfiguration = getPropertiesConfiguration(indexDir);
     _columnMetadataMap = new HashMap<>();
@@ -115,6 +125,9 @@ public class SegmentMetadataImpl implements SegmentMetadata {
     _schema = new Schema();
 
     init(segmentMetadataPropertiesConfiguration);
+    // adding necessary schema hints for upsert table
+    _schema.withSchemaHint(schemaHint);
+
     File creationMetaFile = SegmentDirectoryPaths.findCreationMetaFile(indexDir);
     if (creationMetaFile != null) {
       loadCreationMeta(creationMetaFile);
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/VirtualColumnContext.java b/pinot-core/src/main/java/org/apache/pinot/core/segment/updater/SegmentDeletionListener.java
similarity index 55%
copy from pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/VirtualColumnContext.java
copy to pinot-core/src/main/java/org/apache/pinot/core/segment/updater/SegmentDeletionListener.java
index bb32320..a29feb7 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/VirtualColumnContext.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/segment/updater/SegmentDeletionListener.java
@@ -16,29 +16,8 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.pinot.core.segment.virtualcolumn;
+package org.apache.pinot.core.segment.updater;
 
-import org.apache.pinot.spi.data.FieldSpec;
-
-
-/**
- * Miscellaneous context information about the virtual column.
- * It will be used to build various components (dictionary, reader, etc) in the virtual column provider.
- */
-public class VirtualColumnContext {
-  private FieldSpec _fieldSpec;
-  private int _totalDocCount;
-
-  public VirtualColumnContext(FieldSpec fieldSpec, int totalDocCount) {
-    _fieldSpec = fieldSpec;
-    _totalDocCount = totalDocCount;
-  }
-
-  public FieldSpec getFieldSpec() {
-    return _fieldSpec;
-  }
-
-  public int getTotalDocCount() {
-    return _totalDocCount;
-  }
+public interface SegmentDeletionListener {
+  void onSegmentDeletion(String tableNameWithType, String segmentName);
 }
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/segment/updater/SegmentUpdater.java b/pinot-core/src/main/java/org/apache/pinot/core/segment/updater/SegmentUpdater.java
new file mode 100644
index 0000000..d0d8c0e
--- /dev/null
+++ b/pinot-core/src/main/java/org/apache/pinot/core/segment/updater/SegmentUpdater.java
@@ -0,0 +1,349 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.core.segment.updater;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.common.util.concurrent.Uninterruptibles;
+import io.netty.util.internal.ConcurrentSet;
+import org.apache.commons.configuration.Configuration;
+import org.apache.pinot.common.config.TableNameBuilder;
+import org.apache.pinot.common.utils.CommonConstants;
+import org.apache.pinot.common.utils.LLCSegmentName;
+import org.apache.pinot.core.data.manager.UpsertSegmentDataManager;
+import org.apache.pinot.grigio.common.messages.LogCoordinatorMessage;
+import org.apache.pinot.grigio.common.metrics.GrigioMeter;
+import org.apache.pinot.grigio.common.metrics.GrigioMetrics;
+import org.apache.pinot.grigio.common.metrics.GrigioTimer;
+import org.apache.pinot.grigio.common.rpcQueue.QueueConsumer;
+import org.apache.pinot.grigio.common.rpcQueue.QueueConsumerRecord;
+import org.apache.pinot.grigio.common.storageProvider.UpdateLogEntry;
+import org.apache.pinot.grigio.common.storageProvider.UpdateLogStorageProvider;
+import org.apache.pinot.grigio.common.storageProvider.retentionManager.UpdateLogRetentionManager;
+import org.apache.pinot.grigio.common.utils.CommonUtils;
+import org.apache.pinot.grigio.servers.SegmentUpdaterProvider;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.concurrent.ThreadSafe;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * class to perform fetching updates for upsert related information from kafka to local machine and fill in
+ * virtual columns. It should be started after all segments are loaded in current pinot server.
+ */
+@ThreadSafe
+public class SegmentUpdater implements SegmentDeletionListener {
+  private static final Logger LOGGER = LoggerFactory.getLogger(SegmentUpdater.class);
+
+  private static volatile SegmentUpdater _instance = null;
+
+  private static final long NO_MESSAGE_SLEEP_MS = 100;
+  private static final long SHUTDOWN_WAIT_MS = 2000;
+  private static final long LOGGER_TIME_GAP_MS = 5000;
+
+  private final Configuration _conf;
+  private final int _updateSleepMs;
+  private final String _topicPrefix;
+  private final ExecutorService _ingestionExecutorService;
+  private final QueueConsumer _consumer;
+  private final Map<String, Map<String, Set<UpsertSegmentDataManager>>> _tableSegmentMap = new ConcurrentHashMap<>();
+  private final Map<String, Map<Integer, Long>> _tablePartitionCreationTime = new ConcurrentHashMap<>();
+  private final UpdateLogStorageProvider _updateLogStorageProvider;
+  private final UpdateLogRetentionManager _retentionManager;
+  private final GrigioMetrics _metrics;
+
+  private volatile boolean isStarted = true;
+
+  public SegmentUpdater(Configuration conf, SegmentUpdaterProvider provider, UpdateLogRetentionManager retentionManager,
+                        GrigioMetrics metrics) {
+    _conf = conf;
+    _metrics = metrics;
+    _retentionManager = retentionManager;
+    _topicPrefix = conf.getString(SegmentUpdaterConfig.INPUT_TOPIC_PREFIX);
+    _updateSleepMs = conf.getInt(SegmentUpdaterConfig.SEGMENT_UDPATE_SLEEP_MS,
+        SegmentUpdaterConfig.SEGMENT_UDPATE_SLEEP_MS_DEFAULT);
+    UpsertWaterMarkManager.init(metrics);
+    _consumer = provider.getConsumer();
+    _ingestionExecutorService = Executors.newFixedThreadPool(1);
+    _updateLogStorageProvider = UpdateLogStorageProvider.getInstance();
+    _instance = this;
+  }
+
+  public static SegmentUpdater getInstance() {
+    Preconditions.checkState(_instance != null, "there is no instance for segment updater");
+    return _instance;
+  }
+
+
+  public void start() {
+    String listOfTables = Joiner.on(",").join(_tableSegmentMap.keySet());
+    LOGGER.info("starting segment updater main loop with the following table in server: {}", listOfTables);
+    _ingestionExecutorService.submit(this::updateLoop);
+  }
+
+  public void shutdown() {
+    LOGGER.info("closing the segment updater");
+    isStarted = false;
+    _ingestionExecutorService.shutdown();
+    try {
+      _ingestionExecutorService.awaitTermination(SHUTDOWN_WAIT_MS, TimeUnit.MILLISECONDS);
+    } catch (InterruptedException e) {
+      LOGGER.error("failed to wait for shutdown", e);
+    }
+    _ingestionExecutorService.shutdownNow();
+    try {
+      _updateLogStorageProvider.close();
+    } catch (IOException ex) {
+      LOGGER.error("failed to shutdown update log provider", ex);
+    }
+    LOGGER.info("finished shutdown of segment updater service");
+  }
+
+  /**
+   * this method will perform the following:
+   * 1. fetch a list of updates for pinot upsert from kafka consumer
+   * 2. organize the message by table/segment/List<UpdateLogEntry> map
+   * 3. call to save the update data in local file system
+   * 4. apply updates to appropriate data manager
+   * 5. commit consumer offset
+   */
+  private void updateLoop() {
+    try {
+      LOGGER.info("starting update loop");
+      long lastReportedTime = System.currentTimeMillis();
+      while (isStarted) {
+        long startTime = System.currentTimeMillis();
+        long loopStartTime = startTime;
+        final List<QueueConsumerRecord<String, LogCoordinatorMessage>> records = _consumer.getRequests(_updateSleepMs, TimeUnit.MILLISECONDS);
+        _metrics.addTimedValueMs(GrigioTimer.FETCH_MSG_FROM_CONSUMER_TIME, System.currentTimeMillis() - startTime);
+        int eventCount = records.size();
+        _metrics.addMeteredGlobalValue(GrigioMeter.MESSAGE_FETCH_PER_ROUND, eventCount);
+
+        final Map<String, TableUpdateLogs> tableSegmentToUpdateLogs = new HashMap<>();
+        // organize the update logs by {tableName: {segmentName: {list of updatelogs}}}
+        records.iterator().forEachRemaining(consumerRecord -> {
+          TableUpdateLogs tableUpdateLogs = tableSegmentToUpdateLogs.computeIfAbsent(
+              CommonUtils.getTableNameFromKafkaTopic(consumerRecord.getTopic(), _topicPrefix),
+              t -> new TableUpdateLogs());
+          tableUpdateLogs.addUpdateLogEntry(consumerRecord.getRecord(), consumerRecord.getPartition());
+        });
+
+        startTime = System.currentTimeMillis();
+        AtomicLong timeToStoreUpdateLogs = new AtomicLong(0);
+        for (Map.Entry<String, TableUpdateLogs> entry : tableSegmentToUpdateLogs.entrySet()) {
+          String tableName = TableNameBuilder.ensureTableNameWithType(entry.getKey(), CommonConstants.Helix.TableType.REALTIME);
+          int tableMessageCount = 0;
+          if (_tableSegmentMap.containsKey(tableName)) {
+            final Map<String, Set<UpsertSegmentDataManager>> segmentManagersMap = _tableSegmentMap.get(tableName);
+            final TableUpdateLogs segment2UpdateLogsMap = entry.getValue();
+            updateSegmentVirtualColumns(tableName, segmentManagersMap, segment2UpdateLogsMap, timeToStoreUpdateLogs);
+          } else {
+            LOGGER.warn("got messages for table {} not in this server", tableName);
+          }
+          _metrics.addMeteredTableValue(tableName, GrigioMeter.MESSAGE_FETCH_PER_ROUND, tableMessageCount);
+        }
+        _metrics.addTimedValueMs(GrigioTimer.UPDATE_LOCAL_LOG_FILE_TIME, timeToStoreUpdateLogs.get());
+        _metrics.addTimedValueMs(GrigioTimer.UPDATE_DATAMANAGER_TIME, System.currentTimeMillis() - startTime);
+        if (eventCount == 0) {
+          Uninterruptibles.sleepUninterruptibly(NO_MESSAGE_SLEEP_MS, TimeUnit.MILLISECONDS);
+        } else {
+          if (System.currentTimeMillis() - lastReportedTime > LOGGER_TIME_GAP_MS) {
+            lastReportedTime = System.currentTimeMillis();
+            LOGGER.info("processed {} messages in {} ms", eventCount, System.currentTimeMillis() - loopStartTime);
+            LOGGER.info("latest high water mark is {}", UpsertWaterMarkManager.getInstance().toString());
+          }
+          _consumer.ackOffset();
+          _metrics.addTimedValueMs(GrigioTimer.SEGMENT_UPDATER_LOOP_TIME, System.currentTimeMillis() - loopStartTime);
+        }
+      }
+    } catch (Exception ex) {
+      LOGGER.error("failed at segment updates", ex);
+    } finally {
+      LOGGER.info("exiting segment update loop");
+    }
+  }
+
+  private void storeUpdateLogs(String table, String segment, List<UpdateLogEntry> messages,
+                               AtomicLong timeToStoreUpdateLogs) throws IOException {
+    long startTime = System.currentTimeMillis();
+    _updateLogStorageProvider.addDataToFile(table, segment, messages);
+    timeToStoreUpdateLogs.addAndGet(System.currentTimeMillis() - startTime);
+  }
+
+  /**
+   * Update the virtual columns of affected segments of a table.
+   */
+  private void updateSegmentVirtualColumns(String tableName, Map<String, Set<UpsertSegmentDataManager>> segmentManagersMap,
+                                           TableUpdateLogs segment2UpdateLogsMap, AtomicLong timeToStoreUpdateLogs) throws IOException{
+    for (Map.Entry<String, List<UpdateLogEntry>> segmentEntry : segment2UpdateLogsMap.getSegments2UpdateLog().entrySet()) {
+      final String segmentNameStr = segmentEntry.getKey();
+      updateVirtualColumn(tableName, segmentNameStr,
+          segmentManagersMap.computeIfAbsent(segmentNameStr, sn -> new ConcurrentSet<>()),
+          segment2UpdateLogsMap.get(segmentNameStr), timeToStoreUpdateLogs);
+    }
+  }
+
+  /**
+   * in pinot server, there could be multiple segment data managers per table/segment pair during pinot switch a segment
+   * from consuming to online (mutable segment to immutable segment). In most of cases we expect only one segment manager
+   * in this set of UpsertSegmentDataManager
+   */
+  private void updateVirtualColumn(String table, String segment, Set<UpsertSegmentDataManager> segmentDataManagers,
+                                   List<UpdateLogEntry> messages, AtomicLong timeToStoreUpdateLogs) throws IOException {
+    LOGGER.debug("updating segment {} with {} results for {} data managers", segment, messages.size(),
+        segmentDataManagers.size());
+    if (segmentDataManagers.size() > 0 || _retentionManager.getRetentionManagerForTable(table).shouldIngestForSegment(segment)) {
+      storeUpdateLogs(table, segment, messages, timeToStoreUpdateLogs);
+    }
+    try {
+      for (UpsertSegmentDataManager dataManager: segmentDataManagers) {
+        dataManager.updateVirtualColumns(messages);
+      }
+    } catch (Exception ex) {
+      LOGGER.error("failed to update virtual column for key ", ex);
+    }
+  }
+
+  /**
+   * called when we create a new segment data manager, associate this data manager with the given table/segment info
+   * @param tableNameWithType
+   * @param segmentName
+   * @param dataManager the data manager for the current given table/segment combination
+   */
+  public synchronized void addSegmentDataManager(String tableNameWithType, LLCSegmentName segmentName, UpsertSegmentDataManager dataManager) {
+    // TODO get partition assignment from
+    LOGGER.info("segment updater adding table {} segment {}", tableNameWithType, segmentName.getSegmentName());
+    if (!_tableSegmentMap.containsKey(tableNameWithType)) {
+      synchronized (_tableSegmentMap) {
+        _tableSegmentMap.put(tableNameWithType, new ConcurrentHashMap<>());
+      }
+      LOGGER.info("adding table {} to segment updater consumer", tableNameWithType);
+      handleNewTableInServer(tableNameWithType);
+    }
+    _tableSegmentMap.get(tableNameWithType).computeIfAbsent(segmentName.getSegmentName(), sn -> new HashSet<>()).add(dataManager);
+    synchronized (_tablePartitionCreationTime) {
+      long creationTime = _tablePartitionCreationTime.computeIfAbsent(tableNameWithType, t -> new ConcurrentHashMap<>())
+          .computeIfAbsent(segmentName.getPartitionId(), p -> segmentName.getCreationTimeStamp());
+      _tablePartitionCreationTime.get(tableNameWithType)
+          .put(segmentName.getPartitionId(), Long.max(creationTime, segmentName.getCreationTimeStamp()));
+    }
+  }
+
+  public synchronized void removeSegmentDataManager(String tableNameWithType, String segmentName, UpsertSegmentDataManager toDeleteManager) {
+    LOGGER.info("segment updater removing table {} segment {}", tableNameWithType, segmentName);
+    Map<String, Set<UpsertSegmentDataManager>> segmentMap = _tableSegmentMap.get(tableNameWithType);
+    if (segmentMap != null) {
+      Set<UpsertSegmentDataManager> segmentDataManagers = segmentMap.get(segmentName);
+      if (segmentDataManagers != null) {
+        segmentDataManagers.remove(toDeleteManager);
+        if (segmentDataManagers.size() == 0) {
+          segmentMap.remove(segmentName);
+        }
+      }
+    }
+  }
+
+  /**
+   * handle how to read update logs we need to do for adding a new pinot table
+   * need to do the following:
+   * subscribe to table update log kafka topics
+   * @param tableNameWithType the name of the table without
+   */
+  private void handleNewTableInServer(String tableNameWithType) {
+    LOGGER.info("subscribing to new table {}", tableNameWithType);
+    // init the retention manager to ensure we get the first ideal state
+    _retentionManager.getRetentionManagerForTable(tableNameWithType);
+    _consumer.subscribeForTable(TableNameBuilder.extractRawTableName(tableNameWithType), _topicPrefix);
+  }
+
+  /**
+   * handle clean up when a table no longer has any segment in the current server
+   * @param tableNameWithType
+   */
+  private void handleTableRemovalInServer(String tableNameWithType) {
+    LOGGER.info("unsubscribe to old table {}", tableNameWithType);
+    // key coordinator generate message without table name
+    _consumer.unsubscribeForTable(TableNameBuilder.extractRawTableName(tableNameWithType), _topicPrefix);
+  }
+
+  @Override
+  public synchronized void onSegmentDeletion(String tableNameWithType, String segmentName) {
+    LOGGER.info("deleting segment virtual column from local storage for table {} segment {}", tableNameWithType, segmentName);
+    Map<String, Set<UpsertSegmentDataManager>> segmentManagerMap = _tableSegmentMap.get(tableNameWithType);
+    if (segmentManagerMap != null) {
+      if (segmentManagerMap.containsKey(segmentName) && segmentManagerMap.get(segmentName).size() > 0) {
+        LOGGER.error("trying to remove segment storage with {} segment data manager", segmentManagerMap.get(segmentName).size());
+      }
+      try {
+        segmentManagerMap.remove(segmentName);
+        _retentionManager.getRetentionManagerForTable(tableNameWithType).notifySegmentDeletion(tableNameWithType);
+        _updateLogStorageProvider.removeSegment(tableNameWithType, segmentName);
+      } catch (IOException e) {
+        throw new RuntimeException(String.format("failed to delete table %s segment %s", tableNameWithType, segmentName), e);
+      }
+      if (segmentManagerMap.size() == 0) {
+        _tableSegmentMap.remove(tableNameWithType);
+        handleTableRemovalInServer(tableNameWithType);
+      }
+    } else {
+      LOGGER.error("deleting a segment {}:{} from current server but don't have segment map on updater",
+          tableNameWithType, segmentName);
+    }
+  }
+
+  // A table's update logs grouped by segment names.
+  private class TableUpdateLogs {
+    private Map<String, List<UpdateLogEntry>> _segments2UpdateLog;
+
+    public TableUpdateLogs() {
+                                 _segments2UpdateLog = new HashMap<>();
+                                                                       }
+
+    public Map<String, List<UpdateLogEntry>> getSegments2UpdateLog() {
+      return Collections.unmodifiableMap(_segments2UpdateLog);
+    }
+
+    // Partition is the partition where the record appears in the Segment Update Event message queue.
+    public void addUpdateLogEntry(LogCoordinatorMessage record, int partition) {
+      if (record == null || record.getSegmentName() == null) {
+        LOGGER.error("Empty update log or no segment in the entry: {}", record);
+        return;
+      }
+      _segments2UpdateLog.computeIfAbsent(
+              record.getSegmentName(), s -> new ArrayList<>()).add(new UpdateLogEntry(record, partition));
+    }
+
+    public List<UpdateLogEntry> get(String segmentNameStr) {
+      return Collections.unmodifiableList(_segments2UpdateLog.get(segmentNameStr));
+    }
+  }
+}
\ No newline at end of file
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/VirtualColumnContext.java b/pinot-core/src/main/java/org/apache/pinot/core/segment/updater/SegmentUpdaterConfig.java
similarity index 55%
copy from pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/VirtualColumnContext.java
copy to pinot-core/src/main/java/org/apache/pinot/core/segment/updater/SegmentUpdaterConfig.java
index bb32320..97d48f3 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/VirtualColumnContext.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/segment/updater/SegmentUpdaterConfig.java
@@ -16,29 +16,10 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.pinot.core.segment.virtualcolumn;
+package org.apache.pinot.core.segment.updater;
 
-import org.apache.pinot.spi.data.FieldSpec;
-
-
-/**
- * Miscellaneous context information about the virtual column.
- * It will be used to build various components (dictionary, reader, etc) in the virtual column provider.
- */
-public class VirtualColumnContext {
-  private FieldSpec _fieldSpec;
-  private int _totalDocCount;
-
-  public VirtualColumnContext(FieldSpec fieldSpec, int totalDocCount) {
-    _fieldSpec = fieldSpec;
-    _totalDocCount = totalDocCount;
-  }
-
-  public FieldSpec getFieldSpec() {
-    return _fieldSpec;
-  }
-
-  public int getTotalDocCount() {
-    return _totalDocCount;
-  }
+public class SegmentUpdaterConfig {
+  public static final String SEGMENT_UDPATE_SLEEP_MS = "sleep.ms";
+  public static final String INPUT_TOPIC_PREFIX  = "input.topic.prefix";
+  public static final int SEGMENT_UDPATE_SLEEP_MS_DEFAULT = 100;
 }
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/segment/updater/UpsertWaterMarkManager.java b/pinot-core/src/main/java/org/apache/pinot/core/segment/updater/UpsertWaterMarkManager.java
new file mode 100644
index 0000000..bbea037
--- /dev/null
+++ b/pinot-core/src/main/java/org/apache/pinot/core/segment/updater/UpsertWaterMarkManager.java
@@ -0,0 +1,90 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.core.segment.updater;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
+import org.apache.pinot.grigio.common.metrics.GrigioGauge;
+import org.apache.pinot.grigio.common.metrics.GrigioMetrics;
+import org.apache.pinot.grigio.common.storageProvider.UpdateLogEntry;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+public class UpsertWaterMarkManager {
+
+  private final Map<String, Map<Integer, Long>> _highWaterMarkTablePartitionMap = new ConcurrentHashMap<>();
+  private final GrigioMetrics _metrics;
+
+  private static final Logger LOGGER = LoggerFactory.getLogger(UpsertWaterMarkManager.class);
+  private static volatile UpsertWaterMarkManager _instance;
+
+  private UpsertWaterMarkManager(GrigioMetrics metrics) {
+    _metrics = metrics;
+  }
+
+  public static void init(GrigioMetrics metrics) {
+    synchronized (UpsertWaterMarkManager.class) {
+      Preconditions.checkState(_instance == null, "upsert water mark manager is already init");
+      _instance = new UpsertWaterMarkManager(metrics);
+    }
+  }
+
+  public static UpsertWaterMarkManager getInstance() {
+    Preconditions.checkState(_instance != null, "upsert water mark manager is not yet init");
+    return _instance;
+  }
+
+  // TODO(tingchen) Look into the case where Segment Update Messages might arrive before the corresponding physical data.
+  public void processMessage(String table, String segment, UpdateLogEntry logEntry) {
+    if (logEntry == null) {
+      return;
+    }
+    long version = logEntry.getValue();
+    int partition = logEntry.getPartition();
+    processVersionUpdate(table, partition, version);
+  }
+
+  public void processVersionUpdate(String table, int partition, long version) {
+    Preconditions.checkState(partition >= 0, "logEntry has invalid version {} for table {}",
+        version, table);
+    Map<Integer, Long> partitionToHighWaterMark = _highWaterMarkTablePartitionMap.computeIfAbsent(table, t -> new ConcurrentHashMap<>());
+    long currentVersion = partitionToHighWaterMark.getOrDefault(partition, -1L);  // assumes that valid version is non-negative
+    if (version > currentVersion) {
+      partitionToHighWaterMark.put(partition, version);
+      _metrics.setValueOfTableGauge(String.valueOf(partition), GrigioGauge.SERVER_VERSION_CONSUMED, version);
+    }
+  }
+
+  public Map<Integer, Long> getHighWaterMarkForTable(String tableName) {
+    return ImmutableMap.copyOf(_highWaterMarkTablePartitionMap.getOrDefault(tableName, ImmutableMap.of()));
+  }
+
+  public Map<String, Map<Integer, Long>> getHighWaterMarkTablePartitionMap() {
+    return ImmutableMap.copyOf(_highWaterMarkTablePartitionMap);
+  }
+
+  @Override
+  public String toString() {
+    return Joiner.on(",").withKeyValueSeparator("=").join(_highWaterMarkTablePartitionMap);
+  }
+}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/BaseVirtualColumnProvider.java b/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/BaseVirtualColumnProvider.java
index ece8fa7..c1c7c62 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/BaseVirtualColumnProvider.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/BaseVirtualColumnProvider.java
@@ -21,6 +21,9 @@ package org.apache.pinot.core.segment.virtualcolumn;
 import org.apache.pinot.core.segment.index.ColumnMetadata;
 import org.apache.pinot.core.segment.index.column.ColumnIndexContainer;
 import org.apache.pinot.spi.data.FieldSpec;
+import org.apache.pinot.core.io.reader.DataFileReader;
+import org.apache.pinot.core.segment.index.readers.Dictionary;
+import org.apache.pinot.core.segment.index.readers.InvertedIndexReader;
 
 
 /**
@@ -39,4 +42,19 @@ public abstract class BaseVirtualColumnProvider implements VirtualColumnProvider
   public ColumnIndexContainer buildColumnIndexContainer(VirtualColumnContext context) {
     return new VirtualColumnIndexContainer(buildReader(context), buildInvertedIndex(context), buildDictionary(context));
   }
+
+  @Override
+  public ColumnIndexContainer buildColumnIndexContainer(VirtualColumnContext context, DataFileReader fileReader,
+                                                        Dictionary dictionary, InvertedIndexReader invertedIndexReader) {
+    if (fileReader == null) {
+      fileReader = buildReader(context);
+    }
+    if (dictionary == null) {
+      dictionary = buildDictionary(context);
+    }
+    if (invertedIndexReader == null) {
+      invertedIndexReader = buildInvertedIndex(context);
+    }
+    return new VirtualColumnIndexContainer(fileReader, invertedIndexReader, dictionary);
+  }
 }
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/VirtualColumnContext.java b/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/VirtualColumnContext.java
index bb32320..abfccf0 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/VirtualColumnContext.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/VirtualColumnContext.java
@@ -28,10 +28,12 @@ import org.apache.pinot.spi.data.FieldSpec;
 public class VirtualColumnContext {
   private FieldSpec _fieldSpec;
   private int _totalDocCount;
+  private boolean _isMutableSegment;
 
-  public VirtualColumnContext(FieldSpec fieldSpec, int totalDocCount) {
+  public VirtualColumnContext(FieldSpec fieldSpec, int totalDocCount, boolean isMutableSegment) {
     _fieldSpec = fieldSpec;
     _totalDocCount = totalDocCount;
+    _isMutableSegment = isMutableSegment;
   }
 
   public FieldSpec getFieldSpec() {
@@ -41,4 +43,8 @@ public class VirtualColumnContext {
   public int getTotalDocCount() {
     return _totalDocCount;
   }
+
+  public boolean isMutableSegment() {
+    return _isMutableSegment;
+  }
 }
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/VirtualColumnProvider.java b/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/VirtualColumnProvider.java
index 8a1e3f5..aed96bd 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/VirtualColumnProvider.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/VirtualColumnProvider.java
@@ -30,6 +30,7 @@ import org.apache.pinot.core.segment.index.readers.InvertedIndexReader;
  * comprise a proper column.
  */
 public interface VirtualColumnProvider {
+
   DataFileReader buildReader(VirtualColumnContext context);
 
   Dictionary buildDictionary(VirtualColumnContext context);
@@ -39,4 +40,7 @@ public interface VirtualColumnProvider {
   InvertedIndexReader buildInvertedIndex(VirtualColumnContext context);
 
   ColumnIndexContainer buildColumnIndexContainer(VirtualColumnContext context);
+
+  ColumnIndexContainer buildColumnIndexContainer(VirtualColumnContext context, DataFileReader reader,
+                                                 Dictionary dictionary, InvertedIndexReader invertedIndexReader);
 }
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/mutable/BaseLongVirtualColumnProvider.java b/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/mutable/BaseLongVirtualColumnProvider.java
new file mode 100644
index 0000000..fc69a9f
--- /dev/null
+++ b/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/mutable/BaseLongVirtualColumnProvider.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.core.segment.virtualcolumn.mutable;
+
+import org.apache.pinot.core.segment.index.ColumnMetadata;
+import org.apache.pinot.core.segment.index.readers.Dictionary;
+import org.apache.pinot.core.segment.index.readers.InvertedIndexReader;
+import org.apache.pinot.core.segment.virtualcolumn.BaseVirtualColumnProvider;
+import org.apache.pinot.core.segment.virtualcolumn.VirtualColumnContext;
+import org.apache.pinot.spi.data.FieldSpec;
+
+/**
+ * base class for upsert related virtual column (validFrom and validUntil virtual column)
+ * this only support the direct access through the reader but not the dictionary or inverted index
+ */
+public abstract class BaseLongVirtualColumnProvider extends BaseVirtualColumnProvider {
+
+  @Override
+  public Dictionary buildDictionary(VirtualColumnContext context) {
+    return null;
+  }
+
+  @Override
+  public ColumnMetadata buildMetadata(VirtualColumnContext context) {
+    ColumnMetadata.Builder columnMetadataBuilder = new ColumnMetadata.Builder()
+        .setCardinality(1) // TODO: double check on this
+        .setHasDictionary(false)
+        .setHasInvertedIndex(false)
+        .setFieldType(FieldSpec.FieldType.DIMENSION)
+        .setDataType(FieldSpec.DataType.LONG)
+        .setSingleValue(true)
+        .setIsSorted(false)
+        .setTotalDocs(context.getTotalDocCount());
+    return columnMetadataBuilder.build();
+  }
+
+  @Override
+  public InvertedIndexReader buildInvertedIndex(VirtualColumnContext context) {
+    return null;
+  }
+}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/VirtualColumnContext.java b/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/mutable/BaseVirtualColumnSingleValueReaderWriter.java
similarity index 56%
copy from pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/VirtualColumnContext.java
copy to pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/mutable/BaseVirtualColumnSingleValueReaderWriter.java
index bb32320..fdcfe25 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/VirtualColumnContext.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/mutable/BaseVirtualColumnSingleValueReaderWriter.java
@@ -16,29 +16,14 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.pinot.core.segment.virtualcolumn;
+package org.apache.pinot.core.segment.virtualcolumn.mutable;
 
-import org.apache.pinot.spi.data.FieldSpec;
+import org.apache.pinot.core.io.reader.BaseSingleColumnSingleValueReader;
+import org.apache.pinot.core.io.reader.ReaderContext;
 
+public abstract class BaseVirtualColumnSingleValueReaderWriter<E extends ReaderContext> extends BaseSingleColumnSingleValueReader<E> {
 
-/**
- * Miscellaneous context information about the virtual column.
- * It will be used to build various components (dictionary, reader, etc) in the virtual column provider.
- */
-public class VirtualColumnContext {
-  private FieldSpec _fieldSpec;
-  private int _totalDocCount;
-
-  public VirtualColumnContext(FieldSpec fieldSpec, int totalDocCount) {
-    _fieldSpec = fieldSpec;
-    _totalDocCount = totalDocCount;
-  }
-
-  public FieldSpec getFieldSpec() {
-    return _fieldSpec;
-  }
-
-  public int getTotalDocCount() {
-    return _totalDocCount;
+  public void addNewRecord(int docId) {
+    // default do nothing
   }
 }
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/VirtualColumnProvider.java b/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/mutable/ValidFromInMemoryVirtualColumnProvider.java
similarity index 50%
copy from pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/VirtualColumnProvider.java
copy to pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/mutable/ValidFromInMemoryVirtualColumnProvider.java
index 8a1e3f5..ebb45e7 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/VirtualColumnProvider.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/mutable/ValidFromInMemoryVirtualColumnProvider.java
@@ -16,27 +16,30 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.pinot.core.segment.virtualcolumn;
+package org.apache.pinot.core.segment.virtualcolumn.mutable;
 
 import org.apache.pinot.core.io.reader.DataFileReader;
-import org.apache.pinot.core.segment.index.ColumnMetadata;
-import org.apache.pinot.core.segment.index.column.ColumnIndexContainer;
-import org.apache.pinot.core.segment.index.readers.Dictionary;
-import org.apache.pinot.core.segment.index.readers.InvertedIndexReader;
+import org.apache.pinot.core.segment.virtualcolumn.VirtualColumnContext;
+import org.apache.pinot.grigio.common.messages.LogEventType;
 
+public class ValidFromInMemoryVirtualColumnProvider extends BaseLongVirtualColumnProvider {
 
-/**
- * Virtual column provider interface, which is used to instantiate the various components (dictionary, reader, etc) that
- * comprise a proper column.
- */
-public interface VirtualColumnProvider {
-  DataFileReader buildReader(VirtualColumnContext context);
-
-  Dictionary buildDictionary(VirtualColumnContext context);
-
-  ColumnMetadata buildMetadata(VirtualColumnContext context);
-
-  InvertedIndexReader buildInvertedIndex(VirtualColumnContext context);
+  @Override
+  public DataFileReader buildReader(VirtualColumnContext context) {
+    return new VirtualColumnLongValueReaderWriter(context) {
+      @Override
+      public boolean update(int docId, int value, LogEventType eventType) {
+        if (eventType == LogEventType.INSERT) {
+          updateValue(docId, value);
+          return true;
+        }
+        return false;
+      }
 
-  ColumnIndexContainer buildColumnIndexContainer(VirtualColumnContext context);
+      @Override
+      public boolean update(int docId, long value, LogEventType eventType) {
+        return update(docId, Math.toIntExact(value), eventType);
+      }
+    };
+  }
 }
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/VirtualColumnProvider.java b/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/mutable/ValidUntilInMemoryVirtualColumnProvider.java
similarity index 50%
copy from pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/VirtualColumnProvider.java
copy to pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/mutable/ValidUntilInMemoryVirtualColumnProvider.java
index 8a1e3f5..9aef94c 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/VirtualColumnProvider.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/mutable/ValidUntilInMemoryVirtualColumnProvider.java
@@ -16,27 +16,30 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.pinot.core.segment.virtualcolumn;
+package org.apache.pinot.core.segment.virtualcolumn.mutable;
 
 import org.apache.pinot.core.io.reader.DataFileReader;
-import org.apache.pinot.core.segment.index.ColumnMetadata;
-import org.apache.pinot.core.segment.index.column.ColumnIndexContainer;
-import org.apache.pinot.core.segment.index.readers.Dictionary;
-import org.apache.pinot.core.segment.index.readers.InvertedIndexReader;
+import org.apache.pinot.core.segment.virtualcolumn.VirtualColumnContext;
+import org.apache.pinot.grigio.common.messages.LogEventType;
 
+public class ValidUntilInMemoryVirtualColumnProvider extends BaseLongVirtualColumnProvider {
 
-/**
- * Virtual column provider interface, which is used to instantiate the various components (dictionary, reader, etc) that
- * comprise a proper column.
- */
-public interface VirtualColumnProvider {
-  DataFileReader buildReader(VirtualColumnContext context);
-
-  Dictionary buildDictionary(VirtualColumnContext context);
-
-  ColumnMetadata buildMetadata(VirtualColumnContext context);
-
-  InvertedIndexReader buildInvertedIndex(VirtualColumnContext context);
+  @Override
+  public DataFileReader buildReader(VirtualColumnContext context) {
+    return new VirtualColumnLongValueReaderWriter(context) {
+      @Override
+      public boolean update(int docId, int offset, LogEventType eventType) {
+        if (eventType == LogEventType.DELETE) {
+          this.updateValue(docId, offset);
+          return true;
+        }
+        return false;
+      }
 
-  ColumnIndexContainer buildColumnIndexContainer(VirtualColumnContext context);
+      @Override
+      public boolean update(int docId, long offset, LogEventType eventType) {
+        return update(docId, Math.toIntExact(offset), eventType);
+      }
+    };
+  }
 }
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/mutable/VirtualColumnLongValueReaderWriter.java b/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/mutable/VirtualColumnLongValueReaderWriter.java
new file mode 100644
index 0000000..78f3bda
--- /dev/null
+++ b/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/mutable/VirtualColumnLongValueReaderWriter.java
@@ -0,0 +1,108 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.core.segment.virtualcolumn.mutable;
+
+import org.apache.pinot.core.io.reader.impl.ChunkReaderContext;
+import org.apache.pinot.core.segment.virtualcolumn.VirtualColumnContext;
+import org.apache.pinot.grigio.common.messages.LogEventType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+public abstract class VirtualColumnLongValueReaderWriter extends BaseVirtualColumnSingleValueReaderWriter<ChunkReaderContext> {
+
+  private static final Logger LOGGER = LoggerFactory.getLogger(VirtualColumnLongValueReaderWriter.class);
+
+  private int _totalDocSize;
+  private int _currentMaxDocId;
+  private final int [] _values;
+  private final int DEFAULT_NEW_VALUE = -1;
+
+  public VirtualColumnLongValueReaderWriter(VirtualColumnContext context) {
+    this(context.getTotalDocCount(), context.isMutableSegment());
+  }
+
+  public VirtualColumnLongValueReaderWriter(int totalDocSize, boolean isMutableSegment) {
+    _totalDocSize = totalDocSize;
+    _values = new int[_totalDocSize];
+    if (!isMutableSegment) {
+      Arrays.fill(_values, -1);
+    }
+  }
+
+  @Override
+  public synchronized void addNewRecord(int docId) {
+    if (docId >= _totalDocSize) {
+      throw new RuntimeException(String.format("failed to add docId %s larger than allocated size %s", docId, _totalDocSize));
+    }
+    _currentMaxDocId = docId;
+    _values[docId] = DEFAULT_NEW_VALUE;
+  }
+
+  @Override
+  public ChunkReaderContext createContext() {
+    return null;
+  }
+
+  @Override
+  public long getLong(int row) {
+    return _values[row];
+  }
+
+  @Override
+  public long getLong(int rowId, ChunkReaderContext context) {
+    return _values[rowId];
+  }
+
+  @Override
+  public void readValues(int[] rows, int rowStartPos, int rowSize, int[] values, int valuesStartPos) {
+    System.arraycopy(rows, rowStartPos, values, valuesStartPos, rowSize);
+  }
+
+  @Override
+  public void close() throws IOException {
+  }
+
+  /**
+   * update the internal data to value at a given location
+   * assume the update will be idempotent and we won't check if the existing data have been set
+   *
+   */
+  protected void updateValue(int docId, int value) {
+    if (docId >= _totalDocSize) {
+      throw new RuntimeException(String.format("new record docId %s is larger than capacity %s", docId, _totalDocSize));
+    }
+    if (_values[docId] == 0) {
+      throw new RuntimeException(String.format("failed to update virtual column: with value %s:%s we are trying to " +
+              "update a value that has not been ingested yet, max doc id %s", docId, value, _currentMaxDocId));
+    }
+    _values[docId] = value;
+  }
+
+  // ensure backward compatibility
+  protected void updateValue(int docId, long value) {
+    updateValue(docId, Math.toIntExact(value));
+  }
+
+  public abstract boolean update(int docId, long value, LogEventType eventType);
+
+  public abstract boolean update(int docId, int value, LogEventType eventType);
+}
diff --git a/pinot-core/src/test/java/org/apache/pinot/core/data/manager/realtime/LLRealtimeSegmentDataManagerTest.java b/pinot-core/src/test/java/org/apache/pinot/core/data/manager/realtime/LLRealtimeSegmentDataManagerTest.java
index 715c833..c4b557b 100644
--- a/pinot-core/src/test/java/org/apache/pinot/core/data/manager/realtime/LLRealtimeSegmentDataManagerTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/core/data/manager/realtime/LLRealtimeSegmentDataManagerTest.java
@@ -696,7 +696,7 @@ public class LLRealtimeSegmentDataManagerTest {
     Assert.assertEquals(secondSegmentDataManager.get().getPartitionConsumerSemaphore().availablePermits(), 1);
   }
 
-  public static class FakeLLRealtimeSegmentDataManager extends LLRealtimeSegmentDataManager {
+  public static class FakeLLRealtimeSegmentDataManager extends AppendLLRealtimeSegmentDataManager {
 
     public Field _state;
     public Field _shouldStop;
diff --git a/pinot-core/src/test/java/org/apache/pinot/core/indexsegment/immutable/ImmutableUpsertSegmentImplTest.java b/pinot-core/src/test/java/org/apache/pinot/core/indexsegment/immutable/ImmutableUpsertSegmentImplTest.java
new file mode 100644
index 0000000..817131c
--- /dev/null
+++ b/pinot-core/src/test/java/org/apache/pinot/core/indexsegment/immutable/ImmutableUpsertSegmentImplTest.java
@@ -0,0 +1,127 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.core.indexsegment.immutable;
+
+import org.apache.pinot.core.segment.updater.UpsertWaterMarkManager;
+import org.apache.pinot.core.segment.virtualcolumn.mutable.VirtualColumnLongValueReaderWriter;
+import org.apache.pinot.grigio.common.messages.LogEventType;
+import org.apache.pinot.grigio.common.storageProvider.UpdateLogEntry;
+import org.apache.pinot.grigio.common.storageProvider.UpdateLogEntrySet;
+import org.apache.pinot.grigio.common.storageProvider.UpdateLogStorageProvider;
+import org.testng.Assert;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+
+public class ImmutableUpsertSegmentImplTest {
+
+  UpdateLogStorageProvider _mockProvider;
+  UpsertWaterMarkManager _mockUpsertWaterMarkManager;
+  List<VirtualColumnLongValueReaderWriter> _readerWriters = new ArrayList<>();
+
+  @BeforeMethod
+  public void init() {
+    _mockProvider = mock(UpdateLogStorageProvider.class);
+    _mockUpsertWaterMarkManager = mock(UpsertWaterMarkManager.class);
+  }
+
+  @Test
+  public void testInitVirtualColumn() throws IOException {
+    long start = System.currentTimeMillis();
+    long minOffset = 5000_000l;
+    int totalDocs = 5_000_000;
+    _readerWriters.add(new VirtualColumnLongValueReaderWriter(totalDocs, false) {
+      @Override
+      public boolean update(int docId, long value, LogEventType eventType) {
+        if (eventType == LogEventType.INSERT) {
+          updateValue(docId, value);
+          return true;
+        }
+        return false;
+      }
+
+      @Override
+      public boolean update(int docId, int value, LogEventType eventType) {
+        return update(docId, value, eventType);
+      }
+    });
+    _readerWriters.add(new VirtualColumnLongValueReaderWriter(totalDocs, false) {
+      @Override
+      public boolean update(int docId, long value, LogEventType eventType) {
+        if (eventType == LogEventType.DELETE) {
+          updateValue(docId, value);
+          return true;
+        }
+        return false;
+      }
+
+      @Override
+      public boolean update(int docId, int value, LogEventType eventType) {
+        return update(docId, value, eventType);
+      }
+    });
+    int[] offsetToDocId = new int[totalDocs];
+    for (int i = 0; i < totalDocs; i++) {
+      offsetToDocId[i] = i;
+    }
+    List<UpdateLogEntry> updateLogEntries = new ArrayList<>(totalDocs * 2);
+    for (int i = 0; i < totalDocs; i++) {
+      updateLogEntries.add(new UpdateLogEntry(minOffset + i, 50, LogEventType.INSERT, i%8));
+      updateLogEntries.add(new UpdateLogEntry(minOffset + i, 100, LogEventType.DELETE, i%8));
+    }
+    UpdateLogEntrySet entrySet = new UpdateLogEntrySet(null, 2) {
+      @Override
+      public Iterator<UpdateLogEntry> iterator() {
+        return updateLogEntries.iterator();
+      }
+    };
+    when(_mockProvider.getAllMessages(anyString(), anyString())).thenReturn(entrySet);
+    System.out.println("run time for set up: " + (System.currentTimeMillis() - start));
+
+    start = System.currentTimeMillis();
+
+    ImmutableUpsertSegmentImpl immutableUpsertSegment = new ImmutableUpsertSegmentImpl(_readerWriters, totalDocs,
+        _mockUpsertWaterMarkManager, _mockProvider, minOffset, offsetToDocId);
+
+    immutableUpsertSegment.initVirtualColumn();
+    long runtime = System.currentTimeMillis() - start;
+    System.out.println("run time is " + runtime);
+    // on regular developer laptop this should take less 1 second, but on integration server this might be longer
+    Assert.assertTrue(runtime < 10_000L, "run time should be less than 10 second");
+
+    VirtualColumnLongValueReaderWriter insertReaderWrite = _readerWriters.get(0);
+    VirtualColumnLongValueReaderWriter deleteReaderWrite = _readerWriters.get(1);
+    for (int i = 0; i < totalDocs; i++) {
+      if (insertReaderWrite.getLong(i) != 50 || deleteReaderWrite.getLong(i) != 100) {
+        System.out.println(String.format("position %d has value %d/%d", i, insertReaderWrite.getLong(i),
+            deleteReaderWrite.getLong(i)));
+        Assert.fail("no correct value");
+      }
+    }
+  }
+}
\ No newline at end of file
diff --git a/pinot-core/src/test/java/org/apache/pinot/core/indexsegment/mutable/MutableSegmentImplTestUtils.java b/pinot-core/src/test/java/org/apache/pinot/core/indexsegment/mutable/MutableSegmentImplTestUtils.java
index e4df760..26e7eff 100644
--- a/pinot-core/src/test/java/org/apache/pinot/core/indexsegment/mutable/MutableSegmentImplTestUtils.java
+++ b/pinot-core/src/test/java/org/apache/pinot/core/indexsegment/mutable/MutableSegmentImplTestUtils.java
@@ -59,6 +59,6 @@ public class MutableSegmentImplTestUtils {
             .setRealtimeSegmentZKMetadata(new RealtimeSegmentZKMetadata())
             .setMemoryManager(new DirectMemoryManager(SEGMENT_NAME)).setStatsHistory(statsHistory)
             .setAggregateMetrics(aggregateMetrics).setNullHandlingEnabled(nullHandlingEnabled).build();
-    return new MutableSegmentImpl(realtimeSegmentConfig);
+    return new MutableAppendSegmentImpl(realtimeSegmentConfig);
   }
 }
diff --git a/pinot-core/src/test/java/org/apache/pinot/queries/InterSegmentAggregationSingleValueQueriesTest.java b/pinot-core/src/test/java/org/apache/pinot/queries/InterSegmentAggregationSingleValueQueriesTest.java
index cca0563..f06ad7d 100644
--- a/pinot-core/src/test/java/org/apache/pinot/queries/InterSegmentAggregationSingleValueQueriesTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/queries/InterSegmentAggregationSingleValueQueriesTest.java
@@ -18,8 +18,6 @@
  */
 package org.apache.pinot.queries;
 
-import java.io.Serializable;
-import java.util.function.Function;
 import org.apache.pinot.common.response.broker.BrokerResponseNative;
 import org.apache.pinot.common.response.broker.SelectionResults;
 import org.apache.pinot.spi.utils.BytesUtils;
@@ -29,6 +27,9 @@ import org.apache.pinot.pql.parsers.Pql2Compiler;
 import org.testng.Assert;
 import org.testng.annotations.Test;
 
+import java.io.Serializable;
+import java.util.function.Function;
+
 import static org.testng.Assert.assertFalse;
 import static org.testng.Assert.assertTrue;
 
diff --git a/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/HelixInstanceDataManager.java b/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/HelixInstanceDataManager.java
index 3df1b8f..1af7e29 100644
--- a/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/HelixInstanceDataManager.java
+++ b/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/HelixInstanceDataManager.java
@@ -20,10 +20,7 @@ package org.apache.pinot.server.starter.helix;
 
 import com.google.common.base.Preconditions;
 import java.io.File;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Set;
+import java.util.*;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.locks.Lock;
 import javax.annotation.Nullable;
@@ -48,6 +45,7 @@ import org.apache.pinot.core.indexsegment.immutable.ImmutableSegment;
 import org.apache.pinot.core.indexsegment.immutable.ImmutableSegmentLoader;
 import org.apache.pinot.core.segment.index.loader.IndexLoadingConfig;
 import org.apache.pinot.core.segment.index.loader.LoaderUtils;
+import org.apache.pinot.core.segment.updater.UpsertWaterMarkManager;
 import org.apache.pinot.spi.data.Schema;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -304,6 +302,11 @@ public class HelixInstanceDataManager implements InstanceDataManager {
   }
 
   @Override
+  public Map<String, Map<Integer, Long>> getLowWaterMarks() {
+    return UpsertWaterMarkManager.getInstance().getHighWaterMarkTablePartitionMap();
+  }
+
+  @Override
   public String getSegmentDataDirectory() {
     return _instanceDataManagerConfig.getInstanceDataDir();
   }
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/realtime/provisioning/MemoryEstimator.java b/pinot-tools/src/main/java/org/apache/pinot/tools/realtime/provisioning/MemoryEstimator.java
index 865591b..187b36f 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/realtime/provisioning/MemoryEstimator.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/realtime/provisioning/MemoryEstimator.java
@@ -26,6 +26,7 @@ import java.util.stream.Collectors;
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.io.FileUtils;
 import org.apache.pinot.common.config.TableConfig;
+import org.apache.pinot.core.indexsegment.mutable.MutableAppendSegmentImpl;
 import org.apache.pinot.spi.data.FieldSpec;
 import org.apache.pinot.common.metadata.segment.RealtimeSegmentZKMetadata;
 import org.apache.pinot.spi.utils.DataSize;
@@ -135,7 +136,7 @@ public class MemoryEstimator {
             .setStatsHistory(sampleStatsHistory);
 
     // create mutable segment impl
-    MutableSegmentImpl mutableSegmentImpl = new MutableSegmentImpl(realtimeSegmentConfigBuilder.build());
+    MutableSegmentImpl mutableSegmentImpl = new MutableAppendSegmentImpl(realtimeSegmentConfigBuilder.build());
 
     // read all rows and index them
     try (PinotSegmentRecordReader segmentRecordReader = new PinotSegmentRecordReader(_sampleCompletedSegment);) {
@@ -235,7 +236,7 @@ public class MemoryEstimator {
               .setOffHeap(true).setMemoryManager(memoryManager).setStatsHistory(statsHistory);
 
       // create mutable segment impl
-      MutableSegmentImpl mutableSegmentImpl = new MutableSegmentImpl(realtimeSegmentConfigBuilder.build());
+      MutableSegmentImpl mutableSegmentImpl = new MutableAppendSegmentImpl(realtimeSegmentConfigBuilder.build());
       long memoryForConsumingSegmentPerPartition = memoryManager.getTotalAllocatedBytes();
       mutableSegmentImpl.destroy();
       FileUtils.deleteQuietly(statsFileCopy);


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org