You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ja...@apache.org on 2020/03/18 18:04:17 UTC
[incubator-pinot] 02/05: apply pinot core changes
This is an automated email from the ASF dual-hosted git repository.
jamesshao pushed a commit to branch upsert-refactor
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit 8d2c875e1344f0b61960c0879dd9d1e3c34341c4
Author: james Shao <sj...@uber.com>
AuthorDate: Mon Feb 24 15:47:11 2020 -0800
apply pinot core changes
---
pinot-core/pom.xml | 9 +
.../core/data/manager/BaseTableDataManager.java | 6 +-
.../core/data/manager/InstanceDataManager.java | 6 +
.../manager/UpsertSegmentDataManager.java} | 35 +--
.../manager/config/TableDataManagerConfig.java | 10 +-
.../offline/ImmutableSegmentDataManager.java | 2 +-
.../manager/offline/TableDataManagerProvider.java | 18 +-
.../offline/UpsertImmutableSegmentDataManager.java | 67 ++++
.../AppendLLRealtimeSegmentDataManager.java | 58 ++++
.../realtime/AppendRealtimeTableDataManager.java | 55 ++++
.../realtime/HLRealtimeSegmentDataManager.java | 10 +-
.../realtime/LLRealtimeSegmentDataManager.java | 50 +--
.../manager/realtime/RealtimeTableDataManager.java | 36 ++-
.../UpsertLLRealtimeSegmentDataManager.java | 153 +++++++++
.../realtime/UpsertRealtimeTableDataManager.java | 82 +++++
.../UpsertSegment.java} | 35 +--
.../immutable/ImmutableSegmentImpl.java | 8 +-
.../immutable/ImmutableSegmentLoader.java | 27 +-
.../immutable/ImmutableUpsertSegmentImpl.java | 251 +++++++++++++++
.../mutable/MutableAppendSegmentImpl.java} | 31 +-
.../indexsegment/mutable/MutableSegmentImpl.java | 55 +++-
.../mutable/MutableUpsertSegmentImpl.java | 159 ++++++++++
.../core/realtime/impl/RealtimeSegmentConfig.java | 16 +-
.../core/segment/index/SegmentMetadataImpl.java | 29 +-
.../SegmentDeletionListener.java} | 27 +-
.../pinot/core/segment/updater/SegmentUpdater.java | 349 +++++++++++++++++++++
.../SegmentUpdaterConfig.java} | 29 +-
.../segment/updater/UpsertWaterMarkManager.java | 90 ++++++
.../virtualcolumn/BaseVirtualColumnProvider.java | 18 ++
.../virtualcolumn/VirtualColumnContext.java | 8 +-
.../virtualcolumn/VirtualColumnProvider.java | 4 +
.../mutable/BaseLongVirtualColumnProvider.java | 57 ++++
.../BaseVirtualColumnSingleValueReaderWriter.java} | 27 +-
.../ValidFromInMemoryVirtualColumnProvider.java} | 39 +--
.../ValidUntilInMemoryVirtualColumnProvider.java} | 39 +--
.../VirtualColumnLongValueReaderWriter.java | 108 +++++++
.../realtime/LLRealtimeSegmentDataManagerTest.java | 2 +-
.../immutable/ImmutableUpsertSegmentImplTest.java | 127 ++++++++
.../mutable/MutableSegmentImplTestUtils.java | 2 +-
...erSegmentAggregationSingleValueQueriesTest.java | 5 +-
.../starter/helix/HelixInstanceDataManager.java | 11 +-
.../realtime/provisioning/MemoryEstimator.java | 5 +-
42 files changed, 1909 insertions(+), 246 deletions(-)
diff --git a/pinot-core/pom.xml b/pinot-core/pom.xml
index c7179cf..74528c9 100644
--- a/pinot-core/pom.xml
+++ b/pinot-core/pom.xml
@@ -69,6 +69,15 @@
<artifactId>pinot-common</artifactId>
</dependency>
<dependency>
+ <groupId>org.apache.pinot</groupId>
+ <artifactId>pinot-grigio-common</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.testng</groupId>
+ <artifactId>testng</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
</dependency>
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/BaseTableDataManager.java b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/BaseTableDataManager.java
index e92d85f..a39bdfc 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/BaseTableDataManager.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/BaseTableDataManager.java
@@ -115,7 +115,7 @@ public abstract class BaseTableDataManager implements TableDataManager {
immutableSegment.getSegmentMetadata().getTotalRawDocs());
_serverMetrics.addValueToTableGauge(_tableNameWithType, ServerGauge.SEGMENT_COUNT, 1L);
- ImmutableSegmentDataManager newSegmentManager = new ImmutableSegmentDataManager(immutableSegment);
+ ImmutableSegmentDataManager newSegmentManager = getImmutableSegmentDataManager(immutableSegment);
SegmentDataManager oldSegmentManager = _segmentDataManagerMap.put(segmentName, newSegmentManager);
if (oldSegmentManager == null) {
_logger.info("Added new immutable segment: {} to table: {}", segmentName, _tableNameWithType);
@@ -211,4 +211,8 @@ public abstract class BaseTableDataManager implements TableDataManager {
public String getTableName() {
return _tableNameWithType;
}
+
+ protected ImmutableSegmentDataManager getImmutableSegmentDataManager(ImmutableSegment immutableSegment) {
+ return new ImmutableSegmentDataManager(immutableSegment);
+ }
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/InstanceDataManager.java b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/InstanceDataManager.java
index 4da0c07..175f5a7 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/InstanceDataManager.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/InstanceDataManager.java
@@ -20,6 +20,7 @@ package org.apache.pinot.core.data.manager;
import java.io.File;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import javax.annotation.Nullable;
import javax.annotation.concurrent.ThreadSafe;
@@ -130,4 +131,9 @@ public interface InstanceDataManager {
* Returns the Helix property store.
*/
ZkHelixPropertyStore<ZNRecord> getPropertyStore();
+
+ /**
+ * Return the mappings from partition -> low water marks of all the tables hosted in this server.
+ */
+ Map<String, Map<Integer, Long>> getLowWaterMarks();
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/VirtualColumnContext.java b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/UpsertSegmentDataManager.java
similarity index 53%
copy from pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/VirtualColumnContext.java
copy to pinot-core/src/main/java/org/apache/pinot/core/data/manager/UpsertSegmentDataManager.java
index bb32320..4f589f6 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/VirtualColumnContext.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/UpsertSegmentDataManager.java
@@ -16,29 +16,24 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.pinot.core.segment.virtualcolumn;
+package org.apache.pinot.core.data.manager;
-import org.apache.pinot.spi.data.FieldSpec;
+import org.apache.pinot.grigio.common.storageProvider.UpdateLogEntry;
+import java.util.List;
-/**
- * Miscellaneous context information about the virtual column.
- * It will be used to build various components (dictionary, reader, etc) in the virtual column provider.
- */
-public class VirtualColumnContext {
- private FieldSpec _fieldSpec;
- private int _totalDocCount;
-
- public VirtualColumnContext(FieldSpec fieldSpec, int totalDocCount) {
- _fieldSpec = fieldSpec;
- _totalDocCount = totalDocCount;
- }
+public interface UpsertSegmentDataManager {
- public FieldSpec getFieldSpec() {
- return _fieldSpec;
- }
+ /**
+ * update the upsert-related virtual columns with the new values in this list of update logs
+ * @param messages list of updates logs to update the virtual columns
+ */
+ void updateVirtualColumns(List<UpdateLogEntry> messages);
- public int getTotalDocCount() {
- return _totalDocCount;
- }
+ /**
+ * get the upsert related virtual column debug info given an offset
+ * @param offset the offset we want to look up the virtual column info from
+ * @return debug info describing the upsert-related virtual column info
+ */
+ String getVirtualColumnInfo(long offset);
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/config/TableDataManagerConfig.java b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/config/TableDataManagerConfig.java
index 1c01ef3..3bf51df 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/config/TableDataManagerConfig.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/config/TableDataManagerConfig.java
@@ -19,13 +19,15 @@
package org.apache.pinot.core.data.manager.config;
import com.google.common.base.Preconditions;
-import javax.annotation.Nonnull;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.pinot.common.config.TableConfig;
import org.apache.pinot.common.config.TableNameBuilder;
+import org.apache.pinot.common.utils.CommonConstants;
import org.apache.pinot.common.utils.CommonConstants.Helix.TableType;
+import javax.annotation.Nonnull;
+
/**
* The config used for TableDataManager.
@@ -35,6 +37,7 @@ public class TableDataManagerConfig {
private static final String TABLE_DATA_MANAGER_DATA_DIRECTORY = "directory";
private static final String TABLE_DATA_MANAGER_CONSUMER_DIRECTORY = "consumerDirectory";
private static final String TABLE_DATA_MANAGER_NAME = "name";
+ private static final String TABLE_UPDATE_SEMANTIC = "updateSemantic";
private final Configuration _tableDataManagerConfig;
@@ -62,6 +65,10 @@ public class TableDataManagerConfig {
return _tableDataManagerConfig.getString(TABLE_DATA_MANAGER_NAME);
}
+ public CommonConstants.UpdateSemantic getUpdateSemantic() {
+ return CommonConstants.UpdateSemantic.getUpdateSemantic(_tableDataManagerConfig.getString(TABLE_UPDATE_SEMANTIC));
+ }
+
public static TableDataManagerConfig getDefaultHelixTableDataManagerConfig(
@Nonnull InstanceDataManagerConfig instanceDataManagerConfig, @Nonnull String tableNameWithType) {
Configuration defaultConfig = new PropertiesConfiguration();
@@ -83,5 +90,6 @@ public class TableDataManagerConfig {
// If we wish to override some table level configs using table config, override them here
// Note: the configs in TableDataManagerConfig is immutable once the table is created, which mean it will not pick
// up the latest table config
+ _tableDataManagerConfig.setProperty(TABLE_UPDATE_SEMANTIC, tableConfig.getUpdateSemantic().toString());
}
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/offline/ImmutableSegmentDataManager.java b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/offline/ImmutableSegmentDataManager.java
index 5ffe55f..0401f61 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/offline/ImmutableSegmentDataManager.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/offline/ImmutableSegmentDataManager.java
@@ -27,7 +27,7 @@ import org.apache.pinot.core.indexsegment.immutable.ImmutableSegment;
*/
public class ImmutableSegmentDataManager extends SegmentDataManager {
- private final ImmutableSegment _immutableSegment;
+ protected final ImmutableSegment _immutableSegment;
public ImmutableSegmentDataManager(ImmutableSegment immutableSegment) {
_immutableSegment = immutableSegment;
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/offline/TableDataManagerProvider.java b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/offline/TableDataManagerProvider.java
index aa36055..3ec1bbd 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/offline/TableDataManagerProvider.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/offline/TableDataManagerProvider.java
@@ -18,8 +18,6 @@
*/
package org.apache.pinot.core.data.manager.offline;
-import java.util.concurrent.Semaphore;
-import javax.annotation.Nonnull;
import org.apache.helix.ZNRecord;
import org.apache.helix.store.zk.ZkHelixPropertyStore;
import org.apache.pinot.common.metrics.ServerMetrics;
@@ -27,13 +25,21 @@ import org.apache.pinot.common.utils.CommonConstants;
import org.apache.pinot.core.data.manager.TableDataManager;
import org.apache.pinot.core.data.manager.config.InstanceDataManagerConfig;
import org.apache.pinot.core.data.manager.config.TableDataManagerConfig;
-import org.apache.pinot.core.data.manager.realtime.RealtimeTableDataManager;
+import org.apache.pinot.core.data.manager.realtime.AppendRealtimeTableDataManager;
+import org.apache.pinot.core.data.manager.realtime.UpsertRealtimeTableDataManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.Nonnull;
+import java.util.concurrent.Semaphore;
/**
* Factory for {@link TableDataManager}.
*/
public class TableDataManagerProvider {
+ private static final Logger LOGGER = LoggerFactory.getLogger(TableDataManagerProvider.class);
+
private static Semaphore _segmentBuildSemaphore;
private TableDataManagerProvider() {
@@ -55,7 +61,11 @@ public class TableDataManagerProvider {
tableDataManager = new OfflineTableDataManager();
break;
case REALTIME:
- tableDataManager = new RealtimeTableDataManager(_segmentBuildSemaphore);
+ if (tableDataManagerConfig.getUpdateSemantic() == CommonConstants.UpdateSemantic.UPSERT) {
+ tableDataManager = new UpsertRealtimeTableDataManager(_segmentBuildSemaphore);
+ } else {
+ tableDataManager = new AppendRealtimeTableDataManager(_segmentBuildSemaphore);
+ }
break;
default:
throw new IllegalStateException();
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/offline/UpsertImmutableSegmentDataManager.java b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/offline/UpsertImmutableSegmentDataManager.java
new file mode 100644
index 0000000..790e912
--- /dev/null
+++ b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/offline/UpsertImmutableSegmentDataManager.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.core.data.manager.offline;
+
+import org.apache.pinot.common.config.TableNameBuilder;
+import org.apache.pinot.common.utils.CommonConstants;
+import org.apache.pinot.common.utils.LLCSegmentName;
+import org.apache.pinot.core.data.manager.UpsertSegmentDataManager;
+import org.apache.pinot.core.indexsegment.UpsertSegment;
+import org.apache.pinot.core.indexsegment.immutable.ImmutableSegment;
+import org.apache.pinot.core.segment.updater.SegmentUpdater;
+import org.apache.pinot.grigio.common.storageProvider.UpdateLogEntry;
+
+import java.io.IOException;
+import java.util.List;
+
+public class UpsertImmutableSegmentDataManager extends ImmutableSegmentDataManager implements UpsertSegmentDataManager {
+
+ private String _tableNameWithType;
+
+ public UpsertImmutableSegmentDataManager(ImmutableSegment immutableSegment) throws IOException {
+ super(immutableSegment);
+ _tableNameWithType = TableNameBuilder.ensureTableNameWithType(immutableSegment.getSegmentMetadata().getTableName(),
+ CommonConstants.Helix.TableType.REALTIME);
+ initVirtualColumns();
+ }
+
+ @Override
+ public void updateVirtualColumns(List<UpdateLogEntry> messages) {
+ ((UpsertSegment) _immutableSegment).updateVirtualColumn(messages);
+ }
+
+ @Override
+ public String getVirtualColumnInfo(long offset) {
+ return ((UpsertSegment) _immutableSegment).getVirtualColumnInfo(offset);
+ }
+
+ @Override
+ public void destroy() {
+ SegmentUpdater.getInstance().removeSegmentDataManager(_tableNameWithType, getSegmentName(), this);
+ super.destroy();
+ }
+
+ private void initVirtualColumns() throws IOException {
+ // 1. add listener for update events
+ // 2. load all existing messages
+ // ensure the above orders so we can ensure all events are received by this data manager
+ SegmentUpdater.getInstance().addSegmentDataManager(_tableNameWithType, new LLCSegmentName(getSegmentName()), this);
+ ((UpsertSegment) _immutableSegment).initVirtualColumn();
+ }
+}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/AppendLLRealtimeSegmentDataManager.java b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/AppendLLRealtimeSegmentDataManager.java
new file mode 100644
index 0000000..f725cc8
--- /dev/null
+++ b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/AppendLLRealtimeSegmentDataManager.java
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.core.data.manager.realtime;
+
+import org.apache.pinot.common.config.TableConfig;
+import org.apache.pinot.common.metadata.segment.RealtimeSegmentZKMetadata;
+import org.apache.pinot.common.metrics.ServerMetrics;
+import org.apache.pinot.common.utils.LLCSegmentName;
+import org.apache.pinot.core.indexsegment.mutable.MutableAppendSegmentImpl;
+import org.apache.pinot.core.indexsegment.mutable.MutableSegmentImpl;
+import org.apache.pinot.core.realtime.impl.RealtimeSegmentConfig;
+import org.apache.pinot.core.segment.index.loader.IndexLoadingConfig;
+import org.apache.pinot.spi.data.Schema;
+import org.apache.pinot.spi.data.readers.GenericRow;
+
+import java.util.concurrent.Semaphore;
+
+public class AppendLLRealtimeSegmentDataManager extends LLRealtimeSegmentDataManager {
+
+ public AppendLLRealtimeSegmentDataManager(RealtimeSegmentZKMetadata segmentZKMetadata, TableConfig tableConfig,
+ RealtimeTableDataManager realtimeTableDataManager, String resourceDataDir, IndexLoadingConfig indexLoadingConfig,
+ Schema schema, LLCSegmentName llcSegmentName, Semaphore partitionConsumerSemaphore, ServerMetrics serverMetrics) {
+ super(segmentZKMetadata, tableConfig, realtimeTableDataManager, resourceDataDir, indexLoadingConfig, schema,
+ llcSegmentName, partitionConsumerSemaphore, serverMetrics);
+ }
+
+ @Override
+ protected MutableSegmentImpl createMutableSegment(RealtimeSegmentConfig config) {
+ return new MutableAppendSegmentImpl(config);
+ }
+
+ @Override
+ protected void processTransformedRow(GenericRow row, long offset) {
+ // do nothing
+ }
+
+ @Override
+ protected void postIndexProcessing(GenericRow row, long offset) {
+ // do nothing
+ }
+
+}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/AppendRealtimeTableDataManager.java b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/AppendRealtimeTableDataManager.java
new file mode 100644
index 0000000..741fb08
--- /dev/null
+++ b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/AppendRealtimeTableDataManager.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.core.data.manager.realtime;
+
+import org.apache.pinot.common.config.TableConfig;
+import org.apache.pinot.common.metadata.instance.InstanceZKMetadata;
+import org.apache.pinot.common.metadata.segment.RealtimeSegmentZKMetadata;
+import org.apache.pinot.common.metrics.ServerMetrics;
+import org.apache.pinot.common.utils.LLCSegmentName;
+import org.apache.pinot.core.indexsegment.immutable.ImmutableSegment;
+import org.apache.pinot.core.indexsegment.immutable.ImmutableSegmentLoader;
+import org.apache.pinot.core.segment.index.loader.IndexLoadingConfig;
+import org.apache.pinot.spi.data.Schema;
+
+import java.io.File;
+import java.util.concurrent.Semaphore;
+
+public class AppendRealtimeTableDataManager extends RealtimeTableDataManager {
+
+ public AppendRealtimeTableDataManager(Semaphore segmentBuildSemaphore) {
+ super(segmentBuildSemaphore);
+ }
+
+ @Override
+ protected LLRealtimeSegmentDataManager getLLRealtimeSegmentDataManager(
+ RealtimeSegmentZKMetadata realtimeSegmentZKMetadata, TableConfig tableConfig,
+ RealtimeTableDataManager realtimeTableDataManager, String indexDirPath, IndexLoadingConfig indexLoadingConfig,
+ Schema schema, LLCSegmentName llcSegmentName, Semaphore partitionConsumerSemaphore, ServerMetrics serverMetrics)
+ throws Exception {
+ return new AppendLLRealtimeSegmentDataManager(realtimeSegmentZKMetadata, tableConfig, realtimeTableDataManager,
+ indexDirPath, indexLoadingConfig, schema, llcSegmentName, partitionConsumerSemaphore, serverMetrics);
+ }
+
+ @Override
+ protected ImmutableSegment loadImmutableSegment(File indexDir, IndexLoadingConfig indexLoadingConfig, Schema schema)
+ throws Exception {
+ return ImmutableSegmentLoader.load(indexDir, indexLoadingConfig, schema);
+ }
+}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/HLRealtimeSegmentDataManager.java b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/HLRealtimeSegmentDataManager.java
index 421fa3f..086812e 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/HLRealtimeSegmentDataManager.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/HLRealtimeSegmentDataManager.java
@@ -30,6 +30,7 @@ import java.util.concurrent.atomic.AtomicLong;
import org.apache.commons.io.FileUtils;
import org.apache.pinot.common.config.IndexingConfig;
import org.apache.pinot.common.config.TableConfig;
+import org.apache.pinot.core.indexsegment.mutable.MutableAppendSegmentImpl;
import org.apache.pinot.spi.data.Schema;
import org.apache.pinot.common.metadata.instance.InstanceZKMetadata;
import org.apache.pinot.common.metadata.segment.RealtimeSegmentZKMetadata;
@@ -101,9 +102,9 @@ public class HLRealtimeSegmentDataManager extends RealtimeSegmentDataManager {
// An instance of this class exists only for the duration of the realtime segment that is currently being consumed.
// Once the segment is committed, the segment is handled by OfflineSegmentDataManager
public HLRealtimeSegmentDataManager(final RealtimeSegmentZKMetadata realtimeSegmentZKMetadata,
- final TableConfig tableConfig, InstanceZKMetadata instanceMetadata,
- final RealtimeTableDataManager realtimeTableDataManager, final String resourceDataDir,
- final IndexLoadingConfig indexLoadingConfig, final Schema schema, final ServerMetrics serverMetrics)
+ final TableConfig tableConfig, InstanceZKMetadata instanceMetadata,
+ final RealtimeTableDataManager realtimeTableDataManager, final String resourceDataDir,
+ final IndexLoadingConfig indexLoadingConfig, final Schema schema, final ServerMetrics serverMetrics)
throws Exception {
super();
_segmentVersion = indexLoadingConfig.getSegmentVersion();
@@ -194,7 +195,8 @@ public class HLRealtimeSegmentDataManager extends RealtimeSegmentDataManager {
indexLoadingConfig.isDirectRealtimeOffheapAllocation(), serverMetrics))
.setStatsHistory(realtimeTableDataManager.getStatsHistory())
.setNullHandlingEnabled(indexingConfig.isNullHandlingEnabled()).build();
- realtimeSegment = new MutableSegmentImpl(realtimeSegmentConfig);
+ realtimeSegment = new MutableAppendSegmentImpl(realtimeSegmentConfig);
+
notifier = realtimeTableDataManager;
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/LLRealtimeSegmentDataManager.java b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/LLRealtimeSegmentDataManager.java
index ab04b74..7765e9e 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/LLRealtimeSegmentDataManager.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/LLRealtimeSegmentDataManager.java
@@ -83,7 +83,7 @@ import org.slf4j.LoggerFactory;
/**
* Segment data manager for low level consumer realtime segments, which manages consumption and segment completion.
*/
-public class LLRealtimeSegmentDataManager extends RealtimeSegmentDataManager {
+public abstract class LLRealtimeSegmentDataManager extends RealtimeSegmentDataManager {
protected enum State {
// The state machine starts off with this state. While in this state we consume stream events
// and index them in memory. We continue to be in this state until the end criteria is satisfied
@@ -202,7 +202,7 @@ public class LLRealtimeSegmentDataManager extends RealtimeSegmentDataManager {
private final int _segmentMaxRowCount;
private final String _resourceDataDir;
private final IndexLoadingConfig _indexLoadingConfig;
- private final Schema _schema;
+ protected final Schema _schema;
// Semaphore for each partitionId only, which is to prevent two different Kafka consumers
// from consuming with the same partitionId in parallel in the same host.
// See the comments in {@link RealtimeTableDataManager}.
@@ -212,21 +212,22 @@ public class LLRealtimeSegmentDataManager extends RealtimeSegmentDataManager {
// modify the permit. This boolean make sure the semaphore gets released only once when the partition stops consuming.
private final AtomicBoolean _acquiredConsumerSemaphore;
private final String _metricKeyName;
- private final ServerMetrics _serverMetrics;
- private final MutableSegmentImpl _realtimeSegment;
+ protected final ServerMetrics _serverMetrics;
+ protected final MutableSegmentImpl _realtimeSegment;
private volatile long _currentOffset;
- private volatile State _state;
+ protected volatile State _state;
private volatile int _numRowsConsumed = 0;
private volatile int _numRowsIndexed = 0; // Can be different from _numRowsConsumed when metrics update is enabled.
private volatile int _numRowsErrored = 0;
private volatile int consecutiveErrorCount = 0;
private long _startTimeMs = 0;
- private final String _segmentNameStr;
private final SegmentVersion _segmentVersion;
private final SegmentBuildTimeLeaseExtender _leaseExtender;
private SegmentBuildDescriptor _segmentBuildDescriptor;
private StreamConsumerFactory _streamConsumerFactory;
+ protected final String _segmentNameStr;
+
// Segment end criteria
private volatile long _consumeEndTime = 0;
private volatile long _finalOffset = -1;
@@ -238,14 +239,14 @@ public class LLRealtimeSegmentDataManager extends RealtimeSegmentDataManager {
private Thread _consumerThread;
private final String _streamTopic;
- private final int _streamPartitionId;
+ protected final int _streamPartitionId;
final String _clientId;
- private final LLCSegmentName _llcSegmentName;
+ protected final LLCSegmentName _llcSegmentName;
private final RecordTransformer _recordTransformer;
private PartitionLevelConsumer _partitionLevelConsumer = null;
private StreamMetadataProvider _streamMetadataProvider = null;
private final File _resourceTmpDir;
- private final String _tableNameWithType;
+ protected final String _tableNameWithType;
private final String _timeColumnName;
private final List<String> _invertedIndexColumns;
private final List<String> _textIndexColumns;
@@ -253,7 +254,7 @@ public class LLRealtimeSegmentDataManager extends RealtimeSegmentDataManager {
private final List<String> _varLengthDictionaryColumns;
private final StarTreeIndexSpec _starTreeIndexSpec;
private final String _sortedColumn;
- private Logger segmentLogger;
+ protected Logger segmentLogger;
private final String _tableStreamName;
private final PinotDataBufferMemoryManager _memoryManager;
private AtomicLong _lastUpdatedRowsIndexed = new AtomicLong(0);
@@ -466,16 +467,19 @@ public class LLRealtimeSegmentDataManager extends RealtimeSegmentDataManager {
GenericRow transformedRow = _recordTransformer.transform(decodedRow);
if (transformedRow != null) {
- realtimeRowsConsumedMeter = _serverMetrics
- .addMeteredTableValue(_metricKeyName, ServerMeter.REALTIME_ROWS_CONSUMED, 1, realtimeRowsConsumedMeter);
+ processTransformedRow(transformedRow, _currentOffset);
+ realtimeRowsConsumedMeter =
+ _serverMetrics.addMeteredTableValue(_metricKeyName, ServerMeter.REALTIME_ROWS_CONSUMED, 1,
+ realtimeRowsConsumedMeter);
indexedMessageCount++;
+
+ canTakeMore = _realtimeSegment.index(transformedRow, msgMetadata);
+ postIndexProcessing(transformedRow, _currentOffset);
} else {
realtimeRowsDroppedMeter = _serverMetrics
.addMeteredTableValue(_metricKeyName, ServerMeter.INVALID_REALTIME_ROWS_DROPPED, 1,
realtimeRowsDroppedMeter);
}
-
- canTakeMore = _realtimeSegment.index(transformedRow, msgMetadata);
} catch (Exception e) {
segmentLogger.error("Caught exception while transforming the record: {}", decodedRow, e);
_numRowsErrored++;
@@ -501,6 +505,11 @@ public class LLRealtimeSegmentDataManager extends RealtimeSegmentDataManager {
}
}
+ protected abstract void processTransformedRow(GenericRow row, long offset);
+
+
+ protected abstract void postIndexProcessing(GenericRow row, long offset);
+
public class PartitionConsumer implements Runnable {
public void run() {
long initialConsumptionEnd = 0L;
@@ -795,7 +804,7 @@ public class LLRealtimeSegmentDataManager extends RealtimeSegmentDataManager {
return false;
}
- _realtimeTableDataManager.replaceLLSegment(_segmentNameStr, _indexLoadingConfig);
+ _realtimeTableDataManager.replaceLLSegment(_segmentNameStr, _indexLoadingConfig, _schema);
removeSegmentFile();
return true;
}
@@ -828,7 +837,7 @@ public class LLRealtimeSegmentDataManager extends RealtimeSegmentDataManager {
return false;
}
- _realtimeTableDataManager.replaceLLSegment(_segmentNameStr, _indexLoadingConfig);
+ _realtimeTableDataManager.replaceLLSegment(_segmentNameStr, _indexLoadingConfig, _schema);
return true;
}
@@ -977,7 +986,7 @@ public class LLRealtimeSegmentDataManager extends RealtimeSegmentDataManager {
protected void downloadSegmentAndReplace(LLCRealtimeSegmentZKMetadata metadata) {
closeKafkaConsumers();
- _realtimeTableDataManager.downloadAndReplaceSegment(_segmentNameStr, metadata, _indexLoadingConfig);
+ _realtimeTableDataManager.downloadAndReplaceSegment(_segmentNameStr, metadata, _indexLoadingConfig, _schema);
}
protected long now() {
@@ -1132,7 +1141,8 @@ public class LLRealtimeSegmentDataManager extends RealtimeSegmentDataManager {
// Start new realtime segment
String consumerDir = realtimeTableDataManager.getConsumerDir();
RealtimeSegmentConfig.Builder realtimeSegmentConfigBuilder =
- new RealtimeSegmentConfig.Builder().setSegmentName(_segmentNameStr).setStreamName(_streamTopic)
+ new RealtimeSegmentConfig.Builder().setTableName(_tableNameWithType)
+ .setSegmentName(_segmentNameStr).setStreamName(_streamTopic)
.setSchema(_schema).setCapacity(_segmentMaxRowCount)
.setAvgNumMultiValues(indexLoadingConfig.getRealtimeAvgMultiValueCount())
.setNoDictionaryColumns(indexLoadingConfig.getNoDictionaryColumns())
@@ -1175,7 +1185,7 @@ public class LLRealtimeSegmentDataManager extends RealtimeSegmentDataManager {
}
}
- _realtimeSegment = new MutableSegmentImpl(realtimeSegmentConfigBuilder.build());
+ _realtimeSegment = createMutableSegment(realtimeSegmentConfigBuilder.build());
_startOffset = _segmentZKMetadata.getStartOffset();
_currentOffset = _startOffset;
_resourceTmpDir = new File(resourceDataDir, "_tmp");
@@ -1211,6 +1221,8 @@ public class LLRealtimeSegmentDataManager extends RealtimeSegmentDataManager {
start();
}
+ protected abstract MutableSegmentImpl createMutableSegment(RealtimeSegmentConfig config);
+
/**
* Creates a new stream consumer
* @param reason
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/RealtimeTableDataManager.java b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/RealtimeTableDataManager.java
index 94c9614..2be3ca1 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/RealtimeTableDataManager.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/RealtimeTableDataManager.java
@@ -39,6 +39,7 @@ import org.apache.pinot.common.metadata.ZKMetadataProvider;
import org.apache.pinot.common.metadata.instance.InstanceZKMetadata;
import org.apache.pinot.common.metadata.segment.LLCRealtimeSegmentZKMetadata;
import org.apache.pinot.common.metadata.segment.RealtimeSegmentZKMetadata;
+import org.apache.pinot.common.metrics.ServerMetrics;
import org.apache.pinot.common.utils.CommonConstants.Segment.Realtime.Status;
import org.apache.pinot.common.utils.LLCSegmentName;
import org.apache.pinot.common.utils.NamedThreadFactory;
@@ -54,10 +55,11 @@ import org.apache.pinot.core.segment.index.loader.LoaderUtils;
import org.apache.pinot.core.segment.virtualcolumn.VirtualColumnProviderFactory;
import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.spi.data.Schema;
+import org.apache.pinot.core.indexsegment.immutable.ImmutableSegment;
@ThreadSafe
-public class RealtimeTableDataManager extends BaseTableDataManager {
+public abstract class RealtimeTableDataManager extends BaseTableDataManager {
private final ExecutorService _segmentAsyncExecutorService =
Executors.newSingleThreadExecutor(new NamedThreadFactory("SegmentAsyncExecutorService"));
private SegmentBuildTimeLeaseExtender _leaseExtender;
@@ -225,7 +227,7 @@ public class RealtimeTableDataManager extends BaseTableDataManager {
if (indexDir.exists() && (realtimeSegmentZKMetadata.getStatus() == Status.DONE)) {
// Segment already exists on disk, and metadata has been committed. Treat it like an offline segment
- addSegment(ImmutableSegmentLoader.load(indexDir, indexLoadingConfig, schema));
+ addSegment(loadImmutableSegment(indexDir, indexLoadingConfig, schema));
} else {
// Either we don't have the segment on disk or we have not committed in ZK. We should be starting the consumer
// for realtime segment here. If we wrote it on disk but could not get to commit to zk yet, we should replace the
@@ -247,7 +249,7 @@ public class RealtimeTableDataManager extends BaseTableDataManager {
LLCRealtimeSegmentZKMetadata llcSegmentMetadata = (LLCRealtimeSegmentZKMetadata) realtimeSegmentZKMetadata;
if (realtimeSegmentZKMetadata.getStatus().equals(Status.DONE)) {
// TODO Remove code duplication here and in LLRealtimeSegmentDataManager
- downloadAndReplaceSegment(segmentName, llcSegmentMetadata, indexLoadingConfig);
+ downloadAndReplaceSegment(segmentName, llcSegmentMetadata, indexLoadingConfig, schema);
return;
}
@@ -255,10 +257,9 @@ public class RealtimeTableDataManager extends BaseTableDataManager {
LLCSegmentName llcSegmentName = new LLCSegmentName(segmentName);
int streamPartitionId = llcSegmentName.getPartitionId();
_partitionIdToSemaphoreMap.putIfAbsent(streamPartitionId, new Semaphore(1));
- manager =
- new LLRealtimeSegmentDataManager(realtimeSegmentZKMetadata, tableConfig, this, _indexDir.getAbsolutePath(),
- indexLoadingConfig, schema, llcSegmentName, _partitionIdToSemaphoreMap.get(streamPartitionId),
- _serverMetrics);
+ manager = getLLRealtimeSegmentDataManager(realtimeSegmentZKMetadata, tableConfig, this,
+ _indexDir.getAbsolutePath(), indexLoadingConfig, schema, llcSegmentName,
+ _partitionIdToSemaphoreMap.get(streamPartitionId), _serverMetrics);
}
_logger.info("Initialize RealtimeSegmentDataManager - " + segmentName);
_segmentDataManagerMap.put(segmentName, manager);
@@ -266,7 +267,7 @@ public class RealtimeTableDataManager extends BaseTableDataManager {
}
public void downloadAndReplaceSegment(String segmentName, LLCRealtimeSegmentZKMetadata llcSegmentMetadata,
- IndexLoadingConfig indexLoadingConfig) {
+ IndexLoadingConfig indexLoadingConfig, Schema schema) {
final String uri = llcSegmentMetadata.getDownloadUrl();
File tempSegmentFolder = new File(_indexDir, "tmp-" + segmentName + "." + System.currentTimeMillis());
File tempFile = new File(_indexDir, segmentName + ".tar.gz");
@@ -279,7 +280,7 @@ public class RealtimeTableDataManager extends BaseTableDataManager {
_logger.info("Uncompressed file {} into tmp dir {}", tempFile, tempSegmentFolder);
FileUtils.moveDirectory(tempSegmentFolder.listFiles()[0], segmentFolder);
_logger.info("Replacing LLC Segment {}", segmentName);
- replaceLLSegment(segmentName, indexLoadingConfig);
+ replaceLLSegment(segmentName, indexLoadingConfig, schema);
} catch (Exception e) {
throw new RuntimeException(e);
} finally {
@@ -302,11 +303,10 @@ public class RealtimeTableDataManager extends BaseTableDataManager {
/**
* Replaces a committed LLC REALTIME segment.
*/
- public void replaceLLSegment(String segmentName, IndexLoadingConfig indexLoadingConfig) {
+ public void replaceLLSegment(String segmentName, IndexLoadingConfig indexLoadingConfig, Schema schema) {
try {
File indexDir = new File(_indexDir, segmentName);
- Schema schema = ZKMetadataProvider.getTableSchema(_propertyStore, _tableNameWithType);
- addSegment(ImmutableSegmentLoader.load(indexDir, indexLoadingConfig, schema));
+ addSegment(loadImmutableSegment(indexDir, indexLoadingConfig, schema));
} catch (Exception e) {
throw new RuntimeException(e);
}
@@ -317,6 +317,18 @@ public class RealtimeTableDataManager extends BaseTableDataManager {
}
/**
+ * allow {@link UpsertRealtimeTableDataManager} to override this method
+ */
+ protected abstract LLRealtimeSegmentDataManager getLLRealtimeSegmentDataManager(
+ RealtimeSegmentZKMetadata realtimeSegmentZKMetadata, TableConfig tableConfig,
+ RealtimeTableDataManager realtimeTableDataManager, String indexDirPath, IndexLoadingConfig indexLoadingConfig,
+ Schema schema, LLCSegmentName llcSegmentName, Semaphore partitionConsumerSemaphore, ServerMetrics serverMetrics)
+ throws Exception;
+
+ protected abstract ImmutableSegment loadImmutableSegment(File indexDir, IndexLoadingConfig indexLoadingConfig, Schema schema)
+ throws Exception;
+
+ /**
* Validate a schema against the table config for real-time record consumption.
* Ideally, we should validate these things when schema is added or table is created, but either of these
* may be changed while the table is already provisioned. For the change to take effect, we need to restart the
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/UpsertLLRealtimeSegmentDataManager.java b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/UpsertLLRealtimeSegmentDataManager.java
new file mode 100644
index 0000000..f08037a
--- /dev/null
+++ b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/UpsertLLRealtimeSegmentDataManager.java
@@ -0,0 +1,153 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.core.data.manager.realtime;
+
+import com.google.common.base.Preconditions;
+import org.apache.pinot.common.config.TableConfig;
+import org.apache.pinot.common.metadata.segment.RealtimeSegmentZKMetadata;
+import org.apache.pinot.common.metrics.ServerMeter;
+import org.apache.pinot.common.metrics.ServerMetrics;
+import org.apache.pinot.common.utils.LLCSegmentName;
+import org.apache.pinot.core.data.manager.UpsertSegmentDataManager;
+import org.apache.pinot.core.indexsegment.UpsertSegment;
+import org.apache.pinot.core.indexsegment.mutable.MutableSegmentImpl;
+import org.apache.pinot.core.indexsegment.mutable.MutableUpsertSegmentImpl;
+import org.apache.pinot.core.realtime.impl.RealtimeSegmentConfig;
+import org.apache.pinot.core.segment.index.loader.IndexLoadingConfig;
+import org.apache.pinot.core.segment.updater.SegmentUpdater;
+import org.apache.pinot.grigio.common.messages.KeyCoordinatorQueueMsg;
+import org.apache.pinot.grigio.common.rpcQueue.ProduceTask;
+import org.apache.pinot.grigio.common.rpcQueue.QueueProducer;
+import org.apache.pinot.grigio.common.storageProvider.UpdateLogEntry;
+import org.apache.pinot.grigio.servers.KeyCoordinatorProvider;
+import org.apache.pinot.spi.data.DimensionFieldSpec;
+import org.apache.pinot.spi.data.Schema;
+import org.apache.pinot.spi.data.TimeFieldSpec;
+import org.apache.pinot.spi.data.readers.GenericRow;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.concurrent.Semaphore;
+
+/**
+ * class design is pretty bad right now, need to rework inheritance to abstract the base class or use composition instead
+ */
+public class UpsertLLRealtimeSegmentDataManager extends LLRealtimeSegmentDataManager implements UpsertSegmentDataManager {
+
+ private final QueueProducer _keyCoordinatorQueueProducer;
+
+ public UpsertLLRealtimeSegmentDataManager(RealtimeSegmentZKMetadata segmentZKMetadata, TableConfig tableConfig,
+ RealtimeTableDataManager realtimeTableDataManager, String resourceDataDir, IndexLoadingConfig indexLoadingConfig,
+ Schema schema, LLCSegmentName llcSegmentName, Semaphore partitionConsumerSemaphore, ServerMetrics serverMetrics) throws Exception {
+ super(segmentZKMetadata, tableConfig, realtimeTableDataManager, resourceDataDir, indexLoadingConfig, schema,
+ llcSegmentName, partitionConsumerSemaphore, serverMetrics);
+ Preconditions.checkState(_schema.getPrimaryKeyFieldSpec() != null, "primary key not found");
+ Preconditions.checkState(_schema.getOffsetKeyFieldSpec() != null, "offset key not found");
+ _keyCoordinatorQueueProducer = KeyCoordinatorProvider.getInstance().getCachedProducer(_tableNameWithType);
+ initVirtualColumns();
+ }
+
+ public String getSegmentName() {
+ return _segmentNameStr;
+ }
+
+ @Override
+ public void updateVirtualColumns(List<UpdateLogEntry> messages) {
+ ((UpsertSegment) _realtimeSegment).updateVirtualColumn(messages);
+ }
+
+ @Override
+ public String getVirtualColumnInfo(long offset) {
+ return ((UpsertSegment) _realtimeSegment).getVirtualColumnInfo(offset);
+ }
+
+ @Override
+ public void destroy() {
+ SegmentUpdater.getInstance().removeSegmentDataManager(_tableNameWithType, _llcSegmentName.getSegmentName(), this);
+ super.destroy();
+ }
+
+ @Override
+ protected MutableSegmentImpl createMutableSegment(RealtimeSegmentConfig config) {
+ return new MutableUpsertSegmentImpl(config);
+ }
+
+ @Override
+ protected void processTransformedRow(GenericRow row, long offset) {
+ row.putField(_schema.getOffsetKey(), offset);
+ }
+
+ @Override
+ protected void postIndexProcessing(GenericRow row, long offset) {
+ emitEventToKeyCoordinator(row, offset);
+ }
+
+ @Override
+ protected boolean consumeLoop() throws Exception {
+ boolean result = super.consumeLoop();
+ segmentLogger.info("flushing kafka producer");
+ _keyCoordinatorQueueProducer.flush();
+ segmentLogger.info("done flushing kafka producer");
+ return result;
+ }
+
+ /**
+ * not handling error right now
+ * @param row
+ * @param offset
+ */
+ private void emitEventToKeyCoordinator(GenericRow row, long offset) {
+ final byte[] primaryKeyBytes = getPrimaryKeyBytesFromRow(row);
+ final long timestampMillis = getTimestampFromRow(row);
+ ProduceTask<byte[], KeyCoordinatorQueueMsg> task = new ProduceTask<>(primaryKeyBytes,
+ new KeyCoordinatorQueueMsg(primaryKeyBytes, _segmentNameStr, timestampMillis, offset));
+ task.setCallback(new ProduceTask.Callback() {
+ @Override
+ public void onSuccess() {
+ // do nothing on success
+ }
+
+ @Override
+ public void onFailure(Exception ex) {
+ // right now we just log the error and not really doing anything
+ // TODO: retries/record logics
+ _serverMetrics.addMeteredGlobalValue(ServerMeter.MESSAGE_PRODUCE_FAILED_COUNT, 1);
+ }
+ });
+ _keyCoordinatorQueueProducer.produce(task);
+ }
+
+ private byte[] getPrimaryKeyBytesFromRow(GenericRow row) {
+ DimensionFieldSpec primaryKeyDimension = _schema.getPrimaryKeyFieldSpec();
+ return _schema.getByteArrayFromField(row.getValue(primaryKeyDimension.getName()), primaryKeyDimension);
+ }
+
+ private long getTimestampFromRow(GenericRow row) {
+ TimeFieldSpec spec = _schema.getTimeFieldSpec();
+ return spec.getIncomingGranularitySpec().toMillis(row.getValue(spec.getIncomingTimeColumnName()));
+ }
+
+
+ private void initVirtualColumns() throws IOException {
+ // 1. ensure the data manager can capture all update events
+ // 2. load all existing messages
+ SegmentUpdater.getInstance().addSegmentDataManager(_tableNameWithType, _llcSegmentName, this);
+ ((UpsertSegment) _realtimeSegment).initVirtualColumn();
+ }
+}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/UpsertRealtimeTableDataManager.java b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/UpsertRealtimeTableDataManager.java
new file mode 100644
index 0000000..12569db
--- /dev/null
+++ b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/UpsertRealtimeTableDataManager.java
@@ -0,0 +1,82 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.core.data.manager.realtime;
+
+import org.apache.pinot.common.utils.LLCSegmentName;
+import org.apache.pinot.core.data.manager.offline.UpsertImmutableSegmentDataManager;
+import org.apache.pinot.common.config.TableConfig;
+import org.apache.pinot.common.metadata.instance.InstanceZKMetadata;
+import org.apache.pinot.common.metadata.segment.RealtimeSegmentZKMetadata;
+import org.apache.pinot.common.metrics.ServerMetrics;
+import org.apache.pinot.core.data.manager.offline.ImmutableSegmentDataManager;
+import org.apache.pinot.core.indexsegment.immutable.ImmutableSegment;
+import org.apache.pinot.core.indexsegment.immutable.ImmutableSegmentLoader;
+import org.apache.pinot.core.segment.index.loader.IndexLoadingConfig;
+import org.apache.pinot.grigio.common.storageProvider.UpdateLogStorageProvider;
+import org.apache.pinot.spi.data.Schema;
+
+import javax.annotation.Nonnull;
+import java.io.File;
+import java.io.IOException;
+import java.util.concurrent.Semaphore;
+
+public class UpsertRealtimeTableDataManager extends RealtimeTableDataManager {
+ private UpdateLogStorageProvider _updateLogStorageProvider;
+
+ public UpsertRealtimeTableDataManager(Semaphore segmentBuildSemaphore) {
+ super(segmentBuildSemaphore);
+ _updateLogStorageProvider = UpdateLogStorageProvider.getInstance();
+ }
+
+ @Override
+ public void addSegment(@Nonnull String segmentName, @Nonnull TableConfig tableConfig,
+ @Nonnull IndexLoadingConfig indexLoadingConfig) throws Exception {
+ _updateLogStorageProvider.addSegment(_tableNameWithType, segmentName);
+ super.addSegment(segmentName, tableConfig, indexLoadingConfig);
+ }
+
+ @Override
+ protected LLRealtimeSegmentDataManager getLLRealtimeSegmentDataManager(
+ RealtimeSegmentZKMetadata realtimeSegmentZKMetadata, TableConfig tableConfig,
+ RealtimeTableDataManager realtimeTableDataManager, String indexDirPath, IndexLoadingConfig indexLoadingConfig,
+ Schema schema, LLCSegmentName llcSegmentName, Semaphore partitionConsumerSemaphore, ServerMetrics serverMetrics)
+ throws Exception {
+ return new UpsertLLRealtimeSegmentDataManager(realtimeSegmentZKMetadata, tableConfig, realtimeTableDataManager,
+ indexDirPath, indexLoadingConfig, schema, llcSegmentName, partitionConsumerSemaphore, serverMetrics);
+ }
+
+ @Override
+ protected ImmutableSegmentDataManager getImmutableSegmentDataManager(ImmutableSegment immutableSegment) {
+ try {
+ return new UpsertImmutableSegmentDataManager(immutableSegment);
+ } catch (IOException e) {
+ throw new RuntimeException("failed to init the upsert immutable segment", e);
+ }
+ }
+
+ @Override
+ protected ImmutableSegment loadImmutableSegment(File indexDir, IndexLoadingConfig indexLoadingConfig, Schema schema) {
+ try {
+ return ImmutableSegmentLoader.loadUpsertSegment(indexDir, indexLoadingConfig, schema);
+ } catch (Exception e) {
+ throw new RuntimeException("failed to load immutable segment", e);
+ }
+ }
+
+}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/VirtualColumnContext.java b/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/UpsertSegment.java
similarity index 52%
copy from pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/VirtualColumnContext.java
copy to pinot-core/src/main/java/org/apache/pinot/core/indexsegment/UpsertSegment.java
index bb32320..125cf06 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/VirtualColumnContext.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/UpsertSegment.java
@@ -16,29 +16,26 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.pinot.core.segment.virtualcolumn;
+package org.apache.pinot.core.indexsegment;
-import org.apache.pinot.spi.data.FieldSpec;
+import org.apache.pinot.grigio.common.storageProvider.UpdateLogEntry;
+import java.io.IOException;
-/**
- * Miscellaneous context information about the virtual column.
- * It will be used to build various components (dictionary, reader, etc) in the virtual column provider.
- */
-public class VirtualColumnContext {
- private FieldSpec _fieldSpec;
- private int _totalDocCount;
+public interface UpsertSegment {
- public VirtualColumnContext(FieldSpec fieldSpec, int totalDocCount) {
- _fieldSpec = fieldSpec;
- _totalDocCount = totalDocCount;
- }
+ /**
+ * update the upsert-related virtual columns with the new values in this list of update logs
+ * @param messages list of updates logs to update the virtual columns
+ */
+ void updateVirtualColumn(Iterable<UpdateLogEntry> messages);
- public FieldSpec getFieldSpec() {
- return _fieldSpec;
- }
+ /**
+ * get the upsert related virtual column debug info given an offset
+ * @param offset the offset we want to look up the virtual column info from
+ * @return debug info describing the upsert-related virtual column info
+ */
+ String getVirtualColumnInfo(long offset);
- public int getTotalDocCount() {
- return _totalDocCount;
- }
+ void initVirtualColumn() throws IOException;
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/immutable/ImmutableSegmentImpl.java b/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/immutable/ImmutableSegmentImpl.java
index 600660a..21f056c 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/immutable/ImmutableSegmentImpl.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/immutable/ImmutableSegmentImpl.java
@@ -43,10 +43,10 @@ import org.slf4j.LoggerFactory;
public class ImmutableSegmentImpl implements ImmutableSegment {
private static final Logger LOGGER = LoggerFactory.getLogger(ImmutableSegmentImpl.class);
- private final SegmentDirectory _segmentDirectory;
- private final SegmentMetadataImpl _segmentMetadata;
- private final Map<String, ColumnIndexContainer> _indexContainerMap;
- private final StarTreeIndexContainer _starTreeIndexContainer;
+ protected final SegmentDirectory _segmentDirectory;
+ protected final SegmentMetadataImpl _segmentMetadata;
+ protected final Map<String, ColumnIndexContainer> _indexContainerMap;
+ protected final StarTreeIndexContainer _starTreeIndexContainer;
public ImmutableSegmentImpl(SegmentDirectory segmentDirectory, SegmentMetadataImpl segmentMetadata,
Map<String, ColumnIndexContainer> columnIndexContainerMap,
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/immutable/ImmutableSegmentLoader.java b/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/immutable/ImmutableSegmentLoader.java
index 90eb839..49d7729 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/immutable/ImmutableSegmentLoader.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/immutable/ImmutableSegmentLoader.java
@@ -44,6 +44,8 @@ import org.apache.pinot.spi.data.Schema;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import javax.annotation.Nonnull;
+
public class ImmutableSegmentLoader {
private ImmutableSegmentLoader() {
@@ -70,9 +72,23 @@ public class ImmutableSegmentLoader {
}
public static ImmutableSegment load(File indexDir, IndexLoadingConfig indexLoadingConfig, @Nullable Schema schema)
- throws Exception {
- Preconditions
- .checkArgument(indexDir.isDirectory(), "Index directory: {} does not exist or is not a directory", indexDir);
+ throws Exception {
+ return loadHelper(indexDir, indexLoadingConfig, schema);
+ }
+
+ /**
+ * to load upsert related segment
+ */
+ public static ImmutableUpsertSegmentImpl loadUpsertSegment(File indexDir, IndexLoadingConfig indexLoadingConfig,
+ Schema schema) throws Exception {
+ ImmutableSegmentImpl segment = loadHelper(indexDir, indexLoadingConfig, schema);
+ return ImmutableUpsertSegmentImpl.copyOf(segment);
+ }
+
+ private static ImmutableSegmentImpl loadHelper(File indexDir, IndexLoadingConfig indexLoadingConfig,
+ @Nullable Schema schema) throws Exception {
+ Preconditions.checkArgument(indexDir.isDirectory(), "Index directory: {} does not exist or is not a directory",
+ indexDir);
// Convert segment version if necessary
// NOTE: this step may modify the segment metadata
@@ -100,7 +116,7 @@ public class ImmutableSegmentLoader {
}
// Load the metadata again since converter and pre-processor may have changed it
- SegmentMetadataImpl segmentMetadata = new SegmentMetadataImpl(indexDir);
+ SegmentMetadataImpl segmentMetadata = new SegmentMetadataImpl(indexDir, schema);
// Load the segment
ReadMode readMode = indexLoadingConfig.getReadMode();
@@ -125,7 +141,7 @@ public class ImmutableSegmentLoader {
String columnName = fieldSpec.getName();
VirtualColumnProvider provider =
VirtualColumnProviderFactory.buildProvider(fieldSpec.getVirtualColumnProvider());
- VirtualColumnContext context = new VirtualColumnContext(fieldSpec, segmentMetadata.getTotalDocs());
+ VirtualColumnContext context = new VirtualColumnContext(fieldSpec, segmentMetadata.getTotalDocs(), false);
indexContainerMap.put(columnName, provider.buildColumnIndexContainer(context));
segmentMetadata.getColumnMetadataMap().put(columnName, provider.buildMetadata(context));
}
@@ -141,4 +157,5 @@ public class ImmutableSegmentLoader {
return new ImmutableSegmentImpl(segmentDirectory, segmentMetadata, indexContainerMap, starTreeIndexContainer);
}
+
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/immutable/ImmutableUpsertSegmentImpl.java b/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/immutable/ImmutableUpsertSegmentImpl.java
new file mode 100644
index 0000000..5ab42f5
--- /dev/null
+++ b/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/immutable/ImmutableUpsertSegmentImpl.java
@@ -0,0 +1,251 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.core.indexsegment.immutable;
+
+import com.clearspring.analytics.util.Preconditions;
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.pinot.common.Utils;
+import org.apache.pinot.common.config.TableNameBuilder;
+import org.apache.pinot.common.utils.CommonConstants;
+import org.apache.pinot.core.indexsegment.UpsertSegment;
+import org.apache.pinot.core.io.reader.BaseSingleColumnSingleValueReader;
+import org.apache.pinot.core.io.reader.DataFileReader;
+import org.apache.pinot.core.segment.index.SegmentMetadataImpl;
+import org.apache.pinot.core.segment.index.column.ColumnIndexContainer;
+import org.apache.pinot.core.segment.index.readers.Dictionary;
+import org.apache.pinot.core.segment.store.SegmentDirectory;
+import org.apache.pinot.core.segment.updater.UpsertWaterMarkManager;
+import org.apache.pinot.core.segment.virtualcolumn.mutable.VirtualColumnLongValueReaderWriter;
+import org.apache.pinot.core.startree.v2.store.StarTreeIndexContainer;
+import org.apache.pinot.grigio.common.storageProvider.UpdateLogEntry;
+import org.apache.pinot.grigio.common.storageProvider.UpdateLogEntrySet;
+import org.apache.pinot.grigio.common.storageProvider.UpdateLogStorageProvider;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.Nullable;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class ImmutableUpsertSegmentImpl extends ImmutableSegmentImpl implements UpsertSegment {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(ImmutableUpsertSegmentImpl.class);
+
+ private final List<VirtualColumnLongValueReaderWriter> _virtualColumnsReaderWriter;
+ private final String _tableNameWithType;
+ private final String _segmentName;
+ private final int _totalDoc;
+ private long _minSourceOffset;
+ private final UpsertWaterMarkManager _upsertWaterMarkManager;
+ private final UpdateLogStorageProvider _updateLogStorageProvider;
+ // use array for mapping bewteen offset to docId, where actual offset = min_offset + array_index
+ // use 4 bytes per record
+ private int[] _sourceOffsetToDocIdArray;
+
+ private static final int DEFAULT_DOC_ID_FOR_MISSING_ENTRY = -1;
+
+ public ImmutableUpsertSegmentImpl(SegmentDirectory segmentDirectory,
+ SegmentMetadataImpl segmentMetadata,
+ Map<String, ColumnIndexContainer> columnIndexContainerMap,
+ @Nullable StarTreeIndexContainer starTreeIndexContainer) {
+ super(segmentDirectory, segmentMetadata, columnIndexContainerMap, starTreeIndexContainer);
+ Preconditions.checkState(segmentMetadata.getSchema().isTableForUpsert(), "table should be upsert but it is not");
+ _tableNameWithType = TableNameBuilder.ensureTableNameWithType(segmentMetadata.getTableName(),
+ CommonConstants.Helix.TableType.REALTIME);
+ _segmentName = segmentMetadata.getName();
+ _totalDoc = segmentMetadata.getTotalDocs();
+ _upsertWaterMarkManager = UpsertWaterMarkManager.getInstance();
+ _updateLogStorageProvider = UpdateLogStorageProvider.getInstance();
+ _virtualColumnsReaderWriter = new ArrayList<>();
+ for (Map.Entry<String, ColumnIndexContainer> entry: columnIndexContainerMap.entrySet()) {
+ String columnName = entry.getKey();
+ ColumnIndexContainer container = entry.getValue();
+ if (segmentMetadata.getSchema().isVirtualColumn(columnName) && (container.getForwardIndex() instanceof VirtualColumnLongValueReaderWriter)) {
+ _virtualColumnsReaderWriter.add((VirtualColumnLongValueReaderWriter) container.getForwardIndex());
+ }
+ }
+ buildOffsetToDocIdMap(columnIndexContainerMap.get(segmentMetadata.getSchema().getOffsetKey()));
+ }
+
+ /** constructor used for creating instance in test cases
+ * should not be used for creating regular segment
+ */
+ @VisibleForTesting
+ protected ImmutableUpsertSegmentImpl(List<VirtualColumnLongValueReaderWriter> readerWriters,
+ int totalDoc, UpsertWaterMarkManager manager,
+ UpdateLogStorageProvider updateLogStorageProvider,
+ long minSourceOffset, int[] offsetToDocId) {
+ super(null, null, null, null);
+ _tableNameWithType = "testTable";
+ _segmentName = "testSegment";
+ _virtualColumnsReaderWriter = readerWriters;
+ _totalDoc = totalDoc;
+ _upsertWaterMarkManager = manager;
+ _updateLogStorageProvider = updateLogStorageProvider;
+ _minSourceOffset = minSourceOffset;
+ _sourceOffsetToDocIdArray = offsetToDocId;
+
+ }
+
+ private void buildOffsetToDocIdMap(ColumnIndexContainer offsetColumnIndexContainer) {
+ long start = System.currentTimeMillis();
+ final DataFileReader reader = offsetColumnIndexContainer.getForwardIndex();
+ final Dictionary dictionary = offsetColumnIndexContainer.getDictionary();
+ Map<Long, Integer> kafkaOffsetToDocIdMap = new HashMap<>();
+ long minOffset = Long.MAX_VALUE;
+ long maxOffset = 0;
+ if (reader instanceof BaseSingleColumnSingleValueReader) {
+ BaseSingleColumnSingleValueReader scsvReader = (BaseSingleColumnSingleValueReader) reader;
+ for (int docId = 0; docId < _totalDoc; docId++) {
+ final Long offset;
+ if (dictionary == null) {
+ offset = scsvReader.getLong(docId);
+ } else {
+ offset = (Long) dictionary.get(scsvReader.getInt(docId));
+ }
+ if (offset == null) {
+ LOGGER.error("kafka offset is null at docID {}", docId);
+ } else {
+ minOffset = Math.min(offset, minOffset);
+ maxOffset = Math.max(offset, maxOffset);
+ kafkaOffsetToDocIdMap.put(offset, docId);
+ }
+ }
+ _minSourceOffset = minOffset;
+ int size = Math.toIntExact(maxOffset - minOffset + 1);
+ _sourceOffsetToDocIdArray = new int[size];
+ for (int i = 0; i < size; i++) {
+ _sourceOffsetToDocIdArray[i] = kafkaOffsetToDocIdMap.getOrDefault(i + minOffset,
+ DEFAULT_DOC_ID_FOR_MISSING_ENTRY);
+ }
+ } else {
+ throw new RuntimeException("unexpected forward reader type for kafka offset column " + reader.getClass());
+ }
+ LOGGER.info("built offset to DocId map for segment {} with {} documents in {} ms", _segmentName, _totalDoc, System.currentTimeMillis() - start);
+ }
+
+ public static ImmutableUpsertSegmentImpl copyOf(ImmutableSegmentImpl immutableSegment) {
+
+ return new ImmutableUpsertSegmentImpl(immutableSegment._segmentDirectory, immutableSegment._segmentMetadata,
+ immutableSegment._indexContainerMap, immutableSegment._starTreeIndexContainer);
+ }
+
+ @Override
+ public void updateVirtualColumn(Iterable<UpdateLogEntry> logEntries) {
+ for (UpdateLogEntry logEntry: logEntries) {
+ boolean updated = false;
+ int docId = getDocIdFromSourceOffset(logEntry.getOffset());
+ for (VirtualColumnLongValueReaderWriter readerWriter : _virtualColumnsReaderWriter) {
+ updated = readerWriter.update(docId, logEntry.getValue(), logEntry.getType()) || updated;
+ }
+ if (updated) {
+ _upsertWaterMarkManager.processMessage(_tableNameWithType, _segmentName, logEntry);
+ }
+ }
+ }
+
+ @Override
+ public String getVirtualColumnInfo(long offset) {
+ int docId = getDocIdFromSourceOffset(offset);
+ StringBuilder result = new StringBuilder("matched: ");
+ for (VirtualColumnLongValueReaderWriter readerWriter : _virtualColumnsReaderWriter) {
+ result.append(readerWriter.getInt(docId)).append("; ");
+ }
+ return result.toString();
+ }
+
+ /**
+ * this method will fetch all updates from update logs in local disk and apply those updates to the current virtual columns
+ * it traverses through all records in the current segment and match existing updates log to its kafka offsets
+ * @throws IOException
+ */
+ @Override
+ public void initVirtualColumn() throws IOException {
+ long start = System.currentTimeMillis();
+ final UpdateLogEntrySet updateLogEntries = _updateLogStorageProvider.getAllMessages(_tableNameWithType, _segmentName);
+ LOGGER.info("load {} update log entry from update log storage provider for segment {} in {} ms",
+ updateLogEntries.size(), _segmentName, System.currentTimeMillis() - start);
+
+ start = System.currentTimeMillis();
+ final long maxOffset = _totalDoc + _minSourceOffset;
+ int unmatchedLogEntryCount = 0;
+ try {
+ Map<Integer, Long> partitionToHighestWatermark = new HashMap<>();
+ int readerWriteCount = _virtualColumnsReaderWriter.size();
+ for (UpdateLogEntry logEntry: updateLogEntries) {
+ final int partition = logEntry.getPartition();
+ final long offset = logEntry.getOffset();
+ if (offset >= _minSourceOffset && offset < maxOffset) {
+ final int docId = _sourceOffsetToDocIdArray[Math.toIntExact(offset - _minSourceOffset)];
+ if (docId != DEFAULT_DOC_ID_FOR_MISSING_ENTRY) {
+ // use traditional for loop over int instead of foreach to give hints for JIT to do loop unroll in byte code
+ for (int i = 0; i < readerWriteCount; i++) {
+ _virtualColumnsReaderWriter.get(i).update(docId, logEntry.getValue(), logEntry.getType());
+ }
+ if (logEntry.getValue() > partitionToHighestWatermark.getOrDefault(partition, -1L)) {
+ partitionToHighestWatermark.put(partition, logEntry.getValue());
+ }
+ } else {
+ LOGGER.error("segment {} got in-range update log at offset {} but no matching docId", _segmentName, offset);
+ }
+ } else {
+ unmatchedLogEntryCount++;
+ }
+ }
+ if (unmatchedLogEntryCount > 0) {
+ LOGGER.info("segment {} encountered {} update logs that are outside of its range", _segmentName,
+ unmatchedLogEntryCount);
+ }
+ partitionToHighestWatermark.forEach((partition, value) ->
+ _upsertWaterMarkManager.processVersionUpdate(_tableNameWithType, partition, value));
+
+ } catch (Exception e) {
+ LOGGER.error("failed to load the offset with thread pool");
+ Utils.rethrowException(e);
+ }
+ LOGGER.info("populated all log entries to virtual columns for current immutable segment {} in {} ms",
+ _segmentName, System.currentTimeMillis() - start);
+ }
+
+ /**
+ * given a offset from source kafka topic, return the docId associated with it
+ * throw exception if there is no docId for the associated kafka offset (because kafka offset might not be continuous)
+ * @param offset offset in the source topic
+ * @return the corresponding docId
+ */
+ private int getDocIdFromSourceOffset(long offset) throws RuntimeException {
+ if (offset < _minSourceOffset || offset - _minSourceOffset >= _sourceOffsetToDocIdArray.length) {
+ LOGGER.error("offset {} is outside range for current segment {} start offset {} size {}",
+ offset, _segmentName, _minSourceOffset, _sourceOffsetToDocIdArray.length);
+ throw new RuntimeException("offset outside range");
+ } else {
+ int position = Math.toIntExact(offset - _minSourceOffset);
+ if (_sourceOffsetToDocIdArray[position] == DEFAULT_DOC_ID_FOR_MISSING_ENTRY) {
+ LOGGER.error("no docId associated with offset {} for segment {}", offset, _segmentName);
+ throw new RuntimeException("docId not found");
+ } else {
+ return _sourceOffsetToDocIdArray[position];
+ }
+ }
+
+ }
+}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/VirtualColumnContext.java b/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/mutable/MutableAppendSegmentImpl.java
similarity index 56%
copy from pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/VirtualColumnContext.java
copy to pinot-core/src/main/java/org/apache/pinot/core/indexsegment/mutable/MutableAppendSegmentImpl.java
index bb32320..1c9effc 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/VirtualColumnContext.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/mutable/MutableAppendSegmentImpl.java
@@ -16,29 +16,22 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.pinot.core.segment.virtualcolumn;
+package org.apache.pinot.core.indexsegment.mutable;
-import org.apache.pinot.spi.data.FieldSpec;
+import org.apache.pinot.core.realtime.impl.RealtimeSegmentConfig;
+import org.apache.pinot.spi.data.readers.GenericRow;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+public class MutableAppendSegmentImpl extends MutableSegmentImpl {
+ private static final Logger LOGGER = LoggerFactory.getLogger(MutableUpsertSegmentImpl.class);
-/**
- * Miscellaneous context information about the virtual column.
- * It will be used to build various components (dictionary, reader, etc) in the virtual column provider.
- */
-public class VirtualColumnContext {
- private FieldSpec _fieldSpec;
- private int _totalDocCount;
-
- public VirtualColumnContext(FieldSpec fieldSpec, int totalDocCount) {
- _fieldSpec = fieldSpec;
- _totalDocCount = totalDocCount;
- }
-
- public FieldSpec getFieldSpec() {
- return _fieldSpec;
+ public MutableAppendSegmentImpl(RealtimeSegmentConfig config) {
+ super(config);
}
- public int getTotalDocCount() {
- return _totalDocCount;
+ @Override
+ protected void postProcessRecords(GenericRow row, int docId) {
+ // nothing
}
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/mutable/MutableSegmentImpl.java b/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/mutable/MutableSegmentImpl.java
index f800d95..b80ac64 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/mutable/MutableSegmentImpl.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/mutable/MutableSegmentImpl.java
@@ -31,6 +31,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import javax.annotation.Nullable;
+
import org.apache.pinot.common.config.SegmentPartitionConfig;
import org.apache.pinot.common.segment.SegmentMetadata;
import org.apache.pinot.core.indexsegment.IndexSegmentUtils;
@@ -56,6 +57,7 @@ import org.apache.pinot.core.segment.index.data.source.ColumnDataSource;
import org.apache.pinot.core.segment.index.readers.BloomFilterReader;
import org.apache.pinot.core.segment.index.readers.InvertedIndexReader;
import org.apache.pinot.core.segment.index.readers.NullValueVectorReader;
+import org.apache.pinot.core.segment.index.readers.Dictionary;
import org.apache.pinot.core.segment.virtualcolumn.VirtualColumnContext;
import org.apache.pinot.core.segment.virtualcolumn.VirtualColumnProvider;
import org.apache.pinot.core.segment.virtualcolumn.VirtualColumnProviderFactory;
@@ -74,7 +76,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-public class MutableSegmentImpl implements MutableSegment {
+public abstract class MutableSegmentImpl implements MutableSegment {
// For multi-valued column, forward-index.
// Maximum number of multi-values per row. We assert on this.
private static final int MAX_MULTI_VALUES_PER_ROW = 1000;
@@ -86,11 +88,12 @@ public class MutableSegmentImpl implements MutableSegment {
private static final int NODICT_VARIABLE_WIDTH_ESTIMATED_AVERAGE_VALUE_LENGTH_DEFAULT = 100;
private static final int NODICT_VARIABLE_WIDTH_ESTIMATED_NUMBER_OF_VALUES_DEFAULT = 100_000;
- private final Logger _logger;
+ protected final Logger _logger;
private final long _startTimeMillis = System.currentTimeMillis();
- private final String _segmentName;
- private final Schema _schema;
+ protected final String _tableName;
+ protected final String _segmentName;
+ protected final Schema _schema;
private final int _capacity;
private final SegmentMetadata _segmentMetadata;
private final boolean _offHeap;
@@ -108,18 +111,23 @@ public class MutableSegmentImpl implements MutableSegment {
private final IdMap<FixedIntArray> _recordIdMap;
private boolean _aggregateMetrics;
- private volatile int _numDocsIndexed = 0;
+ protected volatile int _numDocsIndexed = 0;
// to compute the rolling interval
- private volatile long _minTime = Long.MAX_VALUE;
- private volatile long _maxTime = Long.MIN_VALUE;
- private final int _numKeyColumns;
+ protected volatile long _minTime = Long.MAX_VALUE;
+ protected volatile long _maxTime = Long.MIN_VALUE;
+ protected final int _numKeyColumns;
// Cache the physical (non-virtual) field specs
private final Collection<FieldSpec> _physicalFieldSpecs;
private final Collection<DimensionFieldSpec> _physicalDimensionFieldSpecs;
private final Collection<MetricFieldSpec> _physicalMetricFieldSpecs;
+ // Cache some virtual columns
+ protected final Map<String, VirtualColumnProvider> _virtualColumnProviderMap = new HashMap<>();
+ protected final Map<String, Dictionary> _virtualColumnDictionary = new HashMap<>();
+ protected final Map<String, DataFileReader> _virtualColumnIndexReader = new HashMap<>();
+
// default message metadata
private volatile long _lastIndexedTimeMs = Long.MIN_VALUE;
private volatile long _latestIngestionTimeMs = Long.MIN_VALUE;
@@ -127,6 +135,7 @@ public class MutableSegmentImpl implements MutableSegment {
private RealtimeLuceneReaders _realtimeLuceneReaders;
public MutableSegmentImpl(RealtimeSegmentConfig config) {
+ _tableName = config.getTableName();
_segmentName = config.getSegmentName();
_schema = config.getSchema();
_capacity = config.getCapacity();
@@ -161,6 +170,7 @@ public class MutableSegmentImpl implements MutableSegment {
Collection<FieldSpec> allFieldSpecs = _schema.getAllFieldSpecs();
List<FieldSpec> physicalFieldSpecs = new ArrayList<>(allFieldSpecs.size());
+ List<FieldSpec> virtualFieldSpecs = new ArrayList<>(allFieldSpecs.size());
List<DimensionFieldSpec> physicalDimensionFieldSpecs = new ArrayList<>(_schema.getDimensionNames().size());
List<MetricFieldSpec> physicalMetricFieldSpecs = new ArrayList<>(_schema.getMetricNames().size());
@@ -174,6 +184,9 @@ public class MutableSegmentImpl implements MutableSegment {
} else if (fieldType == FieldSpec.FieldType.METRIC) {
physicalMetricFieldSpecs.add((MetricFieldSpec) fieldSpec);
}
+ } else {
+ virtualFieldSpecs.add(fieldSpec);
+
}
}
_physicalFieldSpecs = Collections.unmodifiableCollection(physicalFieldSpecs);
@@ -191,6 +204,7 @@ public class MutableSegmentImpl implements MutableSegment {
int avgNumMultiValues = config.getAvgNumMultiValues();
+
// Initialize for each column
for (FieldSpec fieldSpec : _physicalFieldSpecs) {
String column = fieldSpec.getName();
@@ -269,6 +283,7 @@ public class MutableSegmentImpl implements MutableSegment {
_indexReaderWriterMap.put(column, indexReaderWriter);
+
if (invertedIndexColumns.contains(column)) {
_invertedIndexMap.put(column, new RealtimeInvertedIndexReader());
}
@@ -287,6 +302,21 @@ public class MutableSegmentImpl implements MutableSegment {
}
}
+ for (FieldSpec fieldSpec : _physicalFieldSpecs) {
+ String column = fieldSpec.getName();
+ VirtualColumnContext virtualColumnContext = new VirtualColumnContext(fieldSpec, _capacity, true);
+ final VirtualColumnProvider provider =
+ VirtualColumnProviderFactory.buildProvider(fieldSpec.getVirtualColumnProvider());
+ if (provider == null) {
+ throw new RuntimeException(String.format("failed to create virtual segment provider for field %s", fieldSpec.getName()));
+ }
+ DataFileReader reader = provider.buildReader(virtualColumnContext);
+ Dictionary dictionary = provider.buildDictionary(virtualColumnContext);
+ _virtualColumnProviderMap.put(column, provider);
+ _virtualColumnIndexReader.put(column, reader);
+ _virtualColumnDictionary.put(column, dictionary);
+ }
+
if (_realtimeLuceneReaders != null) {
// add the realtime lucene index readers to the global queue for refresh task to pick up
RealtimeLuceneIndexRefreshState realtimeLuceneIndexRefreshState = RealtimeLuceneIndexRefreshState.getInstance();
@@ -347,6 +377,7 @@ public class MutableSegmentImpl implements MutableSegment {
// Update number of document indexed at last to make the latest record queryable
canTakeMore = _numDocsIndexed++ < _capacity;
+ postProcessRecords(row, docId);
} else {
Preconditions
.checkState(_aggregateMetrics, "Invalid document-id during indexing: " + docId + " expected: " + numDocs);
@@ -491,6 +522,8 @@ public class MutableSegmentImpl implements MutableSegment {
}
}
+ protected abstract void postProcessRecords(GenericRow row, int docId);
+
private boolean aggregateMetrics(GenericRow row, int docId) {
for (MetricFieldSpec metricFieldSpec : _physicalMetricFieldSpecs) {
String column = metricFieldSpec.getName();
@@ -556,9 +589,9 @@ public class MutableSegmentImpl implements MutableSegment {
public ColumnDataSource getDataSource(String columnName) {
FieldSpec fieldSpec = _schema.getFieldSpecFor(columnName);
if (fieldSpec.isVirtualColumn()) {
- VirtualColumnContext virtualColumnContext = new VirtualColumnContext(fieldSpec, _numDocsIndexed);
- VirtualColumnProvider virtualColumnProvider =
- VirtualColumnProviderFactory.buildProvider(_schema.getFieldSpecFor(columnName).getVirtualColumnProvider());
+ // FIXME
+ VirtualColumnContext virtualColumnContext = new VirtualColumnContext(fieldSpec, _numDocsIndexed, true);
+ VirtualColumnProvider virtualColumnProvider = _virtualColumnProviderMap.get(columnName);
return new ColumnDataSource(virtualColumnProvider.buildColumnIndexContainer(virtualColumnContext),
virtualColumnProvider.buildMetadata(virtualColumnContext));
} else {
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/mutable/MutableUpsertSegmentImpl.java b/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/mutable/MutableUpsertSegmentImpl.java
new file mode 100644
index 0000000..a6e7728
--- /dev/null
+++ b/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/mutable/MutableUpsertSegmentImpl.java
@@ -0,0 +1,159 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.core.indexsegment.mutable;
+
+import com.google.common.base.Preconditions;
+import org.apache.pinot.core.indexsegment.UpsertSegment;
+import org.apache.pinot.core.io.reader.DataFileReader;
+import org.apache.pinot.core.realtime.impl.RealtimeSegmentConfig;
+import org.apache.pinot.core.segment.updater.UpsertWaterMarkManager;
+import org.apache.pinot.core.segment.virtualcolumn.mutable.VirtualColumnLongValueReaderWriter;
+import org.apache.pinot.grigio.common.messages.LogEventType;
+import org.apache.pinot.grigio.common.storageProvider.UpdateLogEntry;
+import org.apache.pinot.grigio.common.storageProvider.UpdateLogEntrySet;
+import org.apache.pinot.grigio.common.storageProvider.UpdateLogStorageProvider;
+import org.apache.pinot.spi.data.readers.GenericRow;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+public class MutableUpsertSegmentImpl extends MutableSegmentImpl implements UpsertSegment {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(MutableUpsertSegmentImpl.class);
+ private final UpsertWaterMarkManager upsertWaterMarkManager;
+
+ private final String _kafkaOffsetColumnName;
+
+ private final List<VirtualColumnLongValueReaderWriter> _mutableSegmentReaderWriters = new ArrayList<>();
+ // use map for mapping between kafka offset and docId because we at-most have 1 mutable segment per consumer
+ // will use more memory, but we can update this later
+ private final Map<Long, Integer> _sourceOffsetToDocId = new ConcurrentHashMap<>();
+ // to store the update event that arrive before my current record
+ // TODO remove this in the later version of design if necessary
+ private final Map<Long, UpdateLogEntry> _unmatchedInsertRecords = new ConcurrentHashMap<>();
+ private final Map<Long, UpdateLogEntry> _unmatchedDeleteRecords = new ConcurrentHashMap<>();
+
+ public MutableUpsertSegmentImpl(RealtimeSegmentConfig config) {
+ super(config);
+ _kafkaOffsetColumnName = _schema.getOffsetKey();
+ Preconditions.checkState(_schema.isTableForUpsert(), "table should be upsert");
+ for (Map.Entry<String, DataFileReader> entry: _virtualColumnIndexReader.entrySet()) {
+ String column = entry.getKey();
+ DataFileReader reader = entry.getValue();
+ if (reader instanceof VirtualColumnLongValueReaderWriter) {
+ _logger.info("adding virtual column {} to updatable reader writer list", column);
+ _mutableSegmentReaderWriters.add((VirtualColumnLongValueReaderWriter) reader);
+ }
+ }
+ upsertWaterMarkManager = UpsertWaterMarkManager.getInstance();
+ LOGGER.info("starting upsert segment with {} reader writer", _mutableSegmentReaderWriters.size());
+ }
+
+ @Override
+ public synchronized void updateVirtualColumn(Iterable<UpdateLogEntry> logEntries) {
+ for (UpdateLogEntry logEntry: logEntries) {
+ boolean updated = false;
+ boolean offsetFound = false;
+ Integer docId = _sourceOffsetToDocId.get(logEntry.getOffset());
+ if (docId != null) {
+ offsetFound = true;
+ for (VirtualColumnLongValueReaderWriter readerWriter : _mutableSegmentReaderWriters) {
+ updated = readerWriter.update(docId, logEntry.getValue(), logEntry.getType()) || updated;
+ }
+ if (updated) {
+ // only update high water mark if it indeed updated something
+ upsertWaterMarkManager.processMessage(_tableName, _segmentName, logEntry);
+ }
+ }
+ if (!offsetFound) {
+ putEntryToUnmatchMap(logEntry);
+ }
+ }
+ }
+
+ @Override
+ public String getVirtualColumnInfo(long offset) {
+ Integer docId = _sourceOffsetToDocId.get(offset);
+ StringBuilder result = new StringBuilder("matched: ");
+ if (docId == null) {
+ result = new StringBuilder("no doc id found ");
+ } else {
+ for (VirtualColumnLongValueReaderWriter readerWriter : _mutableSegmentReaderWriters) {
+ result.append(readerWriter.getLong(docId)).append("; ");
+ }
+ }
+ if (_unmatchedInsertRecords.containsKey(offset)) {
+ result.append(" unmatched insert: ").append(_unmatchedInsertRecords.get(offset).getValue());
+ }
+ if (_unmatchedDeleteRecords.containsKey(offset)) {
+ result.append(" unmatched delete: ").append(_unmatchedDeleteRecords.get(offset).getValue());
+ }
+ return result.toString();
+ }
+
+ @Override
+ protected synchronized void postProcessRecords(GenericRow row, int docId) {
+ final Long offset = (Long) row.getValue(_kafkaOffsetColumnName);
+ for (VirtualColumnLongValueReaderWriter readerWriter: _mutableSegmentReaderWriters) {
+ readerWriter.addNewRecord(docId);
+ }
+ _sourceOffsetToDocId.put(offset, docId);
+ checkForOutstandingRecords(_unmatchedDeleteRecords, offset, docId);
+ checkForOutstandingRecords(_unmatchedInsertRecords, offset, docId);
+ }
+
+ @Override
+ public void initVirtualColumn() throws IOException {
+ Preconditions.checkState(_numDocsIndexed == 0, "should init virtual column before ingestion");
+ UpdateLogEntrySet updateLogEntries = UpdateLogStorageProvider.getInstance().getAllMessages(_tableName, _segmentName);
+ LOGGER.info("got {} update log entries for current segment {}", updateLogEntries.size(), _segmentName);
+ // some physical data might have been ingested when we init virtual column, we will go through the normal update
+ // flow to ensure we wont miss records
+ updateVirtualColumn(updateLogEntries);
+ }
+
+ private void checkForOutstandingRecords(Map<Long, UpdateLogEntry> unmatchRecordsMap, Long offset, int docId) {
+ UpdateLogEntry unmatchedEntry = unmatchRecordsMap.remove(offset);
+ if (unmatchedEntry != null) {
+ boolean updated = false;
+ for (VirtualColumnLongValueReaderWriter readerWriter: _mutableSegmentReaderWriters) {
+ updated = readerWriter.update(docId, unmatchedEntry.getValue(), unmatchedEntry.getType()) || updated;
+ }
+ if (updated) {
+ upsertWaterMarkManager.processMessage(_tableName, _segmentName, unmatchedEntry);
+ }
+ }
+ }
+
+ private void putEntryToUnmatchMap(UpdateLogEntry logEntry) {
+ final Map<Long, UpdateLogEntry> unmatchedRecords;
+ if (logEntry.getType() == LogEventType.INSERT) {
+ unmatchedRecords = _unmatchedInsertRecords;
+ } else {
+ unmatchedRecords = _unmatchedDeleteRecords;
+ }
+ unmatchedRecords.put(logEntry.getOffset(), logEntry);
+ }
+
+}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/realtime/impl/RealtimeSegmentConfig.java b/pinot-core/src/main/java/org/apache/pinot/core/realtime/impl/RealtimeSegmentConfig.java
index 9aef2a2..827b0f7 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/realtime/impl/RealtimeSegmentConfig.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/realtime/impl/RealtimeSegmentConfig.java
@@ -27,6 +27,7 @@ import org.apache.pinot.core.io.readerwriter.PinotDataBufferMemoryManager;
public class RealtimeSegmentConfig {
+ private final String _tableName;
private final String _segmentName;
private final String _streamName;
private final Schema _schema;
@@ -45,12 +46,13 @@ public class RealtimeSegmentConfig {
private final boolean _nullHandlingEnabled;
private final String _consumerDir;
- private RealtimeSegmentConfig(String segmentName, String streamName, Schema schema, int capacity,
+ private RealtimeSegmentConfig(String tableName, String segmentName, String streamName, Schema schema, int capacity,
int avgNumMultiValues, Set<String> noDictionaryColumns, Set<String> varLengthDictionaryColumns,
Set<String> invertedIndexColumns, Set<String> textIndexColumns, RealtimeSegmentZKMetadata realtimeSegmentZKMetadata,
boolean offHeap, PinotDataBufferMemoryManager memoryManager, RealtimeSegmentStatsHistory statsHistory,
SegmentPartitionConfig segmentPartitionConfig, boolean aggregateMetrics, boolean nullHandlingEnabled,
String consumerDir) {
+ _tableName = tableName;
_segmentName = segmentName;
_streamName = streamName;
_schema = schema;
@@ -70,6 +72,10 @@ public class RealtimeSegmentConfig {
_consumerDir = consumerDir;
}
+ public String getTableName() {
+ return _tableName;
+ }
+
public String getSegmentName() {
return _segmentName;
}
@@ -144,6 +150,7 @@ public class RealtimeSegmentConfig {
}
public static class Builder {
+ private String _tableName;
private String _segmentName;
private String _streamName;
private Schema _schema;
@@ -165,6 +172,11 @@ public class RealtimeSegmentConfig {
public Builder() {
}
+ public Builder setTableName(String tableName) {
+ _tableName = tableName;
+ return this;
+ }
+
public Builder setSegmentName(String segmentName) {
_segmentName = segmentName;
return this;
@@ -259,7 +271,7 @@ public class RealtimeSegmentConfig {
}
public RealtimeSegmentConfig build() {
- return new RealtimeSegmentConfig(_segmentName, _streamName, _schema, _capacity, _avgNumMultiValues,
+ return new RealtimeSegmentConfig(_tableName, _segmentName, _streamName, _schema, _capacity, _avgNumMultiValues,
_noDictionaryColumns, _varLengthDictionaryColumns, _invertedIndexColumns, _textIndexColumns,
_realtimeSegmentZKMetadata, _offHeap, _memoryManager, _statsHistory, _segmentPartitionConfig,
_aggregateMetrics, _nullHandlingEnabled, _consumerDir);
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/segment/index/SegmentMetadataImpl.java b/pinot-core/src/main/java/org/apache/pinot/core/segment/index/SegmentMetadataImpl.java
index 7a3dad3..43d28e5 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/segment/index/SegmentMetadataImpl.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/segment/index/SegmentMetadataImpl.java
@@ -22,6 +22,15 @@ import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.base.Preconditions;
+import org.apache.commons.configuration.ConfigurationException;
+import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.commons.lang.StringEscapeUtils;
+import org.joda.time.Duration;
+import org.joda.time.Interval;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.Nullable;
import java.io.DataInputStream;
import java.io.File;
import java.io.FileInputStream;
@@ -39,10 +48,6 @@ import java.util.Map;
import java.util.Set;
import java.util.TimeZone;
import java.util.concurrent.TimeUnit;
-import javax.annotation.Nullable;
-import org.apache.commons.configuration.ConfigurationException;
-import org.apache.commons.configuration.PropertiesConfiguration;
-import org.apache.commons.lang.StringEscapeUtils;
import org.apache.pinot.spi.data.MetricFieldSpec;
import org.apache.pinot.spi.data.Schema;
import org.apache.pinot.common.metadata.segment.RealtimeSegmentZKMetadata;
@@ -58,10 +63,6 @@ import org.apache.pinot.core.startree.v2.StarTreeV2Constants;
import org.apache.pinot.core.startree.v2.StarTreeV2Metadata;
import org.apache.pinot.startree.hll.HllConstants;
import org.joda.time.DateTimeZone;
-import org.joda.time.Duration;
-import org.joda.time.Interval;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import static org.apache.pinot.core.segment.creator.impl.V1Constants.MetadataKeys.Segment.*;
import static org.apache.pinot.core.segment.creator.impl.V1Constants.MetadataKeys.StarTree.*;
@@ -108,6 +109,15 @@ public class SegmentMetadataImpl implements SegmentMetadata {
*/
public SegmentMetadataImpl(File indexDir)
throws IOException {
+ this(indexDir, null);
+ }
+
+ /**
+ * For segments on disk and segment is upsert, will use the sechema hint to init virtual column
+ * <p>Index directory passed in should be top level segment directory.
+ * <p>If segment metadata file exists in multiple segment version, load the one in highest segment version.
+ */
+ public SegmentMetadataImpl(File indexDir, Schema schemaHint) throws IOException {
_indexDir = indexDir;
PropertiesConfiguration segmentMetadataPropertiesConfiguration = getPropertiesConfiguration(indexDir);
_columnMetadataMap = new HashMap<>();
@@ -115,6 +125,9 @@ public class SegmentMetadataImpl implements SegmentMetadata {
_schema = new Schema();
init(segmentMetadataPropertiesConfiguration);
+ // adding necessary schema hints for upsert table
+ _schema.withSchemaHint(schemaHint);
+
File creationMetaFile = SegmentDirectoryPaths.findCreationMetaFile(indexDir);
if (creationMetaFile != null) {
loadCreationMeta(creationMetaFile);
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/VirtualColumnContext.java b/pinot-core/src/main/java/org/apache/pinot/core/segment/updater/SegmentDeletionListener.java
similarity index 55%
copy from pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/VirtualColumnContext.java
copy to pinot-core/src/main/java/org/apache/pinot/core/segment/updater/SegmentDeletionListener.java
index bb32320..a29feb7 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/VirtualColumnContext.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/segment/updater/SegmentDeletionListener.java
@@ -16,29 +16,8 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.pinot.core.segment.virtualcolumn;
+package org.apache.pinot.core.segment.updater;
-import org.apache.pinot.spi.data.FieldSpec;
-
-
-/**
- * Miscellaneous context information about the virtual column.
- * It will be used to build various components (dictionary, reader, etc) in the virtual column provider.
- */
-public class VirtualColumnContext {
- private FieldSpec _fieldSpec;
- private int _totalDocCount;
-
- public VirtualColumnContext(FieldSpec fieldSpec, int totalDocCount) {
- _fieldSpec = fieldSpec;
- _totalDocCount = totalDocCount;
- }
-
- public FieldSpec getFieldSpec() {
- return _fieldSpec;
- }
-
- public int getTotalDocCount() {
- return _totalDocCount;
- }
+public interface SegmentDeletionListener {
+ void onSegmentDeletion(String tableNameWithType, String segmentName);
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/segment/updater/SegmentUpdater.java b/pinot-core/src/main/java/org/apache/pinot/core/segment/updater/SegmentUpdater.java
new file mode 100644
index 0000000..d0d8c0e
--- /dev/null
+++ b/pinot-core/src/main/java/org/apache/pinot/core/segment/updater/SegmentUpdater.java
@@ -0,0 +1,349 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.core.segment.updater;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.common.util.concurrent.Uninterruptibles;
+import io.netty.util.internal.ConcurrentSet;
+import org.apache.commons.configuration.Configuration;
+import org.apache.pinot.common.config.TableNameBuilder;
+import org.apache.pinot.common.utils.CommonConstants;
+import org.apache.pinot.common.utils.LLCSegmentName;
+import org.apache.pinot.core.data.manager.UpsertSegmentDataManager;
+import org.apache.pinot.grigio.common.messages.LogCoordinatorMessage;
+import org.apache.pinot.grigio.common.metrics.GrigioMeter;
+import org.apache.pinot.grigio.common.metrics.GrigioMetrics;
+import org.apache.pinot.grigio.common.metrics.GrigioTimer;
+import org.apache.pinot.grigio.common.rpcQueue.QueueConsumer;
+import org.apache.pinot.grigio.common.rpcQueue.QueueConsumerRecord;
+import org.apache.pinot.grigio.common.storageProvider.UpdateLogEntry;
+import org.apache.pinot.grigio.common.storageProvider.UpdateLogStorageProvider;
+import org.apache.pinot.grigio.common.storageProvider.retentionManager.UpdateLogRetentionManager;
+import org.apache.pinot.grigio.common.utils.CommonUtils;
+import org.apache.pinot.grigio.servers.SegmentUpdaterProvider;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.concurrent.ThreadSafe;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * class to perform fetching updates for upsert related information from kafka to local machine and fill in
+ * virtual columns. It should be started after all segments are loaded in current pinot server.
+ */
+@ThreadSafe
+public class SegmentUpdater implements SegmentDeletionListener {
+ private static final Logger LOGGER = LoggerFactory.getLogger(SegmentUpdater.class);
+
+ private static volatile SegmentUpdater _instance = null;
+
+ private static final long NO_MESSAGE_SLEEP_MS = 100;
+ private static final long SHUTDOWN_WAIT_MS = 2000;
+ private static final long LOGGER_TIME_GAP_MS = 5000;
+
+ private final Configuration _conf;
+ private final int _updateSleepMs;
+ private final String _topicPrefix;
+ private final ExecutorService _ingestionExecutorService;
+ private final QueueConsumer _consumer;
+ private final Map<String, Map<String, Set<UpsertSegmentDataManager>>> _tableSegmentMap = new ConcurrentHashMap<>();
+ private final Map<String, Map<Integer, Long>> _tablePartitionCreationTime = new ConcurrentHashMap<>();
+ private final UpdateLogStorageProvider _updateLogStorageProvider;
+ private final UpdateLogRetentionManager _retentionManager;
+ private final GrigioMetrics _metrics;
+
+ private volatile boolean isStarted = true;
+
+ public SegmentUpdater(Configuration conf, SegmentUpdaterProvider provider, UpdateLogRetentionManager retentionManager,
+ GrigioMetrics metrics) {
+ _conf = conf;
+ _metrics = metrics;
+ _retentionManager = retentionManager;
+ _topicPrefix = conf.getString(SegmentUpdaterConfig.INPUT_TOPIC_PREFIX);
+ _updateSleepMs = conf.getInt(SegmentUpdaterConfig.SEGMENT_UDPATE_SLEEP_MS,
+ SegmentUpdaterConfig.SEGMENT_UDPATE_SLEEP_MS_DEFAULT);
+ UpsertWaterMarkManager.init(metrics);
+ _consumer = provider.getConsumer();
+ _ingestionExecutorService = Executors.newFixedThreadPool(1);
+ _updateLogStorageProvider = UpdateLogStorageProvider.getInstance();
+ _instance = this;
+ }
+
+ public static SegmentUpdater getInstance() {
+ Preconditions.checkState(_instance != null, "there is no instance for segment updater");
+ return _instance;
+ }
+
+
+ public void start() {
+ String listOfTables = Joiner.on(",").join(_tableSegmentMap.keySet());
+ LOGGER.info("starting segment updater main loop with the following table in server: {}", listOfTables);
+ _ingestionExecutorService.submit(this::updateLoop);
+ }
+
+ public void shutdown() {
+ LOGGER.info("closing the segment updater");
+ isStarted = false;
+ _ingestionExecutorService.shutdown();
+ try {
+ _ingestionExecutorService.awaitTermination(SHUTDOWN_WAIT_MS, TimeUnit.MILLISECONDS);
+ } catch (InterruptedException e) {
+ LOGGER.error("failed to wait for shutdown", e);
+ }
+ _ingestionExecutorService.shutdownNow();
+ try {
+ _updateLogStorageProvider.close();
+ } catch (IOException ex) {
+ LOGGER.error("failed to shutdown update log provider", ex);
+ }
+ LOGGER.info("finished shutdown of segment updater service");
+ }
+
+ /**
+ * this method will perform the following:
+ * 1. fetch a list of updates for pinot upsert from kafka consumer
+ * 2. organize the message by table/segment/List<UpdateLogEntry> map
+ * 3. call to save the update data in local file system
+ * 4. apply updates to appropriate data manager
+ * 5. commit consumer offset
+ */
+ private void updateLoop() {
+ try {
+ LOGGER.info("starting update loop");
+ long lastReportedTime = System.currentTimeMillis();
+ while (isStarted) {
+ long startTime = System.currentTimeMillis();
+ long loopStartTime = startTime;
+ final List<QueueConsumerRecord<String, LogCoordinatorMessage>> records = _consumer.getRequests(_updateSleepMs, TimeUnit.MILLISECONDS);
+ _metrics.addTimedValueMs(GrigioTimer.FETCH_MSG_FROM_CONSUMER_TIME, System.currentTimeMillis() - startTime);
+ int eventCount = records.size();
+ _metrics.addMeteredGlobalValue(GrigioMeter.MESSAGE_FETCH_PER_ROUND, eventCount);
+
+ final Map<String, TableUpdateLogs> tableSegmentToUpdateLogs = new HashMap<>();
+ // organize the update logs by {tableName: {segmentName: {list of updatelogs}}}
+ records.iterator().forEachRemaining(consumerRecord -> {
+ TableUpdateLogs tableUpdateLogs = tableSegmentToUpdateLogs.computeIfAbsent(
+ CommonUtils.getTableNameFromKafkaTopic(consumerRecord.getTopic(), _topicPrefix),
+ t -> new TableUpdateLogs());
+ tableUpdateLogs.addUpdateLogEntry(consumerRecord.getRecord(), consumerRecord.getPartition());
+ });
+
+ startTime = System.currentTimeMillis();
+ AtomicLong timeToStoreUpdateLogs = new AtomicLong(0);
+ for (Map.Entry<String, TableUpdateLogs> entry : tableSegmentToUpdateLogs.entrySet()) {
+ String tableName = TableNameBuilder.ensureTableNameWithType(entry.getKey(), CommonConstants.Helix.TableType.REALTIME);
+ int tableMessageCount = 0;
+ if (_tableSegmentMap.containsKey(tableName)) {
+ final Map<String, Set<UpsertSegmentDataManager>> segmentManagersMap = _tableSegmentMap.get(tableName);
+ final TableUpdateLogs segment2UpdateLogsMap = entry.getValue();
+ updateSegmentVirtualColumns(tableName, segmentManagersMap, segment2UpdateLogsMap, timeToStoreUpdateLogs);
+ } else {
+ LOGGER.warn("got messages for table {} not in this server", tableName);
+ }
+ _metrics.addMeteredTableValue(tableName, GrigioMeter.MESSAGE_FETCH_PER_ROUND, tableMessageCount);
+ }
+ _metrics.addTimedValueMs(GrigioTimer.UPDATE_LOCAL_LOG_FILE_TIME, timeToStoreUpdateLogs.get());
+ _metrics.addTimedValueMs(GrigioTimer.UPDATE_DATAMANAGER_TIME, System.currentTimeMillis() - startTime);
+ if (eventCount == 0) {
+ Uninterruptibles.sleepUninterruptibly(NO_MESSAGE_SLEEP_MS, TimeUnit.MILLISECONDS);
+ } else {
+ if (System.currentTimeMillis() - lastReportedTime > LOGGER_TIME_GAP_MS) {
+ lastReportedTime = System.currentTimeMillis();
+ LOGGER.info("processed {} messages in {} ms", eventCount, System.currentTimeMillis() - loopStartTime);
+ LOGGER.info("latest high water mark is {}", UpsertWaterMarkManager.getInstance().toString());
+ }
+ _consumer.ackOffset();
+ _metrics.addTimedValueMs(GrigioTimer.SEGMENT_UPDATER_LOOP_TIME, System.currentTimeMillis() - loopStartTime);
+ }
+ }
+ } catch (Exception ex) {
+ LOGGER.error("failed at segment updates", ex);
+ } finally {
+ LOGGER.info("exiting segment update loop");
+ }
+ }
+
+ private void storeUpdateLogs(String table, String segment, List<UpdateLogEntry> messages,
+ AtomicLong timeToStoreUpdateLogs) throws IOException {
+ long startTime = System.currentTimeMillis();
+ _updateLogStorageProvider.addDataToFile(table, segment, messages);
+ timeToStoreUpdateLogs.addAndGet(System.currentTimeMillis() - startTime);
+ }
+
+ /**
+ * Update the virtual columns of affected segments of a table.
+ */
+ private void updateSegmentVirtualColumns(String tableName, Map<String, Set<UpsertSegmentDataManager>> segmentManagersMap,
+ TableUpdateLogs segment2UpdateLogsMap, AtomicLong timeToStoreUpdateLogs) throws IOException{
+ for (Map.Entry<String, List<UpdateLogEntry>> segmentEntry : segment2UpdateLogsMap.getSegments2UpdateLog().entrySet()) {
+ final String segmentNameStr = segmentEntry.getKey();
+ updateVirtualColumn(tableName, segmentNameStr,
+ segmentManagersMap.computeIfAbsent(segmentNameStr, sn -> new ConcurrentSet<>()),
+ segment2UpdateLogsMap.get(segmentNameStr), timeToStoreUpdateLogs);
+ }
+ }
+
+ /**
+ * in pinot server, there could be multiple segment data managers per table/segment pair during pinot switch a segment
+ * from consuming to online (mutable segment to immutable segment). In most of cases we expect only one segment manager
+ * in this set of UpsertSegmentDataManager
+ */
+ private void updateVirtualColumn(String table, String segment, Set<UpsertSegmentDataManager> segmentDataManagers,
+ List<UpdateLogEntry> messages, AtomicLong timeToStoreUpdateLogs) throws IOException {
+ LOGGER.debug("updating segment {} with {} results for {} data managers", segment, messages.size(),
+ segmentDataManagers.size());
+ if (segmentDataManagers.size() > 0 || _retentionManager.getRetentionManagerForTable(table).shouldIngestForSegment(segment)) {
+ storeUpdateLogs(table, segment, messages, timeToStoreUpdateLogs);
+ }
+ try {
+ for (UpsertSegmentDataManager dataManager: segmentDataManagers) {
+ dataManager.updateVirtualColumns(messages);
+ }
+ } catch (Exception ex) {
+ LOGGER.error("failed to update virtual column for key ", ex);
+ }
+ }
+
+ /**
+ * called when we create a new segment data manager, associate this data manager with the given table/segment info
+ * @param tableNameWithType
+ * @param segmentName
+ * @param dataManager the data manager for the current given table/segment combination
+ */
+ public synchronized void addSegmentDataManager(String tableNameWithType, LLCSegmentName segmentName, UpsertSegmentDataManager dataManager) {
+ // TODO get partition assignment from
+ LOGGER.info("segment updater adding table {} segment {}", tableNameWithType, segmentName.getSegmentName());
+ if (!_tableSegmentMap.containsKey(tableNameWithType)) {
+ synchronized (_tableSegmentMap) {
+ _tableSegmentMap.put(tableNameWithType, new ConcurrentHashMap<>());
+ }
+ LOGGER.info("adding table {} to segment updater consumer", tableNameWithType);
+ handleNewTableInServer(tableNameWithType);
+ }
+ _tableSegmentMap.get(tableNameWithType).computeIfAbsent(segmentName.getSegmentName(), sn -> new HashSet<>()).add(dataManager);
+ synchronized (_tablePartitionCreationTime) {
+ long creationTime = _tablePartitionCreationTime.computeIfAbsent(tableNameWithType, t -> new ConcurrentHashMap<>())
+ .computeIfAbsent(segmentName.getPartitionId(), p -> segmentName.getCreationTimeStamp());
+ _tablePartitionCreationTime.get(tableNameWithType)
+ .put(segmentName.getPartitionId(), Long.max(creationTime, segmentName.getCreationTimeStamp()));
+ }
+ }
+
+ public synchronized void removeSegmentDataManager(String tableNameWithType, String segmentName, UpsertSegmentDataManager toDeleteManager) {
+ LOGGER.info("segment updater removing table {} segment {}", tableNameWithType, segmentName);
+ Map<String, Set<UpsertSegmentDataManager>> segmentMap = _tableSegmentMap.get(tableNameWithType);
+ if (segmentMap != null) {
+ Set<UpsertSegmentDataManager> segmentDataManagers = segmentMap.get(segmentName);
+ if (segmentDataManagers != null) {
+ segmentDataManagers.remove(toDeleteManager);
+ if (segmentDataManagers.size() == 0) {
+ segmentMap.remove(segmentName);
+ }
+ }
+ }
+ }
+
+ /**
+ * handle how to read update logs we need to do for adding a new pinot table
+ * need to do the following:
+ * subscribe to table update log kafka topics
+ * @param tableNameWithType the name of the table without
+ */
+ private void handleNewTableInServer(String tableNameWithType) {
+ LOGGER.info("subscribing to new table {}", tableNameWithType);
+ // init the retention manager to ensure we get the first ideal state
+ _retentionManager.getRetentionManagerForTable(tableNameWithType);
+ _consumer.subscribeForTable(TableNameBuilder.extractRawTableName(tableNameWithType), _topicPrefix);
+ }
+
+ /**
+ * handle clean up when a table no longer has any segment in the current server
+ * @param tableNameWithType
+ */
+ private void handleTableRemovalInServer(String tableNameWithType) {
+ LOGGER.info("unsubscribe to old table {}", tableNameWithType);
+ // key coordinator generate message without table name
+ _consumer.unsubscribeForTable(TableNameBuilder.extractRawTableName(tableNameWithType), _topicPrefix);
+ }
+
+ @Override
+ public synchronized void onSegmentDeletion(String tableNameWithType, String segmentName) {
+ LOGGER.info("deleting segment virtual column from local storage for table {} segment {}", tableNameWithType, segmentName);
+ Map<String, Set<UpsertSegmentDataManager>> segmentManagerMap = _tableSegmentMap.get(tableNameWithType);
+ if (segmentManagerMap != null) {
+ if (segmentManagerMap.containsKey(segmentName) && segmentManagerMap.get(segmentName).size() > 0) {
+ LOGGER.error("trying to remove segment storage with {} segment data manager", segmentManagerMap.get(segmentName).size());
+ }
+ try {
+ segmentManagerMap.remove(segmentName);
+ _retentionManager.getRetentionManagerForTable(tableNameWithType).notifySegmentDeletion(tableNameWithType);
+ _updateLogStorageProvider.removeSegment(tableNameWithType, segmentName);
+ } catch (IOException e) {
+ throw new RuntimeException(String.format("failed to delete table %s segment %s", tableNameWithType, segmentName), e);
+ }
+ if (segmentManagerMap.size() == 0) {
+ _tableSegmentMap.remove(tableNameWithType);
+ handleTableRemovalInServer(tableNameWithType);
+ }
+ } else {
+ LOGGER.error("deleting a segment {}:{} from current server but don't have segment map on updater",
+ tableNameWithType, segmentName);
+ }
+ }
+
+ // A table's update logs grouped by segment names.
+ private class TableUpdateLogs {
+ private Map<String, List<UpdateLogEntry>> _segments2UpdateLog;
+
+ public TableUpdateLogs() {
+ _segments2UpdateLog = new HashMap<>();
+ }
+
+ public Map<String, List<UpdateLogEntry>> getSegments2UpdateLog() {
+ return Collections.unmodifiableMap(_segments2UpdateLog);
+ }
+
+ // Partition is the partition where the record appears in the Segment Update Event message queue.
+ public void addUpdateLogEntry(LogCoordinatorMessage record, int partition) {
+ if (record == null || record.getSegmentName() == null) {
+ LOGGER.error("Empty update log or no segment in the entry: {}", record);
+ return;
+ }
+ _segments2UpdateLog.computeIfAbsent(
+ record.getSegmentName(), s -> new ArrayList<>()).add(new UpdateLogEntry(record, partition));
+ }
+
+ public List<UpdateLogEntry> get(String segmentNameStr) {
+ return Collections.unmodifiableList(_segments2UpdateLog.get(segmentNameStr));
+ }
+ }
+}
\ No newline at end of file
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/VirtualColumnContext.java b/pinot-core/src/main/java/org/apache/pinot/core/segment/updater/SegmentUpdaterConfig.java
similarity index 55%
copy from pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/VirtualColumnContext.java
copy to pinot-core/src/main/java/org/apache/pinot/core/segment/updater/SegmentUpdaterConfig.java
index bb32320..97d48f3 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/VirtualColumnContext.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/segment/updater/SegmentUpdaterConfig.java
@@ -16,29 +16,10 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.pinot.core.segment.virtualcolumn;
+package org.apache.pinot.core.segment.updater;
-import org.apache.pinot.spi.data.FieldSpec;
-
-
-/**
- * Miscellaneous context information about the virtual column.
- * It will be used to build various components (dictionary, reader, etc) in the virtual column provider.
- */
-public class VirtualColumnContext {
- private FieldSpec _fieldSpec;
- private int _totalDocCount;
-
- public VirtualColumnContext(FieldSpec fieldSpec, int totalDocCount) {
- _fieldSpec = fieldSpec;
- _totalDocCount = totalDocCount;
- }
-
- public FieldSpec getFieldSpec() {
- return _fieldSpec;
- }
-
- public int getTotalDocCount() {
- return _totalDocCount;
- }
+public class SegmentUpdaterConfig {
+ public static final String SEGMENT_UDPATE_SLEEP_MS = "sleep.ms";
+ public static final String INPUT_TOPIC_PREFIX = "input.topic.prefix";
+ public static final int SEGMENT_UDPATE_SLEEP_MS_DEFAULT = 100;
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/segment/updater/UpsertWaterMarkManager.java b/pinot-core/src/main/java/org/apache/pinot/core/segment/updater/UpsertWaterMarkManager.java
new file mode 100644
index 0000000..bbea037
--- /dev/null
+++ b/pinot-core/src/main/java/org/apache/pinot/core/segment/updater/UpsertWaterMarkManager.java
@@ -0,0 +1,90 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.core.segment.updater;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
+import org.apache.pinot.grigio.common.metrics.GrigioGauge;
+import org.apache.pinot.grigio.common.metrics.GrigioMetrics;
+import org.apache.pinot.grigio.common.storageProvider.UpdateLogEntry;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+public class UpsertWaterMarkManager {
+
+ private final Map<String, Map<Integer, Long>> _highWaterMarkTablePartitionMap = new ConcurrentHashMap<>();
+ private final GrigioMetrics _metrics;
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(UpsertWaterMarkManager.class);
+ private static volatile UpsertWaterMarkManager _instance;
+
+ private UpsertWaterMarkManager(GrigioMetrics metrics) {
+ _metrics = metrics;
+ }
+
+ public static void init(GrigioMetrics metrics) {
+ synchronized (UpsertWaterMarkManager.class) {
+ Preconditions.checkState(_instance == null, "upsert water mark manager is already init");
+ _instance = new UpsertWaterMarkManager(metrics);
+ }
+ }
+
+ public static UpsertWaterMarkManager getInstance() {
+ Preconditions.checkState(_instance != null, "upsert water mark manager is not yet init");
+ return _instance;
+ }
+
+ // TODO(tingchen) Look into the case where Segment Update Messages might arrive before the corresponding physical data.
+ public void processMessage(String table, String segment, UpdateLogEntry logEntry) {
+ if (logEntry == null) {
+ return;
+ }
+ long version = logEntry.getValue();
+ int partition = logEntry.getPartition();
+ processVersionUpdate(table, partition, version);
+ }
+
+ public void processVersionUpdate(String table, int partition, long version) {
+ Preconditions.checkState(partition >= 0, "logEntry has invalid version {} for table {}",
+ version, table);
+ Map<Integer, Long> partitionToHighWaterMark = _highWaterMarkTablePartitionMap.computeIfAbsent(table, t -> new ConcurrentHashMap<>());
+ long currentVersion = partitionToHighWaterMark.getOrDefault(partition, -1L); // assumes that valid version is non-negative
+ if (version > currentVersion) {
+ partitionToHighWaterMark.put(partition, version);
+ _metrics.setValueOfTableGauge(String.valueOf(partition), GrigioGauge.SERVER_VERSION_CONSUMED, version);
+ }
+ }
+
+ public Map<Integer, Long> getHighWaterMarkForTable(String tableName) {
+ return ImmutableMap.copyOf(_highWaterMarkTablePartitionMap.getOrDefault(tableName, ImmutableMap.of()));
+ }
+
+ public Map<String, Map<Integer, Long>> getHighWaterMarkTablePartitionMap() {
+ return ImmutableMap.copyOf(_highWaterMarkTablePartitionMap);
+ }
+
+ @Override
+ public String toString() {
+ return Joiner.on(",").withKeyValueSeparator("=").join(_highWaterMarkTablePartitionMap);
+ }
+}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/BaseVirtualColumnProvider.java b/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/BaseVirtualColumnProvider.java
index ece8fa7..c1c7c62 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/BaseVirtualColumnProvider.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/BaseVirtualColumnProvider.java
@@ -21,6 +21,9 @@ package org.apache.pinot.core.segment.virtualcolumn;
import org.apache.pinot.core.segment.index.ColumnMetadata;
import org.apache.pinot.core.segment.index.column.ColumnIndexContainer;
import org.apache.pinot.spi.data.FieldSpec;
+import org.apache.pinot.core.io.reader.DataFileReader;
+import org.apache.pinot.core.segment.index.readers.Dictionary;
+import org.apache.pinot.core.segment.index.readers.InvertedIndexReader;
/**
@@ -39,4 +42,19 @@ public abstract class BaseVirtualColumnProvider implements VirtualColumnProvider
public ColumnIndexContainer buildColumnIndexContainer(VirtualColumnContext context) {
return new VirtualColumnIndexContainer(buildReader(context), buildInvertedIndex(context), buildDictionary(context));
}
+
+ @Override
+ public ColumnIndexContainer buildColumnIndexContainer(VirtualColumnContext context, DataFileReader fileReader,
+ Dictionary dictionary, InvertedIndexReader invertedIndexReader) {
+ if (fileReader == null) {
+ fileReader = buildReader(context);
+ }
+ if (dictionary == null) {
+ dictionary = buildDictionary(context);
+ }
+ if (invertedIndexReader == null) {
+ invertedIndexReader = buildInvertedIndex(context);
+ }
+ return new VirtualColumnIndexContainer(fileReader, invertedIndexReader, dictionary);
+ }
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/VirtualColumnContext.java b/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/VirtualColumnContext.java
index bb32320..abfccf0 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/VirtualColumnContext.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/VirtualColumnContext.java
@@ -28,10 +28,12 @@ import org.apache.pinot.spi.data.FieldSpec;
public class VirtualColumnContext {
private FieldSpec _fieldSpec;
private int _totalDocCount;
+ private boolean _isMutableSegment;
- public VirtualColumnContext(FieldSpec fieldSpec, int totalDocCount) {
+ public VirtualColumnContext(FieldSpec fieldSpec, int totalDocCount, boolean isMutableSegment) {
_fieldSpec = fieldSpec;
_totalDocCount = totalDocCount;
+ _isMutableSegment = isMutableSegment;
}
public FieldSpec getFieldSpec() {
@@ -41,4 +43,8 @@ public class VirtualColumnContext {
public int getTotalDocCount() {
return _totalDocCount;
}
+
+ public boolean isMutableSegment() {
+ return _isMutableSegment;
+ }
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/VirtualColumnProvider.java b/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/VirtualColumnProvider.java
index 8a1e3f5..aed96bd 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/VirtualColumnProvider.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/VirtualColumnProvider.java
@@ -30,6 +30,7 @@ import org.apache.pinot.core.segment.index.readers.InvertedIndexReader;
* comprise a proper column.
*/
public interface VirtualColumnProvider {
+
DataFileReader buildReader(VirtualColumnContext context);
Dictionary buildDictionary(VirtualColumnContext context);
@@ -39,4 +40,7 @@ public interface VirtualColumnProvider {
InvertedIndexReader buildInvertedIndex(VirtualColumnContext context);
ColumnIndexContainer buildColumnIndexContainer(VirtualColumnContext context);
+
+ ColumnIndexContainer buildColumnIndexContainer(VirtualColumnContext context, DataFileReader reader,
+ Dictionary dictionary, InvertedIndexReader invertedIndexReader);
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/mutable/BaseLongVirtualColumnProvider.java b/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/mutable/BaseLongVirtualColumnProvider.java
new file mode 100644
index 0000000..fc69a9f
--- /dev/null
+++ b/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/mutable/BaseLongVirtualColumnProvider.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.core.segment.virtualcolumn.mutable;
+
+import org.apache.pinot.core.segment.index.ColumnMetadata;
+import org.apache.pinot.core.segment.index.readers.Dictionary;
+import org.apache.pinot.core.segment.index.readers.InvertedIndexReader;
+import org.apache.pinot.core.segment.virtualcolumn.BaseVirtualColumnProvider;
+import org.apache.pinot.core.segment.virtualcolumn.VirtualColumnContext;
+import org.apache.pinot.spi.data.FieldSpec;
+
+/**
+ * base class for upsert related virtual column (validFrom and validUntil virtual column)
+ * this only support the direct access through the reader but not the dictionary or inverted index
+ */
+public abstract class BaseLongVirtualColumnProvider extends BaseVirtualColumnProvider {
+
+ @Override
+ public Dictionary buildDictionary(VirtualColumnContext context) {
+ return null;
+ }
+
+ @Override
+ public ColumnMetadata buildMetadata(VirtualColumnContext context) {
+ ColumnMetadata.Builder columnMetadataBuilder = new ColumnMetadata.Builder()
+ .setCardinality(1) // TODO: double check on this
+ .setHasDictionary(false)
+ .setHasInvertedIndex(false)
+ .setFieldType(FieldSpec.FieldType.DIMENSION)
+ .setDataType(FieldSpec.DataType.LONG)
+ .setSingleValue(true)
+ .setIsSorted(false)
+ .setTotalDocs(context.getTotalDocCount());
+ return columnMetadataBuilder.build();
+ }
+
+ @Override
+ public InvertedIndexReader buildInvertedIndex(VirtualColumnContext context) {
+ return null;
+ }
+}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/VirtualColumnContext.java b/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/mutable/BaseVirtualColumnSingleValueReaderWriter.java
similarity index 56%
copy from pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/VirtualColumnContext.java
copy to pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/mutable/BaseVirtualColumnSingleValueReaderWriter.java
index bb32320..fdcfe25 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/VirtualColumnContext.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/mutable/BaseVirtualColumnSingleValueReaderWriter.java
@@ -16,29 +16,14 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.pinot.core.segment.virtualcolumn;
+package org.apache.pinot.core.segment.virtualcolumn.mutable;
-import org.apache.pinot.spi.data.FieldSpec;
+import org.apache.pinot.core.io.reader.BaseSingleColumnSingleValueReader;
+import org.apache.pinot.core.io.reader.ReaderContext;
+public abstract class BaseVirtualColumnSingleValueReaderWriter<E extends ReaderContext> extends BaseSingleColumnSingleValueReader<E> {
-/**
- * Miscellaneous context information about the virtual column.
- * It will be used to build various components (dictionary, reader, etc) in the virtual column provider.
- */
-public class VirtualColumnContext {
- private FieldSpec _fieldSpec;
- private int _totalDocCount;
-
- public VirtualColumnContext(FieldSpec fieldSpec, int totalDocCount) {
- _fieldSpec = fieldSpec;
- _totalDocCount = totalDocCount;
- }
-
- public FieldSpec getFieldSpec() {
- return _fieldSpec;
- }
-
- public int getTotalDocCount() {
- return _totalDocCount;
+ public void addNewRecord(int docId) {
+ // default do nothing
}
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/VirtualColumnProvider.java b/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/mutable/ValidFromInMemoryVirtualColumnProvider.java
similarity index 50%
copy from pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/VirtualColumnProvider.java
copy to pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/mutable/ValidFromInMemoryVirtualColumnProvider.java
index 8a1e3f5..ebb45e7 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/VirtualColumnProvider.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/mutable/ValidFromInMemoryVirtualColumnProvider.java
@@ -16,27 +16,30 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.pinot.core.segment.virtualcolumn;
+package org.apache.pinot.core.segment.virtualcolumn.mutable;
import org.apache.pinot.core.io.reader.DataFileReader;
-import org.apache.pinot.core.segment.index.ColumnMetadata;
-import org.apache.pinot.core.segment.index.column.ColumnIndexContainer;
-import org.apache.pinot.core.segment.index.readers.Dictionary;
-import org.apache.pinot.core.segment.index.readers.InvertedIndexReader;
+import org.apache.pinot.core.segment.virtualcolumn.VirtualColumnContext;
+import org.apache.pinot.grigio.common.messages.LogEventType;
+public class ValidFromInMemoryVirtualColumnProvider extends BaseLongVirtualColumnProvider {
-/**
- * Virtual column provider interface, which is used to instantiate the various components (dictionary, reader, etc) that
- * comprise a proper column.
- */
-public interface VirtualColumnProvider {
- DataFileReader buildReader(VirtualColumnContext context);
-
- Dictionary buildDictionary(VirtualColumnContext context);
-
- ColumnMetadata buildMetadata(VirtualColumnContext context);
-
- InvertedIndexReader buildInvertedIndex(VirtualColumnContext context);
+ @Override
+ public DataFileReader buildReader(VirtualColumnContext context) {
+ return new VirtualColumnLongValueReaderWriter(context) {
+ @Override
+ public boolean update(int docId, int value, LogEventType eventType) {
+ if (eventType == LogEventType.INSERT) {
+ updateValue(docId, value);
+ return true;
+ }
+ return false;
+ }
- ColumnIndexContainer buildColumnIndexContainer(VirtualColumnContext context);
+ @Override
+ public boolean update(int docId, long value, LogEventType eventType) {
+ return update(docId, Math.toIntExact(value), eventType);
+ }
+ };
+ }
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/VirtualColumnProvider.java b/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/mutable/ValidUntilInMemoryVirtualColumnProvider.java
similarity index 50%
copy from pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/VirtualColumnProvider.java
copy to pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/mutable/ValidUntilInMemoryVirtualColumnProvider.java
index 8a1e3f5..9aef94c 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/VirtualColumnProvider.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/mutable/ValidUntilInMemoryVirtualColumnProvider.java
@@ -16,27 +16,30 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.pinot.core.segment.virtualcolumn;
+package org.apache.pinot.core.segment.virtualcolumn.mutable;
import org.apache.pinot.core.io.reader.DataFileReader;
-import org.apache.pinot.core.segment.index.ColumnMetadata;
-import org.apache.pinot.core.segment.index.column.ColumnIndexContainer;
-import org.apache.pinot.core.segment.index.readers.Dictionary;
-import org.apache.pinot.core.segment.index.readers.InvertedIndexReader;
+import org.apache.pinot.core.segment.virtualcolumn.VirtualColumnContext;
+import org.apache.pinot.grigio.common.messages.LogEventType;
+public class ValidUntilInMemoryVirtualColumnProvider extends BaseLongVirtualColumnProvider {
-/**
- * Virtual column provider interface, which is used to instantiate the various components (dictionary, reader, etc) that
- * comprise a proper column.
- */
-public interface VirtualColumnProvider {
- DataFileReader buildReader(VirtualColumnContext context);
-
- Dictionary buildDictionary(VirtualColumnContext context);
-
- ColumnMetadata buildMetadata(VirtualColumnContext context);
-
- InvertedIndexReader buildInvertedIndex(VirtualColumnContext context);
+ @Override
+ public DataFileReader buildReader(VirtualColumnContext context) {
+ return new VirtualColumnLongValueReaderWriter(context) {
+ @Override
+ public boolean update(int docId, int offset, LogEventType eventType) {
+ if (eventType == LogEventType.DELETE) {
+ this.updateValue(docId, offset);
+ return true;
+ }
+ return false;
+ }
- ColumnIndexContainer buildColumnIndexContainer(VirtualColumnContext context);
+ @Override
+ public boolean update(int docId, long offset, LogEventType eventType) {
+ return update(docId, Math.toIntExact(offset), eventType);
+ }
+ };
+ }
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/mutable/VirtualColumnLongValueReaderWriter.java b/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/mutable/VirtualColumnLongValueReaderWriter.java
new file mode 100644
index 0000000..78f3bda
--- /dev/null
+++ b/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/mutable/VirtualColumnLongValueReaderWriter.java
@@ -0,0 +1,108 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.core.segment.virtualcolumn.mutable;
+
+import org.apache.pinot.core.io.reader.impl.ChunkReaderContext;
+import org.apache.pinot.core.segment.virtualcolumn.VirtualColumnContext;
+import org.apache.pinot.grigio.common.messages.LogEventType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+public abstract class VirtualColumnLongValueReaderWriter extends BaseVirtualColumnSingleValueReaderWriter<ChunkReaderContext> {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(VirtualColumnLongValueReaderWriter.class);
+
+ private int _totalDocSize;
+ private int _currentMaxDocId;
+ private final int [] _values;
+ private final int DEFAULT_NEW_VALUE = -1;
+
+ public VirtualColumnLongValueReaderWriter(VirtualColumnContext context) {
+ this(context.getTotalDocCount(), context.isMutableSegment());
+ }
+
+ public VirtualColumnLongValueReaderWriter(int totalDocSize, boolean isMutableSegment) {
+ _totalDocSize = totalDocSize;
+ _values = new int[_totalDocSize];
+ if (!isMutableSegment) {
+ Arrays.fill(_values, -1);
+ }
+ }
+
+ @Override
+ public synchronized void addNewRecord(int docId) {
+ if (docId >= _totalDocSize) {
+ throw new RuntimeException(String.format("failed to add docId %s larger than allocated size %s", docId, _totalDocSize));
+ }
+ _currentMaxDocId = docId;
+ _values[docId] = DEFAULT_NEW_VALUE;
+ }
+
+ @Override
+ public ChunkReaderContext createContext() {
+ return null;
+ }
+
+ @Override
+ public long getLong(int row) {
+ return _values[row];
+ }
+
+ @Override
+ public long getLong(int rowId, ChunkReaderContext context) {
+ return _values[rowId];
+ }
+
+ @Override
+ public void readValues(int[] rows, int rowStartPos, int rowSize, int[] values, int valuesStartPos) {
+ System.arraycopy(rows, rowStartPos, values, valuesStartPos, rowSize);
+ }
+
+ @Override
+ public void close() throws IOException {
+ }
+
+ /**
+ * update the internal data to value at a given location
+ * assume the update will be idempotent and we won't check if the existing data have been set
+ *
+ */
+ protected void updateValue(int docId, int value) {
+ if (docId >= _totalDocSize) {
+ throw new RuntimeException(String.format("new record docId %s is larger than capacity %s", docId, _totalDocSize));
+ }
+ if (_values[docId] == 0) {
+ throw new RuntimeException(String.format("failed to update virtual column: with value %s:%s we are trying to " +
+ "update a value that has not been ingested yet, max doc id %s", docId, value, _currentMaxDocId));
+ }
+ _values[docId] = value;
+ }
+
+ // ensure backward compatibility
+ protected void updateValue(int docId, long value) {
+ updateValue(docId, Math.toIntExact(value));
+ }
+
+ public abstract boolean update(int docId, long value, LogEventType eventType);
+
+ public abstract boolean update(int docId, int value, LogEventType eventType);
+}
diff --git a/pinot-core/src/test/java/org/apache/pinot/core/data/manager/realtime/LLRealtimeSegmentDataManagerTest.java b/pinot-core/src/test/java/org/apache/pinot/core/data/manager/realtime/LLRealtimeSegmentDataManagerTest.java
index 715c833..c4b557b 100644
--- a/pinot-core/src/test/java/org/apache/pinot/core/data/manager/realtime/LLRealtimeSegmentDataManagerTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/core/data/manager/realtime/LLRealtimeSegmentDataManagerTest.java
@@ -696,7 +696,7 @@ public class LLRealtimeSegmentDataManagerTest {
Assert.assertEquals(secondSegmentDataManager.get().getPartitionConsumerSemaphore().availablePermits(), 1);
}
- public static class FakeLLRealtimeSegmentDataManager extends LLRealtimeSegmentDataManager {
+ public static class FakeLLRealtimeSegmentDataManager extends AppendLLRealtimeSegmentDataManager {
public Field _state;
public Field _shouldStop;
diff --git a/pinot-core/src/test/java/org/apache/pinot/core/indexsegment/immutable/ImmutableUpsertSegmentImplTest.java b/pinot-core/src/test/java/org/apache/pinot/core/indexsegment/immutable/ImmutableUpsertSegmentImplTest.java
new file mode 100644
index 0000000..817131c
--- /dev/null
+++ b/pinot-core/src/test/java/org/apache/pinot/core/indexsegment/immutable/ImmutableUpsertSegmentImplTest.java
@@ -0,0 +1,127 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.core.indexsegment.immutable;
+
+import org.apache.pinot.core.segment.updater.UpsertWaterMarkManager;
+import org.apache.pinot.core.segment.virtualcolumn.mutable.VirtualColumnLongValueReaderWriter;
+import org.apache.pinot.grigio.common.messages.LogEventType;
+import org.apache.pinot.grigio.common.storageProvider.UpdateLogEntry;
+import org.apache.pinot.grigio.common.storageProvider.UpdateLogEntrySet;
+import org.apache.pinot.grigio.common.storageProvider.UpdateLogStorageProvider;
+import org.testng.Assert;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+
+public class ImmutableUpsertSegmentImplTest {
+
+ UpdateLogStorageProvider _mockProvider;
+ UpsertWaterMarkManager _mockUpsertWaterMarkManager;
+ List<VirtualColumnLongValueReaderWriter> _readerWriters = new ArrayList<>();
+
+ @BeforeMethod
+ public void init() {
+ _mockProvider = mock(UpdateLogStorageProvider.class);
+ _mockUpsertWaterMarkManager = mock(UpsertWaterMarkManager.class);
+ }
+
+ @Test
+ public void testInitVirtualColumn() throws IOException {
+ long start = System.currentTimeMillis();
+ long minOffset = 5000_000l;
+ int totalDocs = 5_000_000;
+ _readerWriters.add(new VirtualColumnLongValueReaderWriter(totalDocs, false) {
+ @Override
+ public boolean update(int docId, long value, LogEventType eventType) {
+ if (eventType == LogEventType.INSERT) {
+ updateValue(docId, value);
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public boolean update(int docId, int value, LogEventType eventType) {
+ return update(docId, value, eventType);
+ }
+ });
+ _readerWriters.add(new VirtualColumnLongValueReaderWriter(totalDocs, false) {
+ @Override
+ public boolean update(int docId, long value, LogEventType eventType) {
+ if (eventType == LogEventType.DELETE) {
+ updateValue(docId, value);
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public boolean update(int docId, int value, LogEventType eventType) {
+ return update(docId, value, eventType);
+ }
+ });
+ int[] offsetToDocId = new int[totalDocs];
+ for (int i = 0; i < totalDocs; i++) {
+ offsetToDocId[i] = i;
+ }
+ List<UpdateLogEntry> updateLogEntries = new ArrayList<>(totalDocs * 2);
+ for (int i = 0; i < totalDocs; i++) {
+ updateLogEntries.add(new UpdateLogEntry(minOffset + i, 50, LogEventType.INSERT, i%8));
+ updateLogEntries.add(new UpdateLogEntry(minOffset + i, 100, LogEventType.DELETE, i%8));
+ }
+ UpdateLogEntrySet entrySet = new UpdateLogEntrySet(null, 2) {
+ @Override
+ public Iterator<UpdateLogEntry> iterator() {
+ return updateLogEntries.iterator();
+ }
+ };
+ when(_mockProvider.getAllMessages(anyString(), anyString())).thenReturn(entrySet);
+ System.out.println("run time for set up: " + (System.currentTimeMillis() - start));
+
+ start = System.currentTimeMillis();
+
+ ImmutableUpsertSegmentImpl immutableUpsertSegment = new ImmutableUpsertSegmentImpl(_readerWriters, totalDocs,
+ _mockUpsertWaterMarkManager, _mockProvider, minOffset, offsetToDocId);
+
+ immutableUpsertSegment.initVirtualColumn();
+ long runtime = System.currentTimeMillis() - start;
+ System.out.println("run time is " + runtime);
+ // on regular developer laptop this should take less 1 second, but on integration server this might be longer
+ Assert.assertTrue(runtime < 10_000L, "run time should be less than 10 second");
+
+ VirtualColumnLongValueReaderWriter insertReaderWrite = _readerWriters.get(0);
+ VirtualColumnLongValueReaderWriter deleteReaderWrite = _readerWriters.get(1);
+ for (int i = 0; i < totalDocs; i++) {
+ if (insertReaderWrite.getLong(i) != 50 || deleteReaderWrite.getLong(i) != 100) {
+ System.out.println(String.format("position %d has value %d/%d", i, insertReaderWrite.getLong(i),
+ deleteReaderWrite.getLong(i)));
+ Assert.fail("no correct value");
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/pinot-core/src/test/java/org/apache/pinot/core/indexsegment/mutable/MutableSegmentImplTestUtils.java b/pinot-core/src/test/java/org/apache/pinot/core/indexsegment/mutable/MutableSegmentImplTestUtils.java
index e4df760..26e7eff 100644
--- a/pinot-core/src/test/java/org/apache/pinot/core/indexsegment/mutable/MutableSegmentImplTestUtils.java
+++ b/pinot-core/src/test/java/org/apache/pinot/core/indexsegment/mutable/MutableSegmentImplTestUtils.java
@@ -59,6 +59,6 @@ public class MutableSegmentImplTestUtils {
.setRealtimeSegmentZKMetadata(new RealtimeSegmentZKMetadata())
.setMemoryManager(new DirectMemoryManager(SEGMENT_NAME)).setStatsHistory(statsHistory)
.setAggregateMetrics(aggregateMetrics).setNullHandlingEnabled(nullHandlingEnabled).build();
- return new MutableSegmentImpl(realtimeSegmentConfig);
+ return new MutableAppendSegmentImpl(realtimeSegmentConfig);
}
}
diff --git a/pinot-core/src/test/java/org/apache/pinot/queries/InterSegmentAggregationSingleValueQueriesTest.java b/pinot-core/src/test/java/org/apache/pinot/queries/InterSegmentAggregationSingleValueQueriesTest.java
index cca0563..f06ad7d 100644
--- a/pinot-core/src/test/java/org/apache/pinot/queries/InterSegmentAggregationSingleValueQueriesTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/queries/InterSegmentAggregationSingleValueQueriesTest.java
@@ -18,8 +18,6 @@
*/
package org.apache.pinot.queries;
-import java.io.Serializable;
-import java.util.function.Function;
import org.apache.pinot.common.response.broker.BrokerResponseNative;
import org.apache.pinot.common.response.broker.SelectionResults;
import org.apache.pinot.spi.utils.BytesUtils;
@@ -29,6 +27,9 @@ import org.apache.pinot.pql.parsers.Pql2Compiler;
import org.testng.Assert;
import org.testng.annotations.Test;
+import java.io.Serializable;
+import java.util.function.Function;
+
import static org.testng.Assert.assertFalse;
import static org.testng.Assert.assertTrue;
diff --git a/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/HelixInstanceDataManager.java b/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/HelixInstanceDataManager.java
index 3df1b8f..1af7e29 100644
--- a/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/HelixInstanceDataManager.java
+++ b/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/HelixInstanceDataManager.java
@@ -20,10 +20,7 @@ package org.apache.pinot.server.starter.helix;
import com.google.common.base.Preconditions;
import java.io.File;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Set;
+import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.locks.Lock;
import javax.annotation.Nullable;
@@ -48,6 +45,7 @@ import org.apache.pinot.core.indexsegment.immutable.ImmutableSegment;
import org.apache.pinot.core.indexsegment.immutable.ImmutableSegmentLoader;
import org.apache.pinot.core.segment.index.loader.IndexLoadingConfig;
import org.apache.pinot.core.segment.index.loader.LoaderUtils;
+import org.apache.pinot.core.segment.updater.UpsertWaterMarkManager;
import org.apache.pinot.spi.data.Schema;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -304,6 +302,11 @@ public class HelixInstanceDataManager implements InstanceDataManager {
}
@Override
+ public Map<String, Map<Integer, Long>> getLowWaterMarks() {
+ return UpsertWaterMarkManager.getInstance().getHighWaterMarkTablePartitionMap();
+ }
+
+ @Override
public String getSegmentDataDirectory() {
return _instanceDataManagerConfig.getInstanceDataDir();
}
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/realtime/provisioning/MemoryEstimator.java b/pinot-tools/src/main/java/org/apache/pinot/tools/realtime/provisioning/MemoryEstimator.java
index 865591b..187b36f 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/realtime/provisioning/MemoryEstimator.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/realtime/provisioning/MemoryEstimator.java
@@ -26,6 +26,7 @@ import java.util.stream.Collectors;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.io.FileUtils;
import org.apache.pinot.common.config.TableConfig;
+import org.apache.pinot.core.indexsegment.mutable.MutableAppendSegmentImpl;
import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.common.metadata.segment.RealtimeSegmentZKMetadata;
import org.apache.pinot.spi.utils.DataSize;
@@ -135,7 +136,7 @@ public class MemoryEstimator {
.setStatsHistory(sampleStatsHistory);
// create mutable segment impl
- MutableSegmentImpl mutableSegmentImpl = new MutableSegmentImpl(realtimeSegmentConfigBuilder.build());
+ MutableSegmentImpl mutableSegmentImpl = new MutableAppendSegmentImpl(realtimeSegmentConfigBuilder.build());
// read all rows and index them
try (PinotSegmentRecordReader segmentRecordReader = new PinotSegmentRecordReader(_sampleCompletedSegment);) {
@@ -235,7 +236,7 @@ public class MemoryEstimator {
.setOffHeap(true).setMemoryManager(memoryManager).setStatsHistory(statsHistory);
// create mutable segment impl
- MutableSegmentImpl mutableSegmentImpl = new MutableSegmentImpl(realtimeSegmentConfigBuilder.build());
+ MutableSegmentImpl mutableSegmentImpl = new MutableAppendSegmentImpl(realtimeSegmentConfigBuilder.build());
long memoryForConsumingSegmentPerPartition = memoryManager.getTotalAllocatedBytes();
mutableSegmentImpl.destroy();
FileUtils.deleteQuietly(statsFileCopy);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org