You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by sn...@apache.org on 2019/06/18 04:53:07 UTC
[incubator-pinot] branch master updated: Add deprecated annotation
for segment zk metadata (#4320)
This is an automated email from the ASF dual-hosted git repository.
snlee pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 47e91a2 Add deprecated annotation for segment zk metadata (#4320)
47e91a2 is described below
commit 47e91a2551305ec751c29487d156d46404dab06b
Author: Seunghyun Lee <sn...@linkedin.com>
AuthorDate: Mon Jun 17 21:53:01 2019 -0700
Add deprecated annotation for segment zk metadata (#4320)
* Add deprecated annotation for segment zk metadata
This is a follow up work for #4288 and #4289
1. Added deprecated annotation for segment zk metadata.
2. Remove all getTableName & setTableName usages from testing code
- There is no usage of getTableName() for segment zk metadata except
the case where we fill segment metadata from segment zk metadata
for realtime segment creation.
- For segment metadata, the only usage is during the segment upload
- We still write segment name to both segment zk metadata and segment
metadata for rollback support.
* Addressing comments
---
...ixExternalViewBasedTimeBoundaryServiceTest.java | 5 +++--
.../pinot/common/metadata/ZKMetadataProvider.java | 25 +++++++++++-----------
.../common/metadata/segment/SegmentZKMetadata.java | 15 ++++++++++---
.../apache/pinot/common/utils/CommonConstants.java | 4 +++-
.../common/metadata/SegmentZKMetadataTest.java | 6 ------
.../pinot/controller/api/upload/ZKOperator.java | 7 +++---
.../helix/core/PinotHelixResourceManager.java | 14 ++++++------
.../core/realtime/PinotRealtimeSegmentManager.java | 2 +-
.../controller/helix/PinotResourceManagerTest.java | 9 ++++----
.../helix/core/PinotHelixResourceManagerTest.java | 8 ++-----
.../segment/FlushThresholdUpdaterTest.java | 1 -
.../helix/core/retention/RetentionManagerTest.java | 1 -
.../realtime/HLRealtimeSegmentDataManager.java | 2 +-
.../manager/realtime/RealtimeTableDataManager.java | 5 +++--
.../core/segment/index/SegmentMetadataImpl.java | 1 +
.../realtime/LLRealtimeSegmentDataManagerTest.java | 1 -
.../ControllerPeriodicTasksIntegrationTests.java | 8 +++----
.../converter/ColumnarToStarTreeConverter.java | 5 ++++-
18 files changed, 62 insertions(+), 57 deletions(-)
diff --git a/pinot-broker/src/test/java/org/apache/pinot/broker/routing/HelixExternalViewBasedTimeBoundaryServiceTest.java b/pinot-broker/src/test/java/org/apache/pinot/broker/routing/HelixExternalViewBasedTimeBoundaryServiceTest.java
index 20ccc9c..74add38 100644
--- a/pinot-broker/src/test/java/org/apache/pinot/broker/routing/HelixExternalViewBasedTimeBoundaryServiceTest.java
+++ b/pinot-broker/src/test/java/org/apache/pinot/broker/routing/HelixExternalViewBasedTimeBoundaryServiceTest.java
@@ -142,11 +142,12 @@ public class HelixExternalViewBasedTimeBoundaryServiceTest {
private void addSegmentZKMetadata(String rawTableName, int endTimeInDays, TimeUnit timeUnit) {
for (int i = 1; i <= endTimeInDays; i++) {
OfflineSegmentZKMetadata offlineSegmentZKMetadata = new OfflineSegmentZKMetadata();
- offlineSegmentZKMetadata.setTableName(rawTableName);
offlineSegmentZKMetadata.setSegmentName(rawTableName + i);
offlineSegmentZKMetadata.setEndTime(i * timeUnit.convert(1L, TimeUnit.DAYS));
offlineSegmentZKMetadata.setTimeUnit(timeUnit);
- ZKMetadataProvider.setOfflineSegmentZKMetadata(_propertyStore, offlineSegmentZKMetadata);
+ ZKMetadataProvider
+ .setOfflineSegmentZKMetadata(_propertyStore, TableNameBuilder.OFFLINE.tableNameWithType(rawTableName),
+ offlineSegmentZKMetadata);
}
}
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/metadata/ZKMetadataProvider.java b/pinot-common/src/main/java/org/apache/pinot/common/metadata/ZKMetadataProvider.java
index 7cf7448..2a3ad60 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/metadata/ZKMetadataProvider.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/metadata/ZKMetadataProvider.java
@@ -145,30 +145,29 @@ public class ZKMetadataProvider {
}
public static boolean setOfflineSegmentZKMetadata(ZkHelixPropertyStore<ZNRecord> propertyStore,
- OfflineSegmentZKMetadata offlineSegmentZKMetadata, int expectedVersion) {
+ String offlineTableName, OfflineSegmentZKMetadata offlineSegmentZKMetadata, int expectedVersion) {
// NOTE: Helix will throw ZkBadVersionException if version does not match
try {
- return propertyStore.set(constructPropertyStorePathForSegment(
- TableNameBuilder.OFFLINE.tableNameWithType(offlineSegmentZKMetadata.getTableName()),
- offlineSegmentZKMetadata.getSegmentName()), offlineSegmentZKMetadata.toZNRecord(), expectedVersion,
- AccessOption.PERSISTENT);
+ return propertyStore
+ .set(constructPropertyStorePathForSegment(offlineTableName, offlineSegmentZKMetadata.getSegmentName()),
+ offlineSegmentZKMetadata.toZNRecord(), expectedVersion, AccessOption.PERSISTENT);
} catch (ZkBadVersionException e) {
return false;
}
}
public static boolean setOfflineSegmentZKMetadata(ZkHelixPropertyStore<ZNRecord> propertyStore,
- OfflineSegmentZKMetadata offlineSegmentZKMetadata) {
- return propertyStore.set(constructPropertyStorePathForSegment(
- TableNameBuilder.OFFLINE.tableNameWithType(offlineSegmentZKMetadata.getTableName()),
- offlineSegmentZKMetadata.getSegmentName()), offlineSegmentZKMetadata.toZNRecord(), AccessOption.PERSISTENT);
+ String offlineTableName, OfflineSegmentZKMetadata offlineSegmentZKMetadata) {
+ return propertyStore
+ .set(constructPropertyStorePathForSegment(offlineTableName, offlineSegmentZKMetadata.getSegmentName()),
+ offlineSegmentZKMetadata.toZNRecord(), AccessOption.PERSISTENT);
}
public static boolean setRealtimeSegmentZKMetadata(ZkHelixPropertyStore<ZNRecord> propertyStore,
- RealtimeSegmentZKMetadata realtimeSegmentZKMetadata) {
- return propertyStore.set(constructPropertyStorePathForSegment(
- TableNameBuilder.REALTIME.tableNameWithType(realtimeSegmentZKMetadata.getTableName()),
- realtimeSegmentZKMetadata.getSegmentName()), realtimeSegmentZKMetadata.toZNRecord(), AccessOption.PERSISTENT);
+ String realtimeTableName, RealtimeSegmentZKMetadata realtimeSegmentZKMetadata) {
+ return propertyStore
+ .set(constructPropertyStorePathForSegment(realtimeTableName, realtimeSegmentZKMetadata.getSegmentName()),
+ realtimeSegmentZKMetadata.toZNRecord(), AccessOption.PERSISTENT);
}
@Nullable
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/metadata/segment/SegmentZKMetadata.java b/pinot-common/src/main/java/org/apache/pinot/common/metadata/segment/SegmentZKMetadata.java
index 67366a5..ec13e5a 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/metadata/segment/SegmentZKMetadata.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/metadata/segment/SegmentZKMetadata.java
@@ -46,7 +46,6 @@ public abstract class SegmentZKMetadata implements ZKMetadata {
protected static final String NULL = "null";
private String _segmentName;
- private String _tableName;
private SegmentType _segmentType;
private long _startTime = -1;
private long _endTime = -1;
@@ -62,6 +61,9 @@ public abstract class SegmentZKMetadata implements ZKMetadata {
private Map<String, String> _customMap;
private String _crypterName;
+ @Deprecated
+ private String _tableName;
+
public SegmentZKMetadata() {
}
@@ -103,10 +105,12 @@ public abstract class SegmentZKMetadata implements ZKMetadata {
_segmentName = segmentName;
}
+ @Deprecated
public String getTableName() {
return _tableName;
}
+ @Deprecated
public void setTableName(String tableName) {
_tableName = tableName;
}
@@ -266,7 +270,10 @@ public abstract class SegmentZKMetadata implements ZKMetadata {
public ZNRecord toZNRecord() {
ZNRecord znRecord = new ZNRecord(_segmentName);
znRecord.setSimpleField(CommonConstants.Segment.SEGMENT_NAME, _segmentName);
- znRecord.setSimpleField(CommonConstants.Segment.TABLE_NAME, _tableName);
+
+ if (_tableName != null) {
+ znRecord.setSimpleField(CommonConstants.Segment.TABLE_NAME, _tableName);
+ }
if (_crypterName != null) {
znRecord.setSimpleField(CommonConstants.Segment.CRYPTER_NAME, _crypterName);
@@ -308,7 +315,9 @@ public abstract class SegmentZKMetadata implements ZKMetadata {
public Map<String, String> toMap() {
Map<String, String> configMap = new HashMap<>();
configMap.put(CommonConstants.Segment.SEGMENT_NAME, _segmentName);
- configMap.put(CommonConstants.Segment.TABLE_NAME, _tableName);
+ if (_tableName != null) {
+ configMap.put(CommonConstants.Segment.TABLE_NAME, _tableName);
+ }
configMap.put(CommonConstants.Segment.SEGMENT_TYPE, _segmentType.toString());
if (_timeUnit == null) {
configMap.put(CommonConstants.Segment.TIME_UNIT, null);
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/utils/CommonConstants.java b/pinot-common/src/main/java/org/apache/pinot/common/utils/CommonConstants.java
index 8132cdb..ff68450 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/utils/CommonConstants.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/utils/CommonConstants.java
@@ -330,7 +330,6 @@ public class CommonConstants {
}
public static final String SEGMENT_NAME = "segment.name";
- public static final String TABLE_NAME = "segment.table.name";
public static final String SEGMENT_TYPE = "segment.type";
public static final String CRYPTER_NAME = "segment.crypter";
public static final String INDEX_VERSION = "segment.index.version";
@@ -360,6 +359,9 @@ public class CommonConstants {
public enum SegmentType {
OFFLINE, REALTIME
}
+
+ @Deprecated
+ public static final String TABLE_NAME = "segment.table.name";
}
public static class SegmentOperations {
diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metadata/SegmentZKMetadataTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metadata/SegmentZKMetadataTest.java
index 551f940..db9ba32 100644
--- a/pinot-common/src/test/java/org/apache/pinot/common/metadata/SegmentZKMetadataTest.java
+++ b/pinot-common/src/test/java/org/apache/pinot/common/metadata/SegmentZKMetadataTest.java
@@ -119,7 +119,6 @@ public class SegmentZKMetadataTest {
String segmentName = "testTable_R_1000_2000_groupId0_part0";
ZNRecord record = new ZNRecord(segmentName);
record.setSimpleField(CommonConstants.Segment.SEGMENT_NAME, segmentName);
- record.setSimpleField(CommonConstants.Segment.TABLE_NAME, "testTable");
record.setSimpleField(CommonConstants.Segment.INDEX_VERSION, "v1");
record.setEnumField(CommonConstants.Segment.SEGMENT_TYPE, CommonConstants.Segment.SegmentType.REALTIME);
record.setEnumField(CommonConstants.Segment.Realtime.STATUS, CommonConstants.Segment.Realtime.Status.DONE);
@@ -137,7 +136,6 @@ public class SegmentZKMetadataTest {
private RealtimeSegmentZKMetadata getTestDoneRealtimeSegmentMetadata() {
RealtimeSegmentZKMetadata realtimeSegmentMetadata = new RealtimeSegmentZKMetadata();
realtimeSegmentMetadata.setSegmentName("testTable_R_1000_2000_groupId0_part0");
- realtimeSegmentMetadata.setTableName("testTable");
realtimeSegmentMetadata.setSegmentType(SegmentType.REALTIME);
realtimeSegmentMetadata.setIndexVersion("v1");
realtimeSegmentMetadata.setStartTime(1000);
@@ -156,7 +154,6 @@ public class SegmentZKMetadataTest {
String segmentName = "testTable_R_1000_groupId0_part0";
ZNRecord record = new ZNRecord(segmentName);
record.setSimpleField(CommonConstants.Segment.SEGMENT_NAME, segmentName);
- record.setSimpleField(CommonConstants.Segment.TABLE_NAME, "testTable");
record.setSimpleField(CommonConstants.Segment.INDEX_VERSION, "v1");
record.setEnumField(CommonConstants.Segment.SEGMENT_TYPE, CommonConstants.Segment.SegmentType.REALTIME);
record.setEnumField(CommonConstants.Segment.Realtime.STATUS, CommonConstants.Segment.Realtime.Status.IN_PROGRESS);
@@ -174,7 +171,6 @@ public class SegmentZKMetadataTest {
private RealtimeSegmentZKMetadata getTestInProgressRealtimeSegmentMetadata() {
RealtimeSegmentZKMetadata realtimeSegmentMetadata = new RealtimeSegmentZKMetadata();
realtimeSegmentMetadata.setSegmentName("testTable_R_1000_groupId0_part0");
- realtimeSegmentMetadata.setTableName("testTable");
realtimeSegmentMetadata.setSegmentType(SegmentType.REALTIME);
realtimeSegmentMetadata.setIndexVersion("v1");
realtimeSegmentMetadata.setStartTime(1000);
@@ -193,7 +189,6 @@ public class SegmentZKMetadataTest {
String segmentName = "testTable_O_3000_4000";
ZNRecord record = new ZNRecord(segmentName);
record.setSimpleField(CommonConstants.Segment.SEGMENT_NAME, segmentName);
- record.setSimpleField(CommonConstants.Segment.TABLE_NAME, "testTable");
record.setSimpleField(CommonConstants.Segment.CRYPTER_NAME, "testCrypter");
record.setSimpleField(CommonConstants.Segment.INDEX_VERSION, "v1");
record.setEnumField(CommonConstants.Segment.SEGMENT_TYPE, CommonConstants.Segment.SegmentType.OFFLINE);
@@ -212,7 +207,6 @@ public class SegmentZKMetadataTest {
private OfflineSegmentZKMetadata getTestOfflineSegmentMetadata() {
OfflineSegmentZKMetadata offlineSegmentMetadata = new OfflineSegmentZKMetadata();
offlineSegmentMetadata.setSegmentName("testTable_O_3000_4000");
- offlineSegmentMetadata.setTableName("testTable");
offlineSegmentMetadata.setCrypterName("testCrypter");
offlineSegmentMetadata.setSegmentType(SegmentType.OFFLINE);
offlineSegmentMetadata.setIndexVersion("v1");
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/api/upload/ZKOperator.java b/pinot-controller/src/main/java/org/apache/pinot/controller/api/upload/ZKOperator.java
index 52d2caa..ab1a6ef 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/api/upload/ZKOperator.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/api/upload/ZKOperator.java
@@ -120,7 +120,8 @@ public class ZKOperator {
// Lock the segment by setting the upload start time in ZK
existingSegmentZKMetadata.setSegmentUploadStartTime(System.currentTimeMillis());
- if (!_pinotHelixResourceManager.updateZkMetadata(existingSegmentZKMetadata, znRecord.getVersion())) {
+ if (!_pinotHelixResourceManager
+ .updateZkMetadata(offlineTableName, existingSegmentZKMetadata, znRecord.getVersion())) {
throw new ControllerApplicationException(LOGGER,
"Failed to lock the segment: " + segmentName + " of table: " + offlineTableName + ", retry later",
Response.Status.CONFLICT);
@@ -158,7 +159,7 @@ public class ZKOperator {
// (creation time is not included in the crc)
existingSegmentZKMetadata.setCreationTime(segmentMetadata.getIndexCreationTime());
existingSegmentZKMetadata.setRefreshTime(System.currentTimeMillis());
- if (!_pinotHelixResourceManager.updateZkMetadata(existingSegmentZKMetadata)) {
+ if (!_pinotHelixResourceManager.updateZkMetadata(offlineTableName, existingSegmentZKMetadata)) {
throw new RuntimeException(
"Failed to update ZK metadata for segment: " + segmentName + " of table: " + offlineTableName);
}
@@ -179,7 +180,7 @@ public class ZKOperator {
_pinotHelixResourceManager.refreshSegment(offlineTableName, segmentMetadata, existingSegmentZKMetadata);
}
} catch (Exception e) {
- if (!_pinotHelixResourceManager.updateZkMetadata(existingSegmentZKMetadata)) {
+ if (!_pinotHelixResourceManager.updateZkMetadata(offlineTableName, existingSegmentZKMetadata)) {
LOGGER.error("Failed to update ZK metadata for segment: {} of table: {}", segmentName, offlineTableName);
}
throw e;
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java
index a4d4d0b..5eeeddc 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java
@@ -1595,7 +1595,7 @@ public class PinotHelixResourceManager {
offlineSegmentZKMetadata.setDownloadUrl(downloadUrl);
offlineSegmentZKMetadata.setCrypterName(crypter);
offlineSegmentZKMetadata.setPushTime(System.currentTimeMillis());
- if (!ZKMetadataProvider.setOfflineSegmentZKMetadata(_propertyStore, offlineSegmentZKMetadata)) {
+ if (!ZKMetadataProvider.setOfflineSegmentZKMetadata(_propertyStore, offlineTableName, offlineSegmentZKMetadata)) {
throw new RuntimeException(
"Failed to set segment ZK metadata for table: " + offlineTableName + ", segment: " + segmentName);
}
@@ -1610,12 +1610,14 @@ public class PinotHelixResourceManager {
ZKMetadataProvider.constructPropertyStorePathForSegment(tableNameWithType, segmentName));
}
- public boolean updateZkMetadata(@Nonnull OfflineSegmentZKMetadata segmentMetadata, int expectedVersion) {
- return ZKMetadataProvider.setOfflineSegmentZKMetadata(_propertyStore, segmentMetadata, expectedVersion);
+ public boolean updateZkMetadata(@Nonnull String offlineTableName, @Nonnull OfflineSegmentZKMetadata segmentMetadata,
+ int expectedVersion) {
+ return ZKMetadataProvider
+ .setOfflineSegmentZKMetadata(_propertyStore, offlineTableName, segmentMetadata, expectedVersion);
}
- public boolean updateZkMetadata(@Nonnull OfflineSegmentZKMetadata segmentMetadata) {
- return ZKMetadataProvider.setOfflineSegmentZKMetadata(_propertyStore, segmentMetadata);
+ public boolean updateZkMetadata(@Nonnull String offlineTableName, @Nonnull OfflineSegmentZKMetadata segmentMetadata) {
+ return ZKMetadataProvider.setOfflineSegmentZKMetadata(_propertyStore, offlineTableName, segmentMetadata);
}
public void refreshSegment(@Nonnull String offlineTableName, @Nonnull SegmentMetadata segmentMetadata,
@@ -1627,7 +1629,7 @@ public class PinotHelixResourceManager {
// segment or load from local
offlineSegmentZKMetadata = ZKMetadataUtils.updateSegmentMetadata(offlineSegmentZKMetadata, segmentMetadata);
offlineSegmentZKMetadata.setRefreshTime(System.currentTimeMillis());
- if (!ZKMetadataProvider.setOfflineSegmentZKMetadata(_propertyStore, offlineSegmentZKMetadata)) {
+ if (!ZKMetadataProvider.setOfflineSegmentZKMetadata(_propertyStore, offlineTableName, offlineSegmentZKMetadata)) {
throw new RuntimeException(
"Failed to update ZK metadata for segment: " + segmentName + " of table: " + offlineTableName);
}
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/PinotRealtimeSegmentManager.java b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/PinotRealtimeSegmentManager.java
index 331a4a0..32642dd 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/PinotRealtimeSegmentManager.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/PinotRealtimeSegmentManager.java
@@ -251,7 +251,7 @@ public class PinotRealtimeSegmentManager implements HelixPropertyListener, IZkCh
realtimeSegmentMetadataToAdd.setSegmentName(segmentId);
// Add the new metadata to the property store
- ZKMetadataProvider.setRealtimeSegmentZKMetadata(_pinotHelixResourceManager.getPropertyStore(),
+ ZKMetadataProvider.setRealtimeSegmentZKMetadata(_pinotHelixResourceManager.getPropertyStore(), resourceName,
realtimeSegmentMetadataToAdd);
// Update the ideal state to add the new realtime segment
diff --git a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/PinotResourceManagerTest.java b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/PinotResourceManagerTest.java
index eb57c7f..d65e818 100644
--- a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/PinotResourceManagerTest.java
+++ b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/PinotResourceManagerTest.java
@@ -69,22 +69,21 @@ public class PinotResourceManagerTest extends ControllerTest {
@Test
public void testUpdateSegmentZKMetadata() {
OfflineSegmentZKMetadata segmentZKMetadata = new OfflineSegmentZKMetadata();
- segmentZKMetadata.setTableName("testTable");
segmentZKMetadata.setSegmentName("testSegment");
// Segment ZK metadata does not exist
- Assert.assertFalse(_helixResourceManager.updateZkMetadata(segmentZKMetadata, 0));
+ Assert.assertFalse(_helixResourceManager.updateZkMetadata("testTable_OFFLINE", segmentZKMetadata, 0));
// Set segment ZK metadata
- Assert.assertTrue(_helixResourceManager.updateZkMetadata(segmentZKMetadata));
+ Assert.assertTrue(_helixResourceManager.updateZkMetadata("testTable_OFFLINE", segmentZKMetadata));
// Update ZK metadata
Assert.assertEquals(
_helixResourceManager.getSegmentMetadataZnRecord("testTable_OFFLINE", "testSegment").getVersion(), 0);
- Assert.assertTrue(_helixResourceManager.updateZkMetadata(segmentZKMetadata, 0));
+ Assert.assertTrue(_helixResourceManager.updateZkMetadata("testTable_OFFLINE", segmentZKMetadata, 0));
Assert.assertEquals(
_helixResourceManager.getSegmentMetadataZnRecord("testTable_OFFLINE", "testSegment").getVersion(), 1);
- Assert.assertFalse(_helixResourceManager.updateZkMetadata(segmentZKMetadata, 0));
+ Assert.assertFalse(_helixResourceManager.updateZkMetadata("testTable_OFFLINE", segmentZKMetadata, 0));
}
/**
diff --git a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManagerTest.java b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManagerTest.java
index 7d9f8d9..a850ac4 100644
--- a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManagerTest.java
+++ b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManagerTest.java
@@ -244,29 +244,25 @@ public class PinotHelixResourceManagerTest extends ControllerTest {
// Test retrieving OFFLINE segment ZK metadata
{
OfflineSegmentZKMetadata offlineSegmentZKMetadata = new OfflineSegmentZKMetadata();
- offlineSegmentZKMetadata.setTableName(OFFLINE_TABLE_NAME);
offlineSegmentZKMetadata.setSegmentName(segmentName);
- ZKMetadataProvider.setOfflineSegmentZKMetadata(_propertyStore, offlineSegmentZKMetadata);
+ ZKMetadataProvider.setOfflineSegmentZKMetadata(_propertyStore, OFFLINE_TABLE_NAME, offlineSegmentZKMetadata);
List<OfflineSegmentZKMetadata> retrievedMetadataList =
_helixResourceManager.getOfflineSegmentMetadata(OFFLINE_TABLE_NAME);
Assert.assertEquals(retrievedMetadataList.size(), 1);
OfflineSegmentZKMetadata retrievedMetadata = retrievedMetadataList.get(0);
- Assert.assertEquals(retrievedMetadata.getTableName(), OFFLINE_TABLE_NAME);
Assert.assertEquals(retrievedMetadata.getSegmentName(), segmentName);
}
// Test retrieving REALTIME segment ZK metadata
{
RealtimeSegmentZKMetadata realtimeMetadata = new RealtimeSegmentZKMetadata();
- realtimeMetadata.setTableName(REALTIME_TABLE_NAME);
realtimeMetadata.setSegmentName(segmentName);
realtimeMetadata.setStatus(CommonConstants.Segment.Realtime.Status.DONE);
- ZKMetadataProvider.setRealtimeSegmentZKMetadata(_propertyStore, realtimeMetadata);
+ ZKMetadataProvider.setRealtimeSegmentZKMetadata(_propertyStore, REALTIME_TABLE_NAME, realtimeMetadata);
List<RealtimeSegmentZKMetadata> retrievedMetadataList =
_helixResourceManager.getRealtimeSegmentMetadata(REALTIME_TABLE_NAME);
Assert.assertEquals(retrievedMetadataList.size(), 1);
RealtimeSegmentZKMetadata retrievedMetadata = retrievedMetadataList.get(0);
- Assert.assertEquals(retrievedMetadata.getTableName(), REALTIME_TABLE_NAME);
Assert.assertEquals(retrievedMetadata.getSegmentName(), segmentName);
Assert.assertEquals(realtimeMetadata.getStatus(), CommonConstants.Segment.Realtime.Status.DONE);
}
diff --git a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/realtime/segment/FlushThresholdUpdaterTest.java b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/realtime/segment/FlushThresholdUpdaterTest.java
index aefaf3e..425ebff 100644
--- a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/realtime/segment/FlushThresholdUpdaterTest.java
+++ b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/realtime/segment/FlushThresholdUpdaterTest.java
@@ -244,7 +244,6 @@ public class FlushThresholdUpdaterTest {
newSegMetadata.setStartOffset(startOffset);
newSegMetadata.setEndOffset(Long.MAX_VALUE);
newSegMetadata.setNumReplicas(3);
- newSegMetadata.setTableName(realtimeTableName);
newSegMetadata.setSegmentName(newSegmentName.getSegmentName());
newSegMetadata.setStatus(CommonConstants.Segment.Realtime.Status.IN_PROGRESS);
return newSegMetadata;
diff --git a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/retention/RetentionManagerTest.java b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/retention/RetentionManagerTest.java
index d4adfe1..bb20ca3 100644
--- a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/retention/RetentionManagerTest.java
+++ b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/retention/RetentionManagerTest.java
@@ -299,7 +299,6 @@ public class RetentionManagerTest {
segmentMetadata.setEndOffset(-1L);
segmentMetadata.setNumReplicas(replicaCount);
- segmentMetadata.setTableName(TEST_TABLE_NAME);
return segmentMetadata;
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/HLRealtimeSegmentDataManager.java b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/HLRealtimeSegmentDataManager.java
index 9b5c542..b8b7f17 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/HLRealtimeSegmentDataManager.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/HLRealtimeSegmentDataManager.java
@@ -355,7 +355,7 @@ public class HLRealtimeSegmentDataManager extends RealtimeSegmentDataManager {
metadataToOverwrite.setEndTime(segEndTime);
metadataToOverwrite.setTimeUnit(timeUnit);
metadataToOverwrite.setTotalRawDocs(realtimeSegment.getNumDocsIndexed());
- notifier.notifySegmentCommitted(metadataToOverwrite, segment);
+ notifier.notifySegmentCommitted(tableNameWithType, metadataToOverwrite, segment);
segmentLogger
.info("Completed write of segment completion to Helix, waiting for controller to assign a new segment");
} catch (Exception e) {
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/RealtimeTableDataManager.java b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/RealtimeTableDataManager.java
index cde73a4..8c8d4a4 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/RealtimeTableDataManager.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/RealtimeTableDataManager.java
@@ -180,8 +180,9 @@ public class RealtimeTableDataManager extends BaseTableDataManager {
return consumerDir.getAbsolutePath();
}
- public void notifySegmentCommitted(RealtimeSegmentZKMetadata metadata, ImmutableSegment segment) {
- ZKMetadataProvider.setRealtimeSegmentZKMetadata(_propertyStore, metadata);
+ public void notifySegmentCommitted(String tableNameWithType, RealtimeSegmentZKMetadata metadata,
+ ImmutableSegment segment) {
+ ZKMetadataProvider.setRealtimeSegmentZKMetadata(_propertyStore, tableNameWithType, metadata);
addSegment(segment);
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/segment/index/SegmentMetadataImpl.java b/pinot-core/src/main/java/org/apache/pinot/core/segment/index/SegmentMetadataImpl.java
index bcdca10..352a851 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/segment/index/SegmentMetadataImpl.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/segment/index/SegmentMetadataImpl.java
@@ -240,6 +240,7 @@ public class SegmentMetadataImpl implements SegmentMetadata {
//set the table name
_tableName = segmentMetadataPropertiesConfiguration.getString(TABLE_NAME);
+
// Set segment name.
_segmentName = segmentMetadataPropertiesConfiguration.getString(SEGMENT_NAME);
diff --git a/pinot-core/src/test/java/org/apache/pinot/core/data/manager/realtime/LLRealtimeSegmentDataManagerTest.java b/pinot-core/src/test/java/org/apache/pinot/core/data/manager/realtime/LLRealtimeSegmentDataManagerTest.java
index 3d7bfce..fd5505d 100644
--- a/pinot-core/src/test/java/org/apache/pinot/core/data/manager/realtime/LLRealtimeSegmentDataManagerTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/core/data/manager/realtime/LLRealtimeSegmentDataManagerTest.java
@@ -124,7 +124,6 @@ public class LLRealtimeSegmentDataManagerTest {
private LLCRealtimeSegmentZKMetadata createZkMetadata() {
LLCRealtimeSegmentZKMetadata segmentZKMetadata = new LLCRealtimeSegmentZKMetadata();
- segmentZKMetadata.setTableName(_tableName);
segmentZKMetadata.setSegmentName(_segmentNameStr);
segmentZKMetadata.setStartOffset(_startOffset);
return segmentZKMetadata;
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/ControllerPeriodicTasksIntegrationTests.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/ControllerPeriodicTasksIntegrationTests.java
index ccd76b4..8c9ffe5 100644
--- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/ControllerPeriodicTasksIntegrationTests.java
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/ControllerPeriodicTasksIntegrationTests.java
@@ -168,9 +168,9 @@ public class ControllerPeriodicTasksIntegrationTests extends BaseClusterIntegrat
/**
* Setup offline table, with segments from avro
*/
- private void setupOfflineTableAndSegments(String table, List<File> avroFiles) throws Exception {
+ private void setupOfflineTableAndSegments(String tableName, List<File> avroFiles) throws Exception {
TestUtils.ensureDirectoriesExistAndEmpty(_segmentDir, _tarDir);
- setTableName(table);
+ setTableName(tableName);
_realtimeTableConfig = null;
File schemaFile = getSchemaFile();
@@ -184,11 +184,11 @@ public class ControllerPeriodicTasksIntegrationTests extends BaseClusterIntegrat
Assert.assertNotNull(outgoingTimeUnit);
String timeType = outgoingTimeUnit.toString();
- addOfflineTable(table, timeColumnName, timeType, TENANT_NAME, TENANT_NAME, null, SegmentVersion.v1, null, null, null);
+ addOfflineTable(tableName, timeColumnName, timeType, TENANT_NAME, TENANT_NAME, null, SegmentVersion.v1, null, null, null);
completeTableConfiguration();
ExecutorService executor = Executors.newCachedThreadPool();
- ClusterIntegrationTestUtils.buildSegmentsFromAvro(avroFiles, 0, _segmentDir, _tarDir, table, false,
+ ClusterIntegrationTestUtils.buildSegmentsFromAvro(avroFiles, 0, _segmentDir, _tarDir, tableName, false,
null, null, null, executor);
executor.shutdown();
executor.awaitTermination(10, TimeUnit.MINUTES);
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/segment/converter/ColumnarToStarTreeConverter.java b/pinot-tools/src/main/java/org/apache/pinot/tools/segment/converter/ColumnarToStarTreeConverter.java
index 34094b9..e1f039b 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/segment/converter/ColumnarToStarTreeConverter.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/segment/converter/ColumnarToStarTreeConverter.java
@@ -45,6 +45,9 @@ public class ColumnarToStarTreeConverter {
@Option(name = "-outputDir", required = true, usage = "Path to output directory")
private String _outputDirName = null;
+ @Option(name = "-tableName", required = true, usage = "Table name without type")
+ private String _rawTableName = null;
+
@Option(name = "-starTreeConfigFile", required = false, usage = "Path to Star Tree configuration file")
private String _starTreeConfigFileName = null;
@@ -115,7 +118,7 @@ public class ColumnarToStarTreeConverter {
config.setFormat(FileFormat.PINOT);
config.setOutDir(_outputDirName);
config.setOverwrite(_overwrite);
- config.setTableName(segmentMetadata.getTableName());
+ config.setTableName(_rawTableName);
config.setSegmentName(segmentMetadata.getName());
StarTreeIndexSpec starTreeIndexSpec = null;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org