You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ma...@apache.org on 2023/06/02 22:08:03 UTC
[pinot] branch master updated: Enhancing Table Size API to return table size per replica (#10812)
This is an automated email from the ASF dual-hosted git repository.
mayanks pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 87379fc641 Enhancing Table Size API to return table size per replica (#10812)
87379fc641 is described below
commit 87379fc6411a928b18d5ee8f6ec4a07f5cd0f8d3
Author: Eaugene Thomas <te...@uber.com>
AuthorDate: Sat Jun 3 03:37:57 2023 +0530
Enhancing Table Size API to return table size per replica (#10812)
* 1. Enhancing Table Size API to return size per replica
2. Code Clean up & removing ununsed param "detailed" in REST API
* Fix Check Style
* Fix if condition case
* code clean up
---
.../pinot/controller/api/resources/TableSize.java | 13 +-
.../pinot/controller/util/TableSizeReader.java | 76 +++++++---
.../pinot/controller/api/TableSizeReaderTest.java | 158 +++++++++++----------
3 files changed, 141 insertions(+), 106 deletions(-)
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/TableSize.java b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/TableSize.java
index 4e28b7af14..a9071c8965 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/TableSize.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/TableSize.java
@@ -29,12 +29,10 @@ import io.swagger.annotations.SecurityDefinition;
import io.swagger.annotations.SwaggerDefinition;
import java.util.concurrent.Executor;
import javax.inject.Inject;
-import javax.ws.rs.DefaultValue;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
-import javax.ws.rs.QueryParam;
import javax.ws.rs.core.HttpHeaders;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
@@ -72,18 +70,15 @@ public class TableSize {
@GET
@Path("/tables/{tableName}/size")
@Produces(MediaType.APPLICATION_JSON)
- @ApiOperation(value = "Read table sizes",
- notes = "Get table size details. Table size is the size of untarred segments including replication")
+ @ApiOperation(value = "Read table sizes", notes = "Get table size details. Table size is the size of untarred "
+ + "segments including replication")
@ApiResponses(value = {
- @ApiResponse(code = 200, message = "Success"),
- @ApiResponse(code = 404, message = "Table not found"),
+ @ApiResponse(code = 200, message = "Success"), @ApiResponse(code = 404, message = "Table not found"),
@ApiResponse(code = 500, message = "Internal server error")
})
public TableSizeReader.TableSizeDetails getTableSize(
@ApiParam(value = "Table name without type", required = true, example = "myTable | myTable_OFFLINE")
- @PathParam("tableName") String tableName,
- @ApiParam(value = "Get detailed information", required = false) @DefaultValue("true") @QueryParam("detailed")
- boolean detailed) {
+ @PathParam("tableName") String tableName) {
TableSizeReader tableSizeReader =
new TableSizeReader(_executor, _connectionManager, _controllerMetrics, _pinotHelixResourceManager);
TableSizeReader.TableSizeDetails tableSizeDetails = null;
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/util/TableSizeReader.java b/pinot-controller/src/main/java/org/apache/pinot/controller/util/TableSizeReader.java
index 5d48ad54ea..d03160536d 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/util/TableSizeReader.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/util/TableSizeReader.java
@@ -85,21 +85,30 @@ public class TableSizeReader {
TableConfig realtimeTableConfig =
ZKMetadataProvider.getRealtimeTableConfig(_helixResourceManager.getPropertyStore(), tableName);
- if (offlineTableConfig == null && realtimeTableConfig == null) {
+ boolean hasRealtimeTableConfig = (realtimeTableConfig != null);
+ boolean hasOfflineTableConfig = (offlineTableConfig != null);
+ boolean isMissingAllRealtimeSegments = false;
+ boolean isMissingAllOfflineSegments = false;
+
+ if (!hasRealtimeTableConfig && !hasOfflineTableConfig) {
return null;
}
TableSizeDetails tableSizeDetails = new TableSizeDetails(tableName);
- if (realtimeTableConfig != null) {
+ if (hasRealtimeTableConfig) {
String realtimeTableName = TableNameBuilder.REALTIME.tableNameWithType(tableName);
tableSizeDetails._realtimeSegments = getTableSubtypeSize(realtimeTableName, timeoutMsec);
- tableSizeDetails._reportedSizeInBytes += tableSizeDetails._realtimeSegments._reportedSizeInBytes;
- tableSizeDetails._estimatedSizeInBytes += tableSizeDetails._realtimeSegments._estimatedSizeInBytes;
-
+ // taking max(0,value) as values as set to -1 if all the segments are in error
+ tableSizeDetails._reportedSizeInBytes += Math.max(tableSizeDetails._realtimeSegments._reportedSizeInBytes, 0L);
+ tableSizeDetails._estimatedSizeInBytes += Math.max(tableSizeDetails._realtimeSegments._estimatedSizeInBytes, 0L);
+ tableSizeDetails._reportedSizePerReplicaInBytes +=
+ Math.max(tableSizeDetails._realtimeSegments._reportedSizePerReplicaInBytes, 0L);
+ isMissingAllRealtimeSegments =
+ (tableSizeDetails._realtimeSegments._missingSegments == tableSizeDetails._realtimeSegments._segments.size());
_controllerMetrics.setValueOfTableGauge(realtimeTableName, ControllerGauge.TABLE_TOTAL_SIZE_ON_SERVER,
tableSizeDetails._realtimeSegments._estimatedSizeInBytes);
_controllerMetrics.setValueOfTableGauge(realtimeTableName, ControllerGauge.TABLE_SIZE_PER_REPLICA_ON_SERVER,
- tableSizeDetails._realtimeSegments._estimatedSizeInBytes / _helixResourceManager.getNumReplicas(
- realtimeTableConfig));
+ tableSizeDetails._realtimeSegments._estimatedSizeInBytes / _helixResourceManager
+ .getNumReplicas(realtimeTableConfig));
long largestSegmentSizeOnServer = DEFAULT_SIZE_WHEN_MISSING_OR_ERROR;
for (SegmentSizeDetails segmentSizeDetail : tableSizeDetails._realtimeSegments._segments.values()) {
@@ -108,22 +117,25 @@ public class TableSizeReader {
}
}
if (largestSegmentSizeOnServer != DEFAULT_SIZE_WHEN_MISSING_OR_ERROR) {
- _controllerMetrics.setValueOfTableGauge(realtimeTableName,
- ControllerGauge.LARGEST_SEGMENT_SIZE_ON_SERVER,
+ _controllerMetrics.setValueOfTableGauge(realtimeTableName, ControllerGauge.LARGEST_SEGMENT_SIZE_ON_SERVER,
largestSegmentSizeOnServer);
}
}
- if (offlineTableConfig != null) {
+ if (hasOfflineTableConfig) {
String offlineTableName = TableNameBuilder.OFFLINE.tableNameWithType(tableName);
tableSizeDetails._offlineSegments = getTableSubtypeSize(offlineTableName, timeoutMsec);
- tableSizeDetails._reportedSizeInBytes += tableSizeDetails._offlineSegments._reportedSizeInBytes;
- tableSizeDetails._estimatedSizeInBytes += tableSizeDetails._offlineSegments._estimatedSizeInBytes;
-
+ // taking max(0,value) as values as set to -1 if all the segments are in error
+ tableSizeDetails._reportedSizeInBytes += Math.max(tableSizeDetails._offlineSegments._reportedSizeInBytes, 0L);
+ tableSizeDetails._estimatedSizeInBytes += Math.max(tableSizeDetails._offlineSegments._estimatedSizeInBytes, 0L);
+ tableSizeDetails._reportedSizePerReplicaInBytes +=
+ Math.max(tableSizeDetails._offlineSegments._reportedSizePerReplicaInBytes, 0L);
+ isMissingAllOfflineSegments =
+ (tableSizeDetails._offlineSegments._missingSegments == tableSizeDetails._offlineSegments._segments.size());
_controllerMetrics.setValueOfTableGauge(offlineTableName, ControllerGauge.TABLE_TOTAL_SIZE_ON_SERVER,
tableSizeDetails._offlineSegments._estimatedSizeInBytes);
_controllerMetrics.setValueOfTableGauge(offlineTableName, ControllerGauge.TABLE_SIZE_PER_REPLICA_ON_SERVER,
- tableSizeDetails._offlineSegments._estimatedSizeInBytes / _helixResourceManager.getNumReplicas(
- offlineTableConfig));
+ tableSizeDetails._offlineSegments._estimatedSizeInBytes / _helixResourceManager
+ .getNumReplicas(offlineTableConfig));
long largestSegmentSizeOnServer = DEFAULT_SIZE_WHEN_MISSING_OR_ERROR;
for (SegmentSizeDetails segmentSizeDetail : tableSizeDetails._offlineSegments._segments.values()) {
@@ -132,12 +144,19 @@ public class TableSizeReader {
}
}
if (largestSegmentSizeOnServer != DEFAULT_SIZE_WHEN_MISSING_OR_ERROR) {
- _controllerMetrics.setValueOfTableGauge(offlineTableName,
- ControllerGauge.LARGEST_SEGMENT_SIZE_ON_SERVER,
+ _controllerMetrics.setValueOfTableGauge(offlineTableName, ControllerGauge.LARGEST_SEGMENT_SIZE_ON_SERVER,
largestSegmentSizeOnServer);
}
}
+ // Set the top level sizes to DEFAULT_SIZE_WHEN_MISSING_OR_ERROR when all segments are error
+ if ((hasRealtimeTableConfig && hasOfflineTableConfig && isMissingAllRealtimeSegments && isMissingAllOfflineSegments)
+ || (hasOfflineTableConfig && !hasRealtimeTableConfig && isMissingAllOfflineSegments) || (hasRealtimeTableConfig
+ && !hasOfflineTableConfig && isMissingAllRealtimeSegments)) {
+ tableSizeDetails._reportedSizeInBytes = DEFAULT_SIZE_WHEN_MISSING_OR_ERROR;
+ tableSizeDetails._estimatedSizeInBytes = DEFAULT_SIZE_WHEN_MISSING_OR_ERROR;
+ tableSizeDetails._reportedSizePerReplicaInBytes = DEFAULT_SIZE_WHEN_MISSING_OR_ERROR;
+ }
return tableSizeDetails;
}
@@ -158,6 +177,10 @@ public class TableSizeReader {
@JsonProperty("estimatedSizeInBytes")
public long _estimatedSizeInBytes = 0;
+ // reported size per replica
+ @JsonProperty("reportedSizePerReplicaInBytes")
+ public long _reportedSizePerReplicaInBytes = 0;
+
@JsonProperty("offlineSegments")
public TableSubTypeSizeDetails _offlineSegments;
@@ -186,6 +209,10 @@ public class TableSizeReader {
@JsonProperty("missingSegments")
public int _missingSegments = 0;
+ // reported size per replica
+ @JsonProperty("reportedSizePerReplicaInBytes")
+ public long _reportedSizePerReplicaInBytes = 0;
+
@JsonProperty("segments")
public Map<String, SegmentSizeDetails> _segments = new HashMap<>();
}
@@ -198,6 +225,10 @@ public class TableSizeReader {
@JsonProperty("estimatedSizeInBytes")
public long _estimatedSizeInBytes = 0;
+ // Max Reported size per replica
+ @JsonProperty("maxReportedSizePerReplicaInBytes")
+ public long _maxReportedSizePerReplicaInBytes = 0;
+
@JsonProperty("serverInfo")
public Map<String, SegmentSizeInfo> _serverInfo = new HashMap<>();
}
@@ -248,12 +279,13 @@ public class TableSizeReader {
String segment = entry.getKey();
SegmentSizeDetails sizeDetails = entry.getValue();
// Iterate over all segment size info, update reported size, track max segment size and number of errored servers
- long segmentLevelMax = -1L;
+ sizeDetails._maxReportedSizePerReplicaInBytes = DEFAULT_SIZE_WHEN_MISSING_OR_ERROR;
int errors = 0;
for (SegmentSizeInfo sizeInfo : sizeDetails._serverInfo.values()) {
if (sizeInfo.getDiskSizeInBytes() != DEFAULT_SIZE_WHEN_MISSING_OR_ERROR) {
sizeDetails._reportedSizeInBytes += sizeInfo.getDiskSizeInBytes();
- segmentLevelMax = Math.max(segmentLevelMax, sizeInfo.getDiskSizeInBytes());
+ sizeDetails._maxReportedSizePerReplicaInBytes =
+ Math.max(sizeDetails._maxReportedSizePerReplicaInBytes, sizeInfo.getDiskSizeInBytes());
} else {
errors++;
}
@@ -261,12 +293,15 @@ public class TableSizeReader {
// Update estimated size, track segments that are missing from all servers
if (errors != sizeDetails._serverInfo.size()) {
// Use max segment size from other servers to estimate the segment size not reported
- sizeDetails._estimatedSizeInBytes = sizeDetails._reportedSizeInBytes + errors * segmentLevelMax;
+ sizeDetails._estimatedSizeInBytes =
+ sizeDetails._reportedSizeInBytes + (errors * sizeDetails._maxReportedSizePerReplicaInBytes);
subTypeSizeDetails._reportedSizeInBytes += sizeDetails._reportedSizeInBytes;
subTypeSizeDetails._estimatedSizeInBytes += sizeDetails._estimatedSizeInBytes;
+ subTypeSizeDetails._reportedSizePerReplicaInBytes += sizeDetails._maxReportedSizePerReplicaInBytes;
} else {
// Segment is missing from all servers
missingSegments.add(segment);
+ sizeDetails._maxReportedSizePerReplicaInBytes = DEFAULT_SIZE_WHEN_MISSING_OR_ERROR;
sizeDetails._reportedSizeInBytes = DEFAULT_SIZE_WHEN_MISSING_OR_ERROR;
sizeDetails._estimatedSizeInBytes = DEFAULT_SIZE_WHEN_MISSING_OR_ERROR;
subTypeSizeDetails._missingSegments++;
@@ -284,6 +319,7 @@ public class TableSizeReader {
LOGGER.warn("Failed to get size report for all {} segments of table: {}", numSegments, tableNameWithType);
subTypeSizeDetails._reportedSizeInBytes = DEFAULT_SIZE_WHEN_MISSING_OR_ERROR;
subTypeSizeDetails._estimatedSizeInBytes = DEFAULT_SIZE_WHEN_MISSING_OR_ERROR;
+ subTypeSizeDetails._reportedSizePerReplicaInBytes = DEFAULT_SIZE_WHEN_MISSING_OR_ERROR;
} else {
LOGGER.warn("Missing size report for {} out of {} segments for table {}", subTypeSizeDetails._missingSegments,
numSegments, tableNameWithType);
diff --git a/pinot-controller/src/test/java/org/apache/pinot/controller/api/TableSizeReaderTest.java b/pinot-controller/src/test/java/org/apache/pinot/controller/api/TableSizeReaderTest.java
index 7b460161db..dc82319ed7 100644
--- a/pinot-controller/src/test/java/org/apache/pinot/controller/api/TableSizeReaderTest.java
+++ b/pinot-controller/src/test/java/org/apache/pinot/controller/api/TableSizeReaderTest.java
@@ -81,25 +81,25 @@ public class TableSizeReaderTest {
private PinotHelixResourceManager _helix;
@BeforeClass
- public void setUp()
- throws IOException {
+ public void setUp() throws IOException {
_helix = mock(PinotHelixResourceManager.class);
TableConfig tableConfig =
new TableConfigBuilder(TableType.OFFLINE).setTableName("myTable").setNumReplicas(NUM_REPLICAS).build();
ZkHelixPropertyStore mockPropertyStore = mock(ZkHelixPropertyStore.class);
- when(mockPropertyStore.get(ArgumentMatchers.anyString(), ArgumentMatchers.eq(null),
- ArgumentMatchers.eq(AccessOption.PERSISTENT))).thenAnswer((Answer) invocationOnMock -> {
- String path = (String) invocationOnMock.getArguments()[0];
- if (path.contains("realtime_REALTIME")) {
- return TableConfigUtils.toZNRecord(tableConfig);
- }
- if (path.contains("offline_OFFLINE")) {
- return TableConfigUtils.toZNRecord(tableConfig);
- }
- return null;
- });
+ when(mockPropertyStore
+ .get(ArgumentMatchers.anyString(), ArgumentMatchers.eq(null), ArgumentMatchers.eq(AccessOption.PERSISTENT)))
+ .thenAnswer((Answer) invocationOnMock -> {
+ String path = (String) invocationOnMock.getArguments()[0];
+ if (path.contains("realtime_REALTIME")) {
+ return TableConfigUtils.toZNRecord(tableConfig);
+ }
+ if (path.contains("offline_OFFLINE")) {
+ return TableConfigUtils.toZNRecord(tableConfig);
+ }
+ return null;
+ });
when(_helix.getPropertyStore()).thenReturn(mockPropertyStore);
when(_helix.getNumReplicas(ArgumentMatchers.eq(tableConfig))).thenReturn(NUM_REPLICAS);
@@ -152,8 +152,7 @@ public class TableSizeReaderTest {
private HttpHandler createHandler(final int status, final List<SegmentSizeInfo> segmentSizes, final int sleepTimeMs) {
return new HttpHandler() {
@Override
- public void handle(HttpExchange httpExchange)
- throws IOException {
+ public void handle(HttpExchange httpExchange) throws IOException {
if (sleepTimeMs > 0) {
try {
Thread.sleep(sleepTimeMs);
@@ -219,8 +218,7 @@ public class TableSizeReaderTest {
}
@Test
- public void testNoSuchTable()
- throws InvalidConfigException {
+ public void testNoSuchTable() throws InvalidConfigException {
TableSizeReader reader = new TableSizeReader(_executor, _connectionManager, _controllerMetrics, _helix);
assertNull(reader.getTableSizeDetails("mytable", 5000));
}
@@ -229,16 +227,14 @@ public class TableSizeReaderTest {
throws InvalidConfigException {
when(_helix.getServerToSegmentsMap(anyString())).thenAnswer(new Answer<Object>() {
@Override
- public Object answer(InvocationOnMock invocationOnMock)
- throws Throwable {
+ public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
return subsetOfServerSegments(servers);
}
});
when(_helix.getDataInstanceAdminEndpoints(ArgumentMatchers.anySet())).thenAnswer(new Answer<Object>() {
@Override
- public Object answer(InvocationOnMock invocationOnMock)
- throws Throwable {
+ public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
return serverEndpoints(servers);
}
});
@@ -252,11 +248,7 @@ public class TableSizeReaderTest {
for (String server : servers) {
List<String> segments = _serverMap.get(server)._segments;
for (String segment : segments) {
- List<String> segServers = segmentServers.get(segment);
- if (segServers == null) {
- segServers = new ArrayList<String>();
- segmentServers.put(segment, segServers);
- }
+ List<String> segServers = segmentServers.computeIfAbsent(segment, k -> new ArrayList<String>());
segServers.add(server);
}
}
@@ -267,7 +259,7 @@ public class TableSizeReaderTest {
Map<String, List<String>> segmentServers = segmentToServers(servers);
long reportedSize = 0;
long estimatedSize = 0;
- long maxSegmentSize = 0;
+ long reportedSizePerReplica = 0L;
boolean hasErrors = false;
for (Map.Entry<String, List<String>> segmentEntry : segmentServers.entrySet()) {
final String segmentName = segmentEntry.getKey();
@@ -278,6 +270,9 @@ public class TableSizeReaderTest {
if (segmentDetails._estimatedSizeInBytes != TableSizeReader.DEFAULT_SIZE_WHEN_MISSING_OR_ERROR) {
estimatedSize += segmentDetails._estimatedSizeInBytes;
}
+ if (segmentDetails._maxReportedSizePerReplicaInBytes != TableSizeReader.DEFAULT_SIZE_WHEN_MISSING_OR_ERROR) {
+ reportedSizePerReplica += segmentDetails._maxReportedSizePerReplicaInBytes;
+ }
assertNotNull(segmentDetails);
final List<String> expectedServers = segmentEntry.getValue();
@@ -293,13 +288,17 @@ public class TableSizeReaderTest {
if (numResponses != 0) {
assertEquals(segmentDetails._reportedSizeInBytes, numResponses * expectedSegmentSize);
assertEquals(segmentDetails._estimatedSizeInBytes, expectedServers.size() * expectedSegmentSize);
+ assertEquals(segmentDetails._maxReportedSizePerReplicaInBytes, expectedSegmentSize);
} else {
assertEquals(segmentDetails._reportedSizeInBytes, TableSizeReader.DEFAULT_SIZE_WHEN_MISSING_OR_ERROR);
assertEquals(segmentDetails._estimatedSizeInBytes, TableSizeReader.DEFAULT_SIZE_WHEN_MISSING_OR_ERROR);
+ assertEquals(segmentDetails._maxReportedSizePerReplicaInBytes,
+ TableSizeReader.DEFAULT_SIZE_WHEN_MISSING_OR_ERROR);
}
}
assertEquals(tableSize._reportedSizeInBytes, reportedSize);
assertEquals(tableSize._estimatedSizeInBytes, estimatedSize);
+ assertEquals(tableSize._reportedSizePerReplicaInBytes, reportedSizePerReplica);
if (hasErrors) {
assertTrue(tableSize._reportedSizeInBytes != tableSize._estimatedSizeInBytes);
assertTrue(tableSize._missingSegments > 0);
@@ -307,8 +306,7 @@ public class TableSizeReaderTest {
}
@Test
- public void testGetTableSubTypeSizeAllSuccess()
- throws InvalidConfigException {
+ public void testGetTableSubTypeSizeAllSuccess() throws InvalidConfigException {
final String[] servers = {"server0", "server1"};
String table = "offline";
TableSizeReader.TableSizeDetails tableSizeDetails = testRunner(servers, table);
@@ -320,23 +318,24 @@ public class TableSizeReaderTest {
assertNull(tableSizeDetails._realtimeSegments);
assertEquals(tableSizeDetails._reportedSizeInBytes, offlineSizes._reportedSizeInBytes);
assertEquals(tableSizeDetails._estimatedSizeInBytes, offlineSizes._estimatedSizeInBytes);
+ assertEquals(tableSizeDetails._reportedSizePerReplicaInBytes, offlineSizes._reportedSizePerReplicaInBytes);
String tableNameWithType = TableNameBuilder.OFFLINE.tableNameWithType(table);
assertEquals(MetricValueUtils.getTableGaugeValue(_controllerMetrics, tableNameWithType,
ControllerGauge.TABLE_STORAGE_EST_MISSING_SEGMENT_PERCENT), 0);
- assertEquals(
- MetricValueUtils.getTableGaugeValue(_controllerMetrics, tableNameWithType,
- ControllerGauge.TABLE_SIZE_PER_REPLICA_ON_SERVER), offlineSizes._estimatedSizeInBytes / NUM_REPLICAS);
- assertEquals(
- MetricValueUtils.getTableGaugeValue(_controllerMetrics, tableNameWithType,
- ControllerGauge.TABLE_TOTAL_SIZE_ON_SERVER), offlineSizes._estimatedSizeInBytes);
- assertEquals(
- MetricValueUtils.getTableGaugeValue(_controllerMetrics, tableNameWithType,
- ControllerGauge.LARGEST_SEGMENT_SIZE_ON_SERVER), 160);
+ assertEquals(MetricValueUtils
+ .getTableGaugeValue(_controllerMetrics, tableNameWithType,
+ ControllerGauge.TABLE_SIZE_PER_REPLICA_ON_SERVER),
+ offlineSizes._estimatedSizeInBytes / NUM_REPLICAS);
+ assertEquals(MetricValueUtils
+ .getTableGaugeValue(_controllerMetrics, tableNameWithType, ControllerGauge.TABLE_TOTAL_SIZE_ON_SERVER),
+ offlineSizes._estimatedSizeInBytes);
+ assertEquals(MetricValueUtils
+ .getTableGaugeValue(_controllerMetrics, tableNameWithType, ControllerGauge.LARGEST_SEGMENT_SIZE_ON_SERVER),
+ 160);
}
@Test
- public void testGetTableSubTypeSizeAllErrors()
- throws InvalidConfigException {
+ public void testGetTableSubTypeSizeAllErrors() throws InvalidConfigException {
final String[] servers = {"server2", "server5"};
String table = "offline";
TableSizeReader.TableSizeDetails tableSizeDetails = testRunner(servers, table);
@@ -346,24 +345,23 @@ public class TableSizeReaderTest {
assertEquals(offlineSizes._segments.size(), 3);
assertEquals(offlineSizes._reportedSizeInBytes, TableSizeReader.DEFAULT_SIZE_WHEN_MISSING_OR_ERROR);
assertEquals(tableSizeDetails._estimatedSizeInBytes, TableSizeReader.DEFAULT_SIZE_WHEN_MISSING_OR_ERROR);
+ assertEquals(tableSizeDetails._reportedSizePerReplicaInBytes, TableSizeReader.DEFAULT_SIZE_WHEN_MISSING_OR_ERROR);
String tableNameWithType = TableNameBuilder.OFFLINE.tableNameWithType(table);
- assertEquals(
- MetricValueUtils.getTableGaugeValue(_controllerMetrics, tableNameWithType,
- ControllerGauge.TABLE_STORAGE_EST_MISSING_SEGMENT_PERCENT), 100);
- assertEquals(
- MetricValueUtils.getTableGaugeValue(_controllerMetrics, tableNameWithType,
- ControllerGauge.TABLE_SIZE_PER_REPLICA_ON_SERVER), offlineSizes._estimatedSizeInBytes / NUM_REPLICAS);
- assertEquals(
- MetricValueUtils.getTableGaugeValue(_controllerMetrics, tableNameWithType,
- ControllerGauge.TABLE_TOTAL_SIZE_ON_SERVER), offlineSizes._estimatedSizeInBytes);
- assertFalse(
- MetricValueUtils.tableGaugeExists(_controllerMetrics, tableNameWithType,
- ControllerGauge.LARGEST_SEGMENT_SIZE_ON_SERVER));
+ assertEquals(MetricValueUtils.getTableGaugeValue(_controllerMetrics, tableNameWithType,
+ ControllerGauge.TABLE_STORAGE_EST_MISSING_SEGMENT_PERCENT), 100);
+ assertEquals(MetricValueUtils
+ .getTableGaugeValue(_controllerMetrics, tableNameWithType,
+ ControllerGauge.TABLE_SIZE_PER_REPLICA_ON_SERVER),
+ offlineSizes._estimatedSizeInBytes / NUM_REPLICAS);
+ assertEquals(MetricValueUtils
+ .getTableGaugeValue(_controllerMetrics, tableNameWithType, ControllerGauge.TABLE_TOTAL_SIZE_ON_SERVER),
+ offlineSizes._estimatedSizeInBytes);
+ assertFalse(MetricValueUtils
+ .tableGaugeExists(_controllerMetrics, tableNameWithType, ControllerGauge.LARGEST_SEGMENT_SIZE_ON_SERVER));
}
@Test
- public void testGetTableSubTypeSizesWithErrors()
- throws InvalidConfigException {
+ public void testGetTableSubTypeSizesWithErrors() throws InvalidConfigException {
final String[] servers = {"server0", "server1", "server2", "server5"};
String table = "offline";
TableSizeReader.TableSizeDetails tableSizeDetails = testRunner(servers, "offline");
@@ -374,23 +372,25 @@ public class TableSizeReaderTest {
validateTableSubTypeSize(servers, offlineSizes);
assertNull(tableSizeDetails._realtimeSegments);
String tableNameWithType = TableNameBuilder.OFFLINE.tableNameWithType(table);
- assertEquals(
- MetricValueUtils.getTableGaugeValue(_controllerMetrics, tableNameWithType,
- ControllerGauge.TABLE_STORAGE_EST_MISSING_SEGMENT_PERCENT), 20);
- assertEquals(
- MetricValueUtils.getTableGaugeValue(_controllerMetrics, tableNameWithType,
- ControllerGauge.TABLE_SIZE_PER_REPLICA_ON_SERVER), offlineSizes._estimatedSizeInBytes / NUM_REPLICAS);
- assertEquals(
- MetricValueUtils.getTableGaugeValue(_controllerMetrics, tableNameWithType,
- ControllerGauge.TABLE_TOTAL_SIZE_ON_SERVER), offlineSizes._estimatedSizeInBytes);
- assertEquals(
- MetricValueUtils.getTableGaugeValue(_controllerMetrics, tableNameWithType,
- ControllerGauge.LARGEST_SEGMENT_SIZE_ON_SERVER), 160);
+ assertEquals(tableSizeDetails._reportedSizeInBytes, offlineSizes._reportedSizeInBytes);
+ assertEquals(tableSizeDetails._estimatedSizeInBytes, offlineSizes._estimatedSizeInBytes);
+ assertEquals(tableSizeDetails._reportedSizePerReplicaInBytes, offlineSizes._reportedSizePerReplicaInBytes);
+ assertEquals(MetricValueUtils.getTableGaugeValue(_controllerMetrics, tableNameWithType,
+ ControllerGauge.TABLE_STORAGE_EST_MISSING_SEGMENT_PERCENT), 20);
+ assertEquals(MetricValueUtils
+ .getTableGaugeValue(_controllerMetrics, tableNameWithType,
+ ControllerGauge.TABLE_SIZE_PER_REPLICA_ON_SERVER),
+ offlineSizes._estimatedSizeInBytes / NUM_REPLICAS);
+ assertEquals(MetricValueUtils
+ .getTableGaugeValue(_controllerMetrics, tableNameWithType, ControllerGauge.TABLE_TOTAL_SIZE_ON_SERVER),
+ offlineSizes._estimatedSizeInBytes);
+ assertEquals(MetricValueUtils
+ .getTableGaugeValue(_controllerMetrics, tableNameWithType, ControllerGauge.LARGEST_SEGMENT_SIZE_ON_SERVER),
+ 160);
}
@Test
- public void getTableSizeDetailsRealtimeOnly()
- throws InvalidConfigException {
+ public void getTableSizeDetailsRealtimeOnly() throws InvalidConfigException {
final String[] servers = {"server3", "server4"};
String table = "realtime";
TableSizeReader.TableSizeDetails tableSizeDetails = testRunner(servers, table);
@@ -399,15 +399,19 @@ public class TableSizeReaderTest {
assertEquals(realtimeSegments._segments.size(), 2);
assertEquals(realtimeSegments._estimatedSizeInBytes, realtimeSegments._reportedSizeInBytes);
validateTableSubTypeSize(servers, realtimeSegments);
+ assertEquals(tableSizeDetails._reportedSizeInBytes, realtimeSegments._reportedSizeInBytes);
+ assertEquals(tableSizeDetails._estimatedSizeInBytes, realtimeSegments._estimatedSizeInBytes);
+ assertEquals(tableSizeDetails._reportedSizePerReplicaInBytes, realtimeSegments._reportedSizePerReplicaInBytes);
String tableNameWithType = TableNameBuilder.REALTIME.tableNameWithType(table);
- assertEquals(
- MetricValueUtils.getTableGaugeValue(_controllerMetrics, tableNameWithType,
- ControllerGauge.TABLE_SIZE_PER_REPLICA_ON_SERVER), realtimeSegments._estimatedSizeInBytes / NUM_REPLICAS);
- assertEquals(
- MetricValueUtils.getTableGaugeValue(_controllerMetrics, tableNameWithType,
- ControllerGauge.TABLE_TOTAL_SIZE_ON_SERVER), realtimeSegments._estimatedSizeInBytes);
- assertEquals(
- MetricValueUtils.getTableGaugeValue(_controllerMetrics, tableNameWithType,
- ControllerGauge.LARGEST_SEGMENT_SIZE_ON_SERVER), 120);
+ assertEquals(MetricValueUtils
+ .getTableGaugeValue(_controllerMetrics, tableNameWithType,
+ ControllerGauge.TABLE_SIZE_PER_REPLICA_ON_SERVER),
+ realtimeSegments._estimatedSizeInBytes / NUM_REPLICAS);
+ assertEquals(MetricValueUtils
+ .getTableGaugeValue(_controllerMetrics, tableNameWithType, ControllerGauge.TABLE_TOTAL_SIZE_ON_SERVER),
+ realtimeSegments._estimatedSizeInBytes);
+ assertEquals(MetricValueUtils
+ .getTableGaugeValue(_controllerMetrics, tableNameWithType, ControllerGauge.LARGEST_SEGMENT_SIZE_ON_SERVER),
+ 120);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org