You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ki...@apache.org on 2020/09/14 16:28:24 UTC
[incubator-pinot] branch master updated: Feature/#5390 segment
indexing reload status api (#5718)
This is an automated email from the ASF dual-hosted git repository.
kishoreg pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new dc77271 Feature/#5390 segment indexing reload status api (#5718)
dc77271 is described below
commit dc772712f3d52c0c91caa774c320f9ca54b855dd
Author: Guruguha <gu...@users.noreply.github.com>
AuthorDate: Mon Sep 14 09:28:04 2020 -0700
Feature/#5390 segment indexing reload status api (#5718)
* - initial feature push
* Segment Reload API
- added a new API endpoint for users to query segment reload status
API - Table metadata from Server
- added a new endpoint to fetch segment metadata
- added helper classes and methods to fetch metadata from the server
Tests
- added test to server API to fetch metadata including indexing information
* Code Refactor:
- Moved status classes to logical places
Logs
- Added logging statements
Tests
- Added unit tests for Pinot Controller reload status and segment metadata API
- Added unit tests for Pinot Server reload status and segment metadata API
License Headers
- Add license headers to files added to this feature
* Code Review Changes
- Updating code as per PR review comments
* Adding comments to new classes and methods added as part of this feature
Removing SegmentMetadataFetcher as it seemed redundant
Refactoring code to save failed segment reload status API calls as part of response
* Code refactor to accommodate PR comments
Pinot codestyle corrections
Moving ServerSegmentMetadataReader to util
* Updating API definition for loadStatus
* Code refactor to remove duplicate code
* Code refactor to remove duplicate code
* Code refactor as per PR comments
* Updating segment loadStatus API to return long time than readable string
* - Bug fix on the server API endpoint
* Adding pretty print for server metadata response
* Reverting incorrect filename refactoring
Updating variable names to reflect their value type
* removing unused method variable
* Enabling Pretty print of server response.
Code refactor to clean up lines that went beyond line length
* Removing reload status API
* Pretty print result
* Fix test
Co-authored-by: Neha Pawar <ne...@gmail.com>
---
.../api/resources/PinotSegmentRestletResource.java | 59 +++++-
.../api/resources/ServerTableSizeReader.java | 49 ++---
.../controller/util/CompletionServiceHelper.java | 108 ++++++++++
.../util/ServerSegmentMetadataReader.java | 92 +++++++++
.../pinot/controller/util/TableMetadataReader.java | 74 +++++++
.../controller/api/PinotSegmentsMetadataTest.java | 230 +++++++++++++++++++++
.../immutable/ImmutableSegmentImpl.java | 4 +
.../api/resources/SegmentMetadataFetcher.java | 135 ++++++++++++
.../pinot/server/api/resources/TablesResource.java | 22 +-
.../pinot/server/api/TablesResourceTest.java | 55 ++++-
10 files changed, 773 insertions(+), 55 deletions(-)
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotSegmentRestletResource.java b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotSegmentRestletResource.java
index cd622d2..876d3a9 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotSegmentRestletResource.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotSegmentRestletResource.java
@@ -19,14 +19,17 @@
package org.apache.pinot.controller.api.resources;
import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonNode;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import io.swagger.annotations.ApiParam;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.Executor;
import javax.annotation.Nullable;
import javax.inject.Inject;
import javax.ws.rs.Consumes;
@@ -42,16 +45,20 @@ import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.Response.Status;
+import org.apache.commons.httpclient.HttpConnectionManager;
import org.apache.helix.ZNRecord;
import org.apache.helix.store.zk.ZkHelixPropertyStore;
+import org.apache.pinot.common.exception.InvalidConfigException;
import org.apache.pinot.common.exception.TableNotFoundException;
import org.apache.pinot.common.metadata.ZKMetadataProvider;
import org.apache.pinot.common.metadata.segment.OfflineSegmentZKMetadata;
import org.apache.pinot.common.metadata.segment.RealtimeSegmentZKMetadata;
import org.apache.pinot.common.utils.SegmentName;
import org.apache.pinot.common.utils.URIUtils;
+import org.apache.pinot.controller.ControllerConf;
import org.apache.pinot.controller.helix.core.PinotHelixResourceManager;
import org.apache.pinot.controller.helix.core.PinotResourceManagerResponse;
+import org.apache.pinot.controller.util.TableMetadataReader;
import org.apache.pinot.spi.config.table.TableType;
import org.apache.pinot.spi.utils.JsonUtils;
import org.apache.pinot.spi.utils.builder.TableNameBuilder;
@@ -69,6 +76,7 @@ import org.slf4j.LoggerFactory;
* <li>"/segments/{tableName}/servers": get a map from server to segments hosted by the server</li>
* <li>"/segments/{tableName}/crc": get a map from segment to CRC of the segment (OFFLINE table only)</li>
* <li>"/segments/{tableName}/{segmentName}/metadata: get the metadata for a segment</li>
+ * <li>"/segments/{tableName}/metadata: get the metadata for all segments from the server</li>
* </ul>
* </li>
* <li>
@@ -128,11 +136,20 @@ import org.slf4j.LoggerFactory;
@Api(tags = Constants.SEGMENT_TAG)
@Path("/")
public class PinotSegmentRestletResource {
- private static Logger LOGGER = LoggerFactory.getLogger(PinotSegmentRestletResource.class);
+ private static final Logger LOGGER = LoggerFactory.getLogger(PinotSegmentRestletResource.class);
+
+ @Inject
+ ControllerConf _controllerConf;
@Inject
PinotHelixResourceManager _pinotHelixResourceManager;
+ @Inject
+ Executor _executor;
+
+ @Inject
+ HttpConnectionManager _connectionManager;
+
@GET
@Produces(MediaType.APPLICATION_JSON)
@Path("/segments/{tableName}")
@@ -485,4 +502,44 @@ public class PinotSegmentRestletResource {
throw new ControllerApplicationException(LOGGER, e.getMessage(), Response.Status.FORBIDDEN);
}
}
+
+ @GET
+ @Path("segments/{tableName}/metadata")
+ @Produces(MediaType.APPLICATION_JSON)
+ @ApiOperation(value = "Get the server metadata for all table segments", notes = "Get the server metadata for all table segments")
+ public String getServerMetadata(@ApiParam(value = "Name of the table", required = true) @PathParam("tableName") String tableName,
+ @ApiParam(value = "OFFLINE|REALTIME") @QueryParam("type") String tableTypeStr) {
+ LOGGER.info("Received a request to fetch metadata for all segments for table {}", tableName);
+ TableType tableType = Constants.validateTableType(tableTypeStr);
+ if (tableType == TableType.REALTIME) {
+ throw new ControllerApplicationException(LOGGER,
+ "Table type : " + tableTypeStr + " not yet supported.", Status.NOT_IMPLEMENTED);
+ }
+
+ String tableNameWithType = getExistingTableNamesWithType(tableName, tableType).get(0);
+ String segmentsMetadata;
+ try {
+ JsonNode segmentsMetadataJson = getSegmentsMetadataFromServer(tableNameWithType);
+ segmentsMetadata = JsonUtils.objectToPrettyString(segmentsMetadataJson);
+ } catch (InvalidConfigException e) {
+ throw new ControllerApplicationException(LOGGER, e.getMessage(), Status.BAD_REQUEST);
+ } catch (IOException ioe) {
+ throw new ControllerApplicationException(LOGGER,
+ "Error parsing Pinot server response: " + ioe.getMessage(), Status.INTERNAL_SERVER_ERROR, ioe);
+ }
+ return segmentsMetadata;
+ }
+
+ /**
+ * This is a helper method to get the metadata for all segments for a given table name.
+ * @param tableNameWithType name of the table along with its type
+ * @return Map<String, String> metadata of the table segments -> map of segment name to its metadata
+ */
+ private JsonNode getSegmentsMetadataFromServer(String tableNameWithType)
+ throws InvalidConfigException, IOException {
+ TableMetadataReader tableMetadataReader =
+ new TableMetadataReader(_executor, _connectionManager, _pinotHelixResourceManager);
+ return tableMetadataReader.getSegmentsMetadata(tableNameWithType,
+ _controllerConf.getServerAdminRequestTimeoutSeconds() * 1000);
+ }
}
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/ServerTableSizeReader.java b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/ServerTableSizeReader.java
index f551418..98c6374 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/ServerTableSizeReader.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/ServerTableSizeReader.java
@@ -19,18 +19,16 @@
package org.apache.pinot.controller.api.resources;
import com.google.common.collect.BiMap;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.concurrent.CompletionService;
import java.util.concurrent.Executor;
import org.apache.commons.httpclient.HttpConnectionManager;
-import org.apache.commons.httpclient.URI;
-import org.apache.commons.httpclient.methods.GetMethod;
-import org.apache.pinot.common.http.MultiGetRequest;
import org.apache.pinot.common.restlet.resources.SegmentSizeInfo;
import org.apache.pinot.common.restlet.resources.TableSizeInfo;
+import org.apache.pinot.controller.util.CompletionServiceHelper;
import org.apache.pinot.spi.utils.JsonUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -64,40 +62,25 @@ public class ServerTableSizeReader {
serverUrls.add(tableSizeUri);
}
- // TODO: use some service other than completion service so that we know which server encounters the error
- CompletionService<GetMethod> completionService =
- new MultiGetRequest(_executor, _connectionManager).execute(serverUrls, timeoutMs);
+ // Helper service to run a httpget call to the server
+ CompletionServiceHelper completionServiceHelper = new CompletionServiceHelper(_executor, _connectionManager,
+ endpointsToServers);
+ CompletionServiceHelper.CompletionServiceResponse serviceResponse =
+ completionServiceHelper.doMultiGetRequest(serverUrls, tableNameWithType, timeoutMs);
Map<String, List<SegmentSizeInfo>> serverToSegmentSizeInfoListMap = new HashMap<>();
-
- for (int i = 0; i < numServers; i++) {
- GetMethod getMethod = null;
+ int failedParses = 0;
+ for (Map.Entry<String, String> streamResponse : serviceResponse._httpResponses.entrySet()) {
try {
- getMethod = completionService.take().get();
- URI uri = getMethod.getURI();
- String instance = endpointsToServers.get(uri.getHost() + ":" + uri.getPort());
- if (getMethod.getStatusCode() >= 300) {
- LOGGER.error("Server: {} returned error: {}", instance, getMethod.getStatusCode());
- continue;
- }
TableSizeInfo tableSizeInfo =
- JsonUtils.inputStreamToObject(getMethod.getResponseBodyAsStream(), TableSizeInfo.class);
- serverToSegmentSizeInfoListMap.put(instance, tableSizeInfo.segments);
- } catch (Exception e) {
- // Ignore individual exceptions because the exception has been logged in MultiGetRequest
- // Log the number of failed servers after gathering all responses
- } finally {
- if (getMethod != null) {
- getMethod.releaseConnection();
- }
+ JsonUtils.stringToObject(streamResponse.getValue(), TableSizeInfo.class);
+ serverToSegmentSizeInfoListMap.put(streamResponse.getKey(), tableSizeInfo.segments);
+ } catch (IOException e) {
+ failedParses++;
+ LOGGER.error("Unable to parse server {} response due to an error: ", streamResponse.getKey(), e);
}
}
-
- int numServersResponded = serverToSegmentSizeInfoListMap.size();
- if (numServersResponded != numServers) {
- LOGGER.warn("Finish reading segment sizes for table: {} with {}/{} servers responded", tableNameWithType,
- numServersResponded, numServers);
- } else {
- LOGGER.info("Finish reading segment sizes for table: {}", tableNameWithType);
+ if (failedParses != 0) {
+ LOGGER.warn("Failed to parse {} / {} segment size info responses from servers.", failedParses, serverUrls.size());
}
return serverToSegmentSizeInfoListMap;
}
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/util/CompletionServiceHelper.java b/pinot-controller/src/main/java/org/apache/pinot/controller/util/CompletionServiceHelper.java
new file mode 100644
index 0000000..c9fd4e3
--- /dev/null
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/util/CompletionServiceHelper.java
@@ -0,0 +1,108 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.pinot.controller.util;
+
+import com.google.common.collect.BiMap;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CompletionService;
+import java.util.concurrent.Executor;
+import org.apache.commons.httpclient.HttpConnectionManager;
+import org.apache.commons.httpclient.URI;
+import org.apache.commons.httpclient.methods.GetMethod;
+import org.apache.pinot.common.http.MultiGetRequest;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This is a helper class that can be used to make HttpGet (MultiGet) calls and get the responses back.
+ * The responses are returned as a string.
+ *
+ * The helper also records number of failed responses so that the caller knows if any of the calls
+ * failed to respond. The failed instance is logged for debugging.
+ */
+public class CompletionServiceHelper {
+ private static final Logger LOGGER = LoggerFactory.getLogger(CompletionServiceHelper.class);
+
+ private final Executor _executor;
+ private final HttpConnectionManager _httpConnectionManager;
+ private final BiMap<String, String> _endpointsToServers;
+
+ public CompletionServiceHelper(Executor executor, HttpConnectionManager httpConnectionManager,
+ BiMap<String, String> endpointsToServers) {
+ _executor = executor;
+ _httpConnectionManager = httpConnectionManager;
+ _endpointsToServers = endpointsToServers;
+ }
+
+ public CompletionServiceResponse doMultiGetRequest(List<String> serverURLs, String tableNameWithType, int timeoutMs) {
+ CompletionServiceResponse completionServiceResponse = new CompletionServiceResponse();
+
+ // TODO: use some service other than completion service so that we know which server encounters the error
+ CompletionService<GetMethod> completionService =
+ new MultiGetRequest(_executor, _httpConnectionManager).execute(serverURLs, timeoutMs);
+ for (int i = 0; i < serverURLs.size(); i++) {
+ GetMethod getMethod = null;
+ try {
+ getMethod = completionService.take().get();
+ URI uri = getMethod.getURI();
+ String instance = _endpointsToServers.get(uri.getHost() + ":" + uri.getPort());
+ if (getMethod.getStatusCode() >= 300) {
+ LOGGER.error("Server: {} returned error: {}", instance, getMethod.getStatusCode());
+ completionServiceResponse._failedResponseCount++;
+ continue;
+ }
+ completionServiceResponse._httpResponses.put(instance, getMethod.getResponseBodyAsString());
+ } catch (Exception e) {
+ // Ignore individual exceptions because the exception has been logged in MultiGetRequest
+ // Log the number of failed servers after gathering all responses
+ } finally {
+ if (getMethod != null) {
+ getMethod.releaseConnection();
+ }
+ }
+ }
+
+ int numServersResponded = completionServiceResponse._httpResponses.size();
+ if (numServersResponded != serverURLs.size()) {
+ LOGGER.warn("Finished reading information for table: {} with {}/{} server responses", tableNameWithType,
+ numServersResponded, serverURLs.size());
+ } else {
+ LOGGER.info("Finished reading information for table: {}", tableNameWithType);
+ }
+ return completionServiceResponse;
+ }
+
+ /**
+ * Helper class to maintain the completion service response to be sent back to the caller.
+ */
+ static public class CompletionServiceResponse {
+ // Map of the server instance to the response from that server
+ public Map<String, String> _httpResponses;
+ // Number of failures encountered when requesting
+ public int _failedResponseCount;
+
+ public CompletionServiceResponse() {
+ _httpResponses = new HashMap<>();
+ _failedResponseCount = 0;
+ }
+ }
+}
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/util/ServerSegmentMetadataReader.java b/pinot-controller/src/main/java/org/apache/pinot/controller/util/ServerSegmentMetadataReader.java
new file mode 100644
index 0000000..65fc63f
--- /dev/null
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/util/ServerSegmentMetadataReader.java
@@ -0,0 +1,92 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.controller.util;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.google.common.collect.BiMap;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Executor;
+import org.apache.commons.httpclient.HttpConnectionManager;
+import org.apache.pinot.spi.utils.JsonUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This is a helper class that calls the server API endpoints to fetch server metadata and the segment reload status
+ * Only the servers returning success are returned by the method. For servers returning errors (http error or otherwise),
+ * no entry is created in the return list
+ */
+public class ServerSegmentMetadataReader {
+ private static final Logger LOGGER = LoggerFactory.getLogger(ServerSegmentMetadataReader.class);
+
+ private final Executor _executor;
+ private final HttpConnectionManager _connectionManager;
+
+ public ServerSegmentMetadataReader(Executor executor, HttpConnectionManager connectionManager) {
+ _executor = executor;
+ _connectionManager = connectionManager;
+ }
+
+ /**
+ * This method is called when the API request is to fetch segment metadata for all segments of the table.
+ * This method makes a MultiGet call to all servers that host their respective segments and gets the results.
+ * @return list of segments and their metadata as a JSON string
+ */
+ public List<String> getSegmentMetadataFromServer(String tableNameWithType,
+ Map<String, List<String>> serversToSegmentsMap,
+ BiMap<String, String> endpoints, int timeoutMs) {
+ LOGGER.debug("Reading segment metadata from servers for table {}.", tableNameWithType);
+ List<String> serverURLs = new ArrayList<>();
+ for (Map.Entry<String, List<String>> serverToSegments : serversToSegmentsMap.entrySet()) {
+ List<String> segments = serverToSegments.getValue();
+ for (String segment : segments) {
+ serverURLs.add(generateSegmentMetadataServerURL(tableNameWithType, segment, endpoints.get(serverToSegments.getKey())));
+ }
+ }
+ BiMap<String, String> endpointsToServers = endpoints.inverse();
+ CompletionServiceHelper completionServiceHelper = new CompletionServiceHelper(_executor, _connectionManager, endpointsToServers);
+ CompletionServiceHelper.CompletionServiceResponse serviceResponse =
+ completionServiceHelper.doMultiGetRequest(serverURLs, tableNameWithType, timeoutMs);
+ List<String> segmentsMetadata = new ArrayList<>();
+
+ int failedParses = 0;
+ for (Map.Entry<String, String> streamResponse : serviceResponse._httpResponses.entrySet()) {
+ try {
+ String segmentMetadata = streamResponse.getValue();
+ segmentsMetadata.add(segmentMetadata);
+ } catch (Exception e) {
+ failedParses++;
+ LOGGER.error("Unable to parse server {} response due to an error: ", streamResponse.getKey(), e);
+ }
+ }
+ if (failedParses != 0) {
+ LOGGER.error("Unable to parse server {} / {} response due to an error: ", failedParses, serverURLs.size());
+ }
+
+ LOGGER.debug("Retrieved segment metadata from servers.");
+ return segmentsMetadata;
+ }
+
+ private String generateSegmentMetadataServerURL(String tableNameWithType, String segmentName, String endpoint) {
+ return String.format("http://%s/tables/%s/segments/%s/metadata", endpoint, tableNameWithType, segmentName);
+ }
+}
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/util/TableMetadataReader.java b/pinot-controller/src/main/java/org/apache/pinot/controller/util/TableMetadataReader.java
new file mode 100644
index 0000000..b7a9658
--- /dev/null
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/util/TableMetadataReader.java
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.controller.util;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.google.common.collect.BiMap;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Executor;
+import org.apache.commons.httpclient.HttpConnectionManager;
+import org.apache.pinot.common.exception.InvalidConfigException;
+import org.apache.pinot.controller.helix.core.PinotHelixResourceManager;
+import org.apache.pinot.spi.utils.JsonUtils;
+
+/**
+ * This class acts as a bridge between the API call to controller and the internal API call made to the
+ * server to get segment metadata.
+ *
+ * Currently has two helper methods: one to retrieve the reload time and one to retrieve the segment metadata including
+ * the column indexes available.
+ */
+public class TableMetadataReader {
+ private final Executor _executor;
+ private final HttpConnectionManager _connectionManager;
+ private final PinotHelixResourceManager _pinotHelixResourceManager;
+
+ public TableMetadataReader(Executor executor, HttpConnectionManager connectionManager,
+ PinotHelixResourceManager helixResourceManager) {
+ _executor = executor;
+ _connectionManager = connectionManager;
+ _pinotHelixResourceManager = helixResourceManager;
+ }
+
+ /**
+ * This method retrieves the full segment metadata for a given table.
+ * Currently supports only OFFLINE tables.
+ * @return a map of segmentName to its metadata
+ */
+ public JsonNode getSegmentsMetadata(String tableNameWithType, int timeoutMs)
+ throws InvalidConfigException, IOException {
+ final Map<String, List<String>> serverToSegments =
+ _pinotHelixResourceManager.getServerToSegmentsMap(tableNameWithType);
+ BiMap<String, String> endpoints = _pinotHelixResourceManager.getDataInstanceAdminEndpoints(serverToSegments.keySet());
+ ServerSegmentMetadataReader serverSegmentMetadataReader = new ServerSegmentMetadataReader(_executor, _connectionManager);
+
+ List<String> segmentsMetadata = serverSegmentMetadataReader.getSegmentMetadataFromServer(tableNameWithType,
+ serverToSegments, endpoints, timeoutMs);
+
+ Map<String, JsonNode> response = new HashMap<>();
+ for (String segmentMetadata : segmentsMetadata) {
+ JsonNode responseJson = JsonUtils.stringToJsonNode(segmentMetadata);
+ response.put(responseJson.get("segmentName").asText(), responseJson);
+ }
+ return JsonUtils.objectToJsonNode(response);
+ }
+}
diff --git a/pinot-controller/src/test/java/org/apache/pinot/controller/api/PinotSegmentsMetadataTest.java b/pinot-controller/src/test/java/org/apache/pinot/controller/api/PinotSegmentsMetadataTest.java
new file mode 100644
index 0000000..96efa87
--- /dev/null
+++ b/pinot-controller/src/test/java/org/apache/pinot/controller/api/PinotSegmentsMetadataTest.java
@@ -0,0 +1,230 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.controller.api;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.collect.BiMap;
+import com.google.common.collect.HashBiMap;
+import com.sun.net.httpserver.HttpHandler;
+import com.sun.net.httpserver.HttpServer;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.InetSocketAddress;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Executor;
+import java.util.concurrent.Executors;
+import org.apache.commons.httpclient.HttpConnectionManager;
+import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
+import org.apache.pinot.controller.helix.core.PinotHelixResourceManager;
+import org.apache.pinot.controller.util.ServerSegmentMetadataReader;
+import org.apache.pinot.spi.utils.JsonUtils;
+import org.mockito.stubbing.Answer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.Assert;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class PinotSegmentsMetadataTest {
+ private static final Logger LOGGER = LoggerFactory.getLogger(PinotSegmentsMetadataTest.class);
+ private final Executor executor = Executors.newFixedThreadPool(1);
+ private final HttpConnectionManager connectionManager = new MultiThreadedHttpConnectionManager();
+ private final String URI_PATH = "/tables/";
+ private final int timeoutMsec = 10000;
+ private final Map<String, SegmentsServerMock> serverMap = new HashMap<>();
+ private PinotHelixResourceManager helix;
+
+ private BiMap<String, String> serverEndpoints(List<String> servers) {
+ BiMap<String, String> endpoints = HashBiMap.create(servers.size());
+ for (String server : servers) {
+ endpoints.put(server, serverMap.get(server).endpoint);
+ }
+ return endpoints;
+ }
+
+ @BeforeClass
+ public void setUp()
+ throws IOException {
+ helix = mock(PinotHelixResourceManager.class);
+ when(helix.hasOfflineTable(anyString())).thenAnswer((Answer) invocationOnMock -> {
+ String table = (String) invocationOnMock.getArguments()[0];
+ return table.contains("offline");
+ });
+
+ when(helix.hasRealtimeTable(anyString())).thenAnswer((Answer) invocationOnMock -> {
+ String table = (String) invocationOnMock.getArguments()[0];
+ return table.contains("realtime");
+ });
+
+ int counter = 0;
+ // server0
+ SegmentsServerMock s = new SegmentsServerMock("s1");
+ s.updateMetadataMock();
+ s.start(URI_PATH, createSegmentMetadataHandler(200, s.segmentMetadata, 0));
+ serverMap.put(serverName(counter), s);
+ ++counter;
+
+ // server1
+ s = new SegmentsServerMock("s2");
+ s.updateMetadataMock();
+ s.start(URI_PATH, createSegmentMetadataHandler(200, s.segmentMetadata, 0));
+ serverMap.put(serverName(counter), s);
+ ++counter;
+
+ // server2
+ s = new SegmentsServerMock("s3");
+ s.updateMetadataMock();
+ s.start(URI_PATH, createSegmentMetadataHandler(404, s.segmentMetadata, 0));
+ serverMap.put(serverName(counter), s);
+ ++counter;
+ }
+
+ private String serverName(int counter) {
+ return "server" + counter;
+ }
+
+ @AfterClass
+ public void tearDown() {
+ for (Map.Entry<String, SegmentsServerMock> fakeServerEntry : serverMap.entrySet()) {
+ fakeServerEntry.getValue().httpServer.stop(0);
+ }
+ }
+
+ private HttpHandler createSegmentMetadataHandler(final int status, final String segmentMetadata, final int sleepTimeMs) {
+ return httpExchange -> {
+ if (sleepTimeMs > 0) {
+ try {
+ Thread.sleep(sleepTimeMs);
+ } catch (InterruptedException e) {
+ LOGGER.info("Handler interrupted during sleep");
+ }
+ }
+
+ String json = JsonUtils.objectToString(segmentMetadata);
+ httpExchange.sendResponseHeaders(status, json.length());
+ OutputStream responseBody = httpExchange.getResponseBody();
+ responseBody.write(json.getBytes());
+ responseBody.close();
+ };
+ }
+
+ private List<String> testMetadataResponse(String table, Map<String, List<String>> serverToSegmentsMap,
+ BiMap<String, String> endpoints) {
+ ServerSegmentMetadataReader metadataReader = new ServerSegmentMetadataReader(executor, connectionManager);
+ return metadataReader.getSegmentMetadataFromServer(table, serverToSegmentsMap, endpoints, timeoutMsec);
+ }
+
+ private Map<String, List<String>> getServerToSegments(List<String> servers) {
+ Map<String, List<String>> serverToSegmentsMap = new HashMap<>();
+ for (String server : servers) {
+ serverToSegmentsMap.put(server, Collections.singletonList(serverMap.get(server).segment));
+ }
+ return serverToSegmentsMap;
+ }
+
+ @Test
+ public void testServerSegmentMetadataFetchSuccess() {
+ final List<String> servers = MetadataConstants.SEGMENT_SERVERS.subList(0, 1);
+ Map<String, List<String>> serverToSegmentsMap = getServerToSegments(servers);
+ BiMap<String, String> endpoints = serverEndpoints(servers);
+ String table = "offline";
+ List<String> metadata = testMetadataResponse(table, serverToSegmentsMap, endpoints);
+ Assert.assertEquals(1, metadata.size());
+ }
+
+ @Test
+ public void testServerSegmentMetadataFetchError() {
+ final List<String> servers = MetadataConstants.SEGMENT_SERVERS.subList(0, 2);
+ Map<String, List<String>> serverToSegmentsMap = getServerToSegments(servers);
+ int expectedNonResponsiveServers = 0;
+ int totalResponses = 0;
+ for (String server : serverToSegmentsMap.keySet()) {
+ if (server.equalsIgnoreCase("server2"))
+ expectedNonResponsiveServers += serverToSegmentsMap.get(server).size();
+ totalResponses += serverToSegmentsMap.get(server).size();
+ }
+ BiMap<String, String> endpoints = serverEndpoints(servers);
+ String table = "offline";
+ List<String> metadata = testMetadataResponse(table, serverToSegmentsMap, endpoints);
+ Assert.assertEquals(1, metadata.size());
+ Assert.assertEquals(expectedNonResponsiveServers, totalResponses - metadata.size());
+ }
+
+ public static class SegmentsServerMock {
+ String segment;
+ String endpoint;
+ InetSocketAddress socket = new InetSocketAddress(0);
+ String segmentMetadata;
+ HttpServer httpServer;
+
+ public SegmentsServerMock(String segment) {
+ this.segment = segment;
+ }
+
+ private void updateMetadataMock() throws IOException {
+ JsonNode jsonNode = JsonUtils.stringToJsonNode(MetadataConstants.SEGMENT_METADATA_STR);
+ ObjectNode objectNode = jsonNode.deepCopy();
+ objectNode.put("segmentName", segment);
+ segmentMetadata = JsonUtils.objectToString(objectNode);
+ }
+
+ private void start(String path, HttpHandler handler)
+ throws IOException {
+ httpServer = HttpServer.create(socket, 0);
+ httpServer.createContext(path, handler);
+ new Thread(() -> httpServer.start()).start();
+ endpoint = "localhost:" + httpServer.getAddress().getPort();
+ }
+ }
+
+ public static class MetadataConstants {
+ public static final List<String> SEGMENT_SERVERS = Arrays.asList("server1", "server2", "server3", "server4", "server5");
+ public static final String SEGMENT_METADATA_STR = "{\n" +
+ " \"segmentName\" : \"testTable_OFFLINE_default_s1\",\n" +
+ " \"schemaName\" : null,\n" +
+ " \"crc\" : 1804064321,\n" +
+ " \"creationTimeMillis\" : 1595127594768,\n" +
+ " \"creationTimeReadable\" : \"2020-07-19T02:59:54:768 UTC\",\n" +
+ " \"timeGranularitySec\" : null,\n" +
+ " \"startTimeMillis\" : null,\n" +
+ " \"startTimeReadable\" : null,\n" +
+ " \"endTimeMillis\" : null,\n" +
+ " \"endTimeReadable\" : null,\n" +
+ " \"pushTimeMillis\" : -9223372036854775808,\n" +
+ " \"pushTimeReadable\" : null,\n" +
+ " \"refreshTimeMillis\" : -9223372036854775808,\n" +
+ " \"refreshTimeReadable\" : null,\n" +
+ " \"segmentVersion\" : \"v3\",\n" +
+ " \"creatorName\" : null,\n" +
+ " \"paddingCharacter\" : \"\\u0000\",\n" +
+ " \"columns\" : [ ],\n" +
+ " \"indexes\" : [ { } ]\n" +
+ "}";
+ }
+}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/immutable/ImmutableSegmentImpl.java b/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/immutable/ImmutableSegmentImpl.java
index 52537f5..ddae753 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/immutable/ImmutableSegmentImpl.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/immutable/ImmutableSegmentImpl.java
@@ -144,4 +144,8 @@ public class ImmutableSegmentImpl implements ImmutableSegment {
// NOTE: Use PinotSegmentRecordReader to read immutable segment
throw new UnsupportedOperationException();
}
+
+ public Map<String, ColumnIndexContainer> getIndexContainerMap() {
+ return _indexContainerMap;
+ }
}
diff --git a/pinot-server/src/main/java/org/apache/pinot/server/api/resources/SegmentMetadataFetcher.java b/pinot-server/src/main/java/org/apache/pinot/server/api/resources/SegmentMetadataFetcher.java
new file mode 100644
index 0000000..bc761c9
--- /dev/null
+++ b/pinot-server/src/main/java/org/apache/pinot/server/api/resources/SegmentMetadataFetcher.java
@@ -0,0 +1,135 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.server.api.resources;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.pinot.core.data.manager.SegmentDataManager;
+import org.apache.pinot.core.data.manager.offline.ImmutableSegmentDataManager;
+import org.apache.pinot.core.indexsegment.immutable.ImmutableSegment;
+import org.apache.pinot.core.indexsegment.immutable.ImmutableSegmentImpl;
+import org.apache.pinot.core.segment.index.column.ColumnIndexContainer;
+import org.apache.pinot.core.segment.index.metadata.SegmentMetadataImpl;
+import org.apache.pinot.spi.utils.JsonUtils;
+
+/**
+ * This is a wrapper class for fetching segment metadata related information.
+ */
+public class SegmentMetadataFetcher {
+ private static final String BLOOM_FILTER = "bloom-filter";
+ private static final String DICTIONARY = "dictionary";
+ private static final String FORWARD_INDEX = "forward-index";
+ private static final String INVERTED_INDEX = "inverted-index";
+ private static final String NULL_VALUE_VECTOR_READER = "null-value-vector-reader";
+ private static final String RANGE_INDEX = "range-index";
+
+ private static final String INDEX_NOT_AVAILABLE = "NO";
+ private static final String INDEX_AVAILABLE = "YES";
+
+ /**
+ * This is a helper method that fetches the segment metadata for a given segment.
+ * @param columns Columns to include for metadata
+ */
+ public static String getSegmentMetadata(SegmentDataManager segmentDataManager, List<String> columns)
+ throws JsonProcessingException {
+ SegmentMetadataImpl segmentMetadata = (SegmentMetadataImpl) segmentDataManager.getSegment().getSegmentMetadata();
+ Set<String> columnSet;
+ if (columns.size() == 1 && columns.get(0).equals("*")) {
+ columnSet = null;
+ } else {
+ columnSet = new HashSet<>(columns);
+ }
+ ObjectNode segmentMetadataJson = (ObjectNode) segmentMetadata.toJson(columnSet);
+ segmentMetadataJson.set("indexes", JsonUtils.objectToJsonNode(getIndexesForSegmentColumns(segmentDataManager)));
+ return JsonUtils.objectToString(segmentMetadataJson);
+ }
+
+ /**
+ * Get the JSON object with the segment column's indexing metadata.
+ */
+ private static Map<String, Map<String, String>> getIndexesForSegmentColumns(SegmentDataManager segmentDataManager) {
+ Map<String, Map<String, String>> columnIndexMap = null;
+ if (segmentDataManager instanceof ImmutableSegmentDataManager) {
+ ImmutableSegmentDataManager immutableSegmentDataManager = (ImmutableSegmentDataManager) segmentDataManager;
+ ImmutableSegment immutableSegment = immutableSegmentDataManager.getSegment();
+ if (immutableSegment instanceof ImmutableSegmentImpl) {
+ ImmutableSegmentImpl immutableSegmentImpl = (ImmutableSegmentImpl) immutableSegment;
+ Map<String, ColumnIndexContainer> columnIndexContainerMap = immutableSegmentImpl.getIndexContainerMap();
+ columnIndexMap = getImmutableSegmentColumnIndexes(columnIndexContainerMap);
+ }
+ }
+ return columnIndexMap;
+ }
+
+ /**
+ * Helper to loop through column index container to create a index map as follows for each column:
+ * {<"bloom-filter", "YES">, <"dictionary", "NO">}
+ */
+ private static Map<String, Map<String, String>> getImmutableSegmentColumnIndexes(Map<String, ColumnIndexContainer> columnIndexContainerMap) {
+ Map<String, Map<String, String>> columnIndexMap = new LinkedHashMap<>();
+ for (Map.Entry<String, ColumnIndexContainer> entry : columnIndexContainerMap.entrySet()) {
+ ColumnIndexContainer columnIndexContainer = entry.getValue();
+ Map<String, String> indexStatus = new LinkedHashMap<>();
+ if (Objects.isNull(columnIndexContainer.getBloomFilter())) {
+ indexStatus.put(BLOOM_FILTER, INDEX_NOT_AVAILABLE);
+ } else {
+ indexStatus.put(BLOOM_FILTER, INDEX_AVAILABLE);
+ }
+
+ if (Objects.isNull(columnIndexContainer.getDictionary())) {
+ indexStatus.put(DICTIONARY, INDEX_NOT_AVAILABLE);
+ } else {
+ indexStatus.put(DICTIONARY, INDEX_AVAILABLE);
+ }
+
+ if (Objects.isNull(columnIndexContainer.getForwardIndex())) {
+ indexStatus.put(FORWARD_INDEX, INDEX_NOT_AVAILABLE);
+ } else {
+ indexStatus.put(FORWARD_INDEX, INDEX_AVAILABLE);
+ }
+
+ if (Objects.isNull(columnIndexContainer.getInvertedIndex())) {
+ indexStatus.put(INVERTED_INDEX, INDEX_NOT_AVAILABLE);
+ } else {
+ indexStatus.put(INVERTED_INDEX, INDEX_AVAILABLE);
+ }
+
+ if (Objects.isNull(columnIndexContainer.getNullValueVector())) {
+ indexStatus.put(NULL_VALUE_VECTOR_READER, INDEX_NOT_AVAILABLE);
+ } else {
+ indexStatus.put(NULL_VALUE_VECTOR_READER, INDEX_AVAILABLE);
+ }
+
+ if (Objects.isNull(columnIndexContainer.getNullValueVector())) {
+ indexStatus.put(RANGE_INDEX, INDEX_NOT_AVAILABLE);
+ } else {
+ indexStatus.put(RANGE_INDEX, INDEX_AVAILABLE);
+ }
+
+ columnIndexMap.put(entry.getKey(), indexStatus);
+ }
+ return columnIndexMap;
+ }
+}
diff --git a/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TablesResource.java b/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TablesResource.java
index 6b8eee0..d670da7 100644
--- a/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TablesResource.java
+++ b/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TablesResource.java
@@ -27,10 +27,8 @@ import java.io.File;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.List;
import java.util.Map;
-import java.util.Set;
import java.util.UUID;
import javax.inject.Inject;
import javax.ws.rs.DefaultValue;
@@ -144,21 +142,13 @@ public class TablesResource {
throw new WebApplicationException(String.format("Table %s segments %s does not exist", tableName, segmentName),
Response.Status.NOT_FOUND);
}
+
try {
- SegmentMetadataImpl segmentMetadata = (SegmentMetadataImpl) segmentDataManager.getSegment().getSegmentMetadata();
- Set<String> columnSet;
- if (columns.size() == 1 && columns.get(0).equals("*")) {
- columnSet = null;
- } else {
- columnSet = new HashSet<>(columns);
- }
- try {
- return segmentMetadata.toJson(columnSet).toString();
- } catch (Exception e) {
- LOGGER.error("Failed to convert table {} segment {} to json", tableName, segmentMetadata);
- throw new WebApplicationException("Failed to convert segment metadata to json",
- Response.Status.INTERNAL_SERVER_ERROR);
- }
+ return SegmentMetadataFetcher.getSegmentMetadata(segmentDataManager, columns);
+ } catch (Exception e) {
+ LOGGER.error("Failed to convert table {} segment {} to json", tableName, segmentName);
+ throw new WebApplicationException("Failed to convert segment metadata to json",
+ Response.Status.INTERNAL_SERVER_ERROR);
} finally {
tableDataManager.releaseSegment(segmentDataManager);
}
diff --git a/pinot-server/src/test/java/org/apache/pinot/server/api/TablesResourceTest.java b/pinot-server/src/test/java/org/apache/pinot/server/api/TablesResourceTest.java
index 91aca47..071c991 100644
--- a/pinot-server/src/test/java/org/apache/pinot/server/api/TablesResourceTest.java
+++ b/pinot-server/src/test/java/org/apache/pinot/server/api/TablesResourceTest.java
@@ -33,15 +33,11 @@ import org.apache.pinot.core.segment.creator.impl.V1Constants;
import org.apache.pinot.core.segment.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.spi.utils.JsonUtils;
import org.apache.pinot.spi.utils.builder.TableNameBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.testng.Assert;
import org.testng.annotations.Test;
public class TablesResourceTest extends BaseResourceTest {
- private static final Logger LOGGER = LoggerFactory.getLogger(TablesResourceTest.class);
-
@Test
public void getTables()
throws Exception {
@@ -128,7 +124,8 @@ public class TablesResourceTest extends BaseResourceTest {
Assert.assertEquals(jsonResponse.get("columns").size(), 0);
jsonResponse = JsonUtils.stringToJsonNode(
- _webTarget.path(segmentMetadataPath).queryParam("columns", "column1").queryParam("columns", "column2").request()
+ _webTarget.path(segmentMetadataPath)
+ .queryParam("columns", "column1").queryParam("columns", "column2").request()
.get(String.class));
Assert.assertEquals(jsonResponse.get("columns").size(), 2);
@@ -214,4 +211,52 @@ public class TablesResourceTest extends BaseResourceTest {
FileUtils.forceDelete(tempMetadataDir);
}
+
+ @Test
+ public void testOfflineTableSegmentMetadata() throws Exception {
+ IndexSegment defaultSegment = _offlineIndexSegments.get(0);
+ String segmentMetadataPath =
+ "/tables/" + TableNameBuilder.OFFLINE.tableNameWithType(TABLE_NAME) + "/segments/" + defaultSegment
+ .getSegmentName() + "/metadata";
+
+ JsonNode jsonResponse =
+ JsonUtils.stringToJsonNode(_webTarget.path(segmentMetadataPath).request().get(String.class));
+
+ SegmentMetadataImpl segmentMetadata = (SegmentMetadataImpl) defaultSegment.getSegmentMetadata();
+ Assert.assertEquals(jsonResponse.get("segmentName").asText(), segmentMetadata.getName());
+ Assert.assertEquals(jsonResponse.get("crc").asText(), segmentMetadata.getCrc());
+ Assert.assertEquals(jsonResponse.get("creationTimeMillis").asLong(), segmentMetadata.getIndexCreationTime());
+ Assert.assertEquals(jsonResponse.get("paddingCharacter").asText(),
+ String.valueOf(segmentMetadata.getPaddingCharacter()));
+ Assert.assertEquals(jsonResponse.get("refreshTimeMillis").asLong(), segmentMetadata.getRefreshTime());
+ Assert.assertEquals(jsonResponse.get("pushTimeMillis").asLong(), segmentMetadata.getPushTime());
+ Assert.assertTrue(jsonResponse.has("pushTimeReadable"));
+ Assert.assertTrue(jsonResponse.has("refreshTimeReadable"));
+ Assert.assertTrue(jsonResponse.has("startTimeReadable"));
+ Assert.assertTrue(jsonResponse.has("endTimeReadable"));
+ Assert.assertTrue(jsonResponse.has("creationTimeReadable"));
+ Assert.assertEquals(jsonResponse.get("columns").size(), 0);
+ Assert.assertEquals(jsonResponse.get("indexes").size(), 17);
+
+
+ jsonResponse = JsonUtils.stringToJsonNode(
+ _webTarget.path(segmentMetadataPath)
+ .queryParam("columns", "column1").queryParam("columns", "column2").request()
+ .get(String.class));
+ Assert.assertEquals(jsonResponse.get("columns").size(), 2);
+ Assert.assertEquals(jsonResponse.get("indexes").size(), 17);
+
+ jsonResponse = JsonUtils.stringToJsonNode(
+ (_webTarget.path(segmentMetadataPath).queryParam("columns", "*").request().get(String.class)));
+ Assert.assertEquals(jsonResponse.get("columns").size(), segmentMetadata.getAllColumns().size());
+
+ Response response = _webTarget.path("/tables/UNKNOWN_TABLE/segments/" + defaultSegment.getSegmentName()).request()
+ .get(Response.class);
+ Assert.assertEquals(response.getStatus(), Response.Status.NOT_FOUND.getStatusCode());
+
+ response = _webTarget
+ .path("/tables/" + TableNameBuilder.REALTIME.tableNameWithType(TABLE_NAME) + "/segments/UNKNOWN_SEGMENT")
+ .request().get(Response.class);
+ Assert.assertEquals(response.getStatus(), Response.Status.NOT_FOUND.getStatusCode());
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org