You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ro...@apache.org on 2022/06/30 02:51:20 UTC
[pinot] branch master updated: adding endpoint to put all ZNRecords to a particular parent path (#8994)
This is an automated email from the ASF dual-hosted git repository.
rongr pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new d2031e9479 adding endpoint to put all ZNRecords to a particular parent path (#8994)
d2031e9479 is described below
commit d2031e9479f8b56326ee025e9ac5f549d9665327
Author: Rong Rong <wa...@gmail.com>
AuthorDate: Wed Jun 29 19:51:13 2022 -0700
adding endpoint to put all ZNRecords to a particular parent path (#8994)
* adding endpoint to put all ZNRecords to a particular parent path
Co-authored-by: Rong Rong <ro...@startree.ai>
---
.../api/resources/ZookeeperResource.java | 55 ++++++++++++++++++++++
.../api/resources/ZookeeperResourceTest.java | 28 +++++++++--
.../utils/builder/ControllerRequestURLBuilder.java | 4 ++
3 files changed, 82 insertions(+), 5 deletions(-)
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/ZookeeperResource.java b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/ZookeeperResource.java
index 13d4560c06..7d033de212 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/ZookeeperResource.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/ZookeeperResource.java
@@ -57,6 +57,7 @@ import org.apache.zookeeper.data.Stat;
import org.codehaus.jackson.map.DeserializationConfig;
import org.codehaus.jackson.map.ObjectMapper;
import org.codehaus.jackson.map.SerializationConfig;
+import org.codehaus.jackson.type.TypeReference;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -140,6 +141,60 @@ public class ZookeeperResource {
}
}
+ @PUT
+ @Path("/zk/putChildren")
+ @Authenticate(AccessType.UPDATE)
+ @Produces(MediaType.APPLICATION_JSON)
+ @Consumes(MediaType.APPLICATION_JSON)
+ @ApiOperation(value = "Update the content of multiple znRecord node under the same path")
+ @ApiResponses(value = {
+ @ApiResponse(code = 200, message = "Success"),
+ @ApiResponse(code = 404, message = "ZK Path not found"),
+ @ApiResponse(code = 204, message = "No Content"),
+ @ApiResponse(code = 500, message = "Internal server error")
+ })
+ public SuccessResponse putChildren(
+ @ApiParam(value = "Zookeeper path of parent, must start with /", required = true) @QueryParam("path") String path,
+ @ApiParam(value = "Content") @QueryParam("data") @Nullable String data,
+ @ApiParam(value = "expectedVersion", defaultValue = "-1") @QueryParam("expectedVersion") @DefaultValue("-1")
+ int expectedVersion,
+ @ApiParam(value = "accessOption", defaultValue = "1") @QueryParam("accessOption") @DefaultValue("1")
+ int accessOption,
+ @Nullable String payload) {
+
+ path = validateAndNormalizeZKPath(path, false);
+
+ if (StringUtils.isEmpty(data)) {
+ data = payload;
+ }
+ if (StringUtils.isEmpty(data)) {
+ throw new ControllerApplicationException(LOGGER, "Must provide data through query parameter or payload",
+ Response.Status.BAD_REQUEST);
+ }
+ List<ZNRecord> znRecords;
+ try {
+ znRecords = MAPPER.readValue(data, new TypeReference<List<ZNRecord>>() { });
+ } catch (Exception e) {
+ throw new ControllerApplicationException(LOGGER, "Failed to deserialize the data", Response.Status.BAD_REQUEST,
+ e);
+ }
+
+ for (ZNRecord znRecord : znRecords) {
+ String childPath = path + "/" + znRecord.getId();
+ try {
+ boolean result = _pinotHelixResourceManager.setZKData(childPath, znRecord, expectedVersion, accessOption);
+ if (!result) {
+ throw new ControllerApplicationException(LOGGER, "Failed to update path: " + childPath,
+ Response.Status.INTERNAL_SERVER_ERROR);
+ }
+ } catch (Exception e) {
+ throw new ControllerApplicationException(LOGGER, "Failed to update path: " + childPath,
+ Response.Status.INTERNAL_SERVER_ERROR, e);
+ }
+ }
+ return new SuccessResponse("Successfully updated " + znRecords.size() + " ZnRecords under path: " + path);
+ }
+
@PUT
@Path("/zk/put")
@Authenticate(AccessType.UPDATE)
diff --git a/pinot-controller/src/test/java/org/apache/pinot/controller/api/resources/ZookeeperResourceTest.java b/pinot-controller/src/test/java/org/apache/pinot/controller/api/resources/ZookeeperResourceTest.java
index b2e557dbe3..80d17ab993 100644
--- a/pinot-controller/src/test/java/org/apache/pinot/controller/api/resources/ZookeeperResourceTest.java
+++ b/pinot-controller/src/test/java/org/apache/pinot/controller/api/resources/ZookeeperResourceTest.java
@@ -71,10 +71,10 @@ public class ZookeeperResourceTest {
urlGet = TEST_INSTANCE.getControllerRequestURLBuilder().forZkGetChildren(path);
result = ControllerTest.sendGetRequest(urlGet);
- List<ZNRecord> recordList = ZookeeperResource.MAPPER.readValue(result.getBytes(StandardCharsets.UTF_8),
+ List<ZNRecord> recordList1 = ZookeeperResource.MAPPER.readValue(result.getBytes(StandardCharsets.UTF_8),
new TypeReference<List<ZNRecord>>() { });
- Assert.assertEquals(recordList.size(), 1);
- Assert.assertEquals(recordList.get(0), znRecord);
+ Assert.assertEquals(recordList1.size(), 1);
+ Assert.assertEquals(recordList1.get(0), znRecord);
String lorem = "Loremipsumdolorsitametconsecteturadipisicingelitseddoeiusmod"
+ "temporincididuntutlaboreetdoloremagnaaliquaUtenimadminimveniam"
@@ -112,8 +112,26 @@ public class ZookeeperResourceTest {
urlGet = TEST_INSTANCE.getControllerRequestURLBuilder().forZkGetChildren(path);
result = ControllerTest.sendGetRequest(urlGet);
- recordList = ZookeeperResource.MAPPER.readValue(result.getBytes(StandardCharsets.UTF_8),
+ List<ZNRecord> recordList3 = ZookeeperResource.MAPPER.readValue(result.getBytes(StandardCharsets.UTF_8),
new TypeReference<List<ZNRecord>>() { });
- Assert.assertEquals(recordList.size(), 2);
+ Assert.assertEquals(recordList3.size(), 2);
+
+ // CASE 4: put all children back into a different path
+ String path4 = path + "/testCase4";
+ params = "path=" + path4 + "&expectedVersion=" + expectedVersion + "&accessOption=" + accessOption;
+
+ // validate that zk/putChildren will insert all correctly to another path
+ urlPut = TEST_INSTANCE.getControllerRequestURLBuilder().forZkPutChildren(path);
+ String encodedChildrenData = ZookeeperResource.MAPPER.writeValueAsString(recordList3);
+ result = ControllerTest.sendPutRequest(urlPut + "?" + params, encodedChildrenData);
+
+ // validate that zk/getChildren from new path should result in the same recordList
+ urlGet = TEST_INSTANCE.getControllerRequestURLBuilder().forZkGetChildren(path);
+ result = ControllerTest.sendGetRequest(urlGet);
+
+ List<ZNRecord> recordList4 = ZookeeperResource.MAPPER.readValue(result.getBytes(StandardCharsets.UTF_8),
+ new TypeReference<List<ZNRecord>>() { });
+ Assert.assertEquals(recordList4.get(0), recordList3.get(0));
+ Assert.assertEquals(recordList4.get(1), recordList3.get(1));
}
}
diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/builder/ControllerRequestURLBuilder.java b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/builder/ControllerRequestURLBuilder.java
index b0a9f7e139..c74be58e68 100644
--- a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/builder/ControllerRequestURLBuilder.java
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/builder/ControllerRequestURLBuilder.java
@@ -438,6 +438,10 @@ public class ControllerRequestURLBuilder {
return StringUtil.join("/", _baseUrl, "zk/put");
}
+ public String forZkPutChildren(String path) {
+ return StringUtil.join("/", _baseUrl, "zk/putChildren", "?path=" + path);
+ }
+
public String forZkGet(String path) {
return StringUtil.join("/", _baseUrl, "zk/get", "?path=" + path);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org