You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@helix.apache.org by hu...@apache.org on 2020/04/01 22:47:05 UTC
[helix] 10/49: Add REST read endpoints to Helix Rest to provide
resource access to metadata store directory (#744)
This is an automated email from the ASF dual-hosted git repository.
hulee pushed a commit to branch zooscalability
in repository https://gitbox.apache.org/repos/asf/helix.git
commit 7c3c94637e2ab87005862be7e3be4d444e7dd131
Author: Huizhi Lu <hu...@linkedin.com>
AuthorDate: Tue Feb 11 18:09:25 2020 -0800
Add REST read endpoints to Helix Rest to provide resource access to metadata store directory (#744)
Change list:
- Add read endpoints to read sharding keys and realms
- Add 3 unit tests to test the new endpoints in TestMetadataStoreDirectoryAccessor
---
.../org/apache/helix/rest/common/ServletType.java | 7 +-
.../constant/MetadataStoreRoutingConstants.java | 11 ++
.../MetadataStoreDirectoryAccessor.java | 160 +++++++++++++++++++
.../TestMetadataStoreDirectoryAccessor.java | 175 +++++++++++++++++++++
4 files changed, 351 insertions(+), 2 deletions(-)
diff --git a/helix-rest/src/main/java/org/apache/helix/rest/common/ServletType.java b/helix-rest/src/main/java/org/apache/helix/rest/common/ServletType.java
index f068f95..7c77429 100644
--- a/helix-rest/src/main/java/org/apache/helix/rest/common/ServletType.java
+++ b/helix-rest/src/main/java/org/apache/helix/rest/common/ServletType.java
@@ -21,6 +21,7 @@ package org.apache.helix.rest.common;
import org.apache.helix.rest.server.resources.helix.AbstractHelixResource;
import org.apache.helix.rest.server.resources.metadata.NamespacesAccessor;
+import org.apache.helix.rest.server.resources.metadatastore.MetadataStoreDirectoryAccessor;
import org.apache.helix.rest.server.resources.zookeeper.ZooKeeperAccessor;
@@ -32,7 +33,8 @@ public enum ServletType {
new String[] {
AbstractHelixResource.class.getPackage().getName(),
NamespacesAccessor.class.getPackage().getName(),
- ZooKeeperAccessor.class.getPackage().getName()
+ ZooKeeperAccessor.class.getPackage().getName(),
+ MetadataStoreDirectoryAccessor.class.getPackage().getName()
}),
/**
@@ -41,7 +43,8 @@ public enum ServletType {
COMMON_SERVLET("/namespaces/%s/*",
new String[] {
AbstractHelixResource.class.getPackage().getName(),
- ZooKeeperAccessor.class.getPackage().getName()
+ ZooKeeperAccessor.class.getPackage().getName(),
+ MetadataStoreDirectoryAccessor.class.getPackage().getName()
});
private final String _servletPathSpecTemplate;
diff --git a/helix-rest/src/main/java/org/apache/helix/rest/metadatastore/constant/MetadataStoreRoutingConstants.java b/helix-rest/src/main/java/org/apache/helix/rest/metadatastore/constant/MetadataStoreRoutingConstants.java
index e4240e7..846aa30 100644
--- a/helix-rest/src/main/java/org/apache/helix/rest/metadatastore/constant/MetadataStoreRoutingConstants.java
+++ b/helix-rest/src/main/java/org/apache/helix/rest/metadatastore/constant/MetadataStoreRoutingConstants.java
@@ -27,4 +27,15 @@ public class MetadataStoreRoutingConstants {
// Leader election ZNode for ZkRoutingDataWriter
public static final String LEADER_ELECTION_ZNODE = "/_ZK_ROUTING_DATA_WRITER_LEADER";
+
+ // Field name in JSON REST response of getting metadata store realms in one namespace.
+ public static final String METADATA_STORE_REALMS = "metadataStoreRealms";
+
+ // Field name in JSON REST response of getting sharding keys in one realm.
+ public static final String SINGLE_METADATA_STORE_REALM = "metadataStoreRealm";
+
+ // Field name in JSON REST response of getting sharding keys.
+ public static final String SHARDING_KEYS = "shardingKeys";
+
+
}
diff --git a/helix-rest/src/main/java/org/apache/helix/rest/server/resources/metadatastore/MetadataStoreDirectoryAccessor.java b/helix-rest/src/main/java/org/apache/helix/rest/server/resources/metadatastore/MetadataStoreDirectoryAccessor.java
new file mode 100644
index 0000000..78543d6
--- /dev/null
+++ b/helix-rest/src/main/java/org/apache/helix/rest/server/resources/metadatastore/MetadataStoreDirectoryAccessor.java
@@ -0,0 +1,160 @@
+package org.apache.helix.rest.server.resources.metadatastore;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.NoSuchElementException;
+import javax.annotation.PostConstruct;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.Response;
+
+import com.google.common.collect.ImmutableMap;
+import org.apache.helix.rest.common.ContextPropertyKeys;
+import org.apache.helix.rest.common.HelixRestNamespace;
+import org.apache.helix.rest.common.HelixRestUtils;
+import org.apache.helix.rest.metadatastore.MetadataStoreDirectory;
+import org.apache.helix.rest.metadatastore.ZkMetadataStoreDirectory;
+import org.apache.helix.rest.metadatastore.constant.MetadataStoreRoutingConstants;
+import org.apache.helix.rest.metadatastore.exceptions.InvalidRoutingDataException;
+import org.apache.helix.rest.server.resources.AbstractResource;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * Provides REST endpoints for accessing metadata store directory service,
+ * which responds to read/write requests of metadata store realms, sharding keys, etc..
+ */
+@Path("")
+public class MetadataStoreDirectoryAccessor extends AbstractResource {
+ private static final Logger LOG = LoggerFactory.getLogger(MetadataStoreDirectoryAccessor.class);
+
+ private HelixRestNamespace _namespace;
+
+ // Double-checked locking for thread-safe object.
+ private MetadataStoreDirectory _metadataStoreDirectory;
+
+ @PostConstruct
+ private void postConstruct() {
+ getHelixNamespace();
+ buildMetadataStoreDirectory();
+ }
+
+ /**
+ * Gets all metadata store realms in a namespace with the endpoint.
+ *
+ * @return Json representation of all realms.
+ */
+ @GET
+ @Path("/metadata-store-realms")
+ public Response getAllMetadataStoreRealms() {
+ Map<String, Collection<String>> responseMap;
+ try {
+ Collection<String> realms =
+ _metadataStoreDirectory.getAllMetadataStoreRealms(_namespace.getName());
+
+ responseMap = new HashMap<>(1);
+ responseMap.put(MetadataStoreRoutingConstants.METADATA_STORE_REALMS, realms);
+ } catch (NoSuchElementException ex) {
+ return notFound(ex.getMessage());
+ }
+
+ return JSONRepresentation(responseMap);
+ }
+
+ /**
+ * Gets sharding keys mapped at path "HTTP GET /sharding-keys" which returns all sharding keys in
+ * a namespace, or path "HTTP GET /sharding-keys?realm={realmName}" which returns sharding keys in
+ * a realm.
+ *
+ * @param realm Query param in endpoint path
+ * @return Json representation of a map: shardingKeys -> collection of sharding keys.
+ */
+ @GET
+ @Path("/sharding-keys")
+ public Response getShardingKeys(@QueryParam("realm") String realm) {
+ Map<String, Object> responseMap;
+ Collection<String> shardingKeys;
+ try {
+ // If realm is not set in query param, the endpoint is: "/sharding-keys"
+ // to get all sharding keys in a namespace.
+ if (realm == null) {
+ shardingKeys =
+ _metadataStoreDirectory.getAllShardingKeys(_namespace.getName());
+ // To avoid allocating unnecessary resource, limit the map's capacity only for
+ // SHARDING_KEYS.
+ responseMap = new HashMap<>(1);
+ } else {
+ // For endpoint: "/sharding-keys?realm={realmName}"
+ shardingKeys = _metadataStoreDirectory
+ .getAllShardingKeysInRealm(_namespace.getName(), realm);
+ // To avoid allocating unnecessary resource, limit the map's capacity only for
+ // SHARDING_KEYS and SINGLE_METADATA_STORE_REALM.
+ responseMap = new HashMap<>(2);
+ responseMap.put(MetadataStoreRoutingConstants.SINGLE_METADATA_STORE_REALM, realm);
+ }
+ } catch (NoSuchElementException ex) {
+ return notFound(ex.getMessage());
+ }
+
+ responseMap.put(MetadataStoreRoutingConstants.SHARDING_KEYS, shardingKeys);
+
+ return JSONRepresentation(responseMap);
+ }
+
+ private void getHelixNamespace() {
+ // A default servlet does not have context property key METADATA, so the namespace
+ // is retrieved from property ALL_NAMESPACES.
+ if (HelixRestUtils.isDefaultServlet(_servletRequest.getServletPath())) {
+ // It is safe to ignore uncheck warnings for this cast.
+ @SuppressWarnings("unchecked")
+ List<HelixRestNamespace> namespaces = (List<HelixRestNamespace>) _application.getProperties()
+ .get(ContextPropertyKeys.ALL_NAMESPACES.name());
+ for (HelixRestNamespace ns : namespaces) {
+ if (HelixRestNamespace.DEFAULT_NAMESPACE_NAME.equals(ns.getName())) {
+ _namespace = ns;
+ break;
+ }
+ }
+ } else {
+ // Get namespace from property METADATA for a common servlet.
+ _namespace = (HelixRestNamespace) _application.getProperties()
+ .get(ContextPropertyKeys.METADATA.name());
+ }
+ }
+
+ private void buildMetadataStoreDirectory() {
+ Map<String, String> routingZkAddressMap =
+ ImmutableMap.of(_namespace.getName(), _namespace.getMetadataStoreAddress());
+ try {
+ _metadataStoreDirectory = new ZkMetadataStoreDirectory(routingZkAddressMap);
+ } catch (InvalidRoutingDataException ex) {
+ // In this case, the InvalidRoutingDataException should not happen because routing
+ // ZK address is always valid here.
+ LOG.warn("Unable to create metadata store directory for routing ZK address: {}",
+ routingZkAddressMap, ex);
+ }
+ }
+}
diff --git a/helix-rest/src/test/java/org/apache/helix/rest/server/resources/zookeeper/TestMetadataStoreDirectoryAccessor.java b/helix-rest/src/test/java/org/apache/helix/rest/server/resources/zookeeper/TestMetadataStoreDirectoryAccessor.java
new file mode 100644
index 0000000..7242c84
--- /dev/null
+++ b/helix-rest/src/test/java/org/apache/helix/rest/server/resources/zookeeper/TestMetadataStoreDirectoryAccessor.java
@@ -0,0 +1,175 @@
+package org.apache.helix.rest.server.resources.zookeeper;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import javax.ws.rs.core.Response;
+
+import com.google.common.collect.ImmutableSet;
+import org.apache.helix.ZNRecord;
+import org.apache.helix.manager.zk.ZNRecordSerializer;
+import org.apache.helix.rest.metadatastore.constant.MetadataStoreRoutingConstants;
+import org.apache.helix.rest.server.AbstractTestClass;
+import org.apache.helix.rest.server.util.JerseyUriRequestBuilder;
+import org.testng.Assert;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+
+public class TestMetadataStoreDirectoryAccessor extends AbstractTestClass {
+ /*
+ * The following are constants to be used for testing.
+ */
+ private static final String TEST_REALM_1 = "testRealm1";
+ private static final List<String> TEST_SHARDING_KEYS_1 =
+ Arrays.asList("/sharding/key/1/a", "/sharding/key/1/b", "/sharding/key/1/c");
+ private static final String TEST_REALM_2 = "testRealm2";
+ private static final List<String> TEST_SHARDING_KEYS_2 =
+ Arrays.asList("/sharding/key/1/d", "/sharding/key/1/e", "/sharding/key/1/f");
+
+ // List of all ZK addresses, each of which corresponds to a namespace/routing ZK
+ private List<String> _zkList;
+
+ @BeforeClass
+ public void beforeClass() {
+ _zkList = new ArrayList<>(ZK_SERVER_MAP.keySet());
+
+ // Write dummy mappings in ZK
+ // Create a node that represents a realm address and add 3 sharding keys to it
+ ZNRecord znRecord = new ZNRecord("RoutingInfo");
+
+ _zkList.forEach(zk -> {
+ ZK_SERVER_MAP.get(zk).getZkClient().setZkSerializer(new ZNRecordSerializer());
+ // Write first realm and sharding keys pair
+ znRecord.setListField(MetadataStoreRoutingConstants.ZNRECORD_LIST_FIELD_KEY,
+ TEST_SHARDING_KEYS_1);
+ ZK_SERVER_MAP.get(zk).getZkClient()
+ .createPersistent(MetadataStoreRoutingConstants.ROUTING_DATA_PATH + "/" + TEST_REALM_1,
+ true);
+ ZK_SERVER_MAP.get(zk).getZkClient()
+ .writeData(MetadataStoreRoutingConstants.ROUTING_DATA_PATH + "/" + TEST_REALM_1,
+ znRecord);
+
+ // Create another realm and sharding keys pair
+ znRecord.setListField(MetadataStoreRoutingConstants.ZNRECORD_LIST_FIELD_KEY,
+ TEST_SHARDING_KEYS_2);
+ ZK_SERVER_MAP.get(zk).getZkClient()
+ .createPersistent(MetadataStoreRoutingConstants.ROUTING_DATA_PATH + "/" + TEST_REALM_2,
+ true);
+ ZK_SERVER_MAP.get(zk).getZkClient()
+ .writeData(MetadataStoreRoutingConstants.ROUTING_DATA_PATH + "/" + TEST_REALM_2,
+ znRecord);
+ });
+ }
+
+ @Test
+ public void testGetAllMetadataStoreRealms() throws IOException {
+ String responseBody =
+ get("metadata-store-realms", null, Response.Status.OK.getStatusCode(), true);
+ // It is safe to cast the object and suppress warnings.
+ @SuppressWarnings("unchecked")
+ Map<String, Collection<String>> queriedRealmsMap =
+ OBJECT_MAPPER.readValue(responseBody, Map.class);
+
+ Assert.assertTrue(
+ queriedRealmsMap.containsKey(MetadataStoreRoutingConstants.METADATA_STORE_REALMS));
+
+ Set<String> queriedRealmsSet =
+ new HashSet<>(queriedRealmsMap.get(MetadataStoreRoutingConstants.METADATA_STORE_REALMS));
+ Set<String> expectedRealms = ImmutableSet.of(TEST_REALM_1, TEST_REALM_2);
+
+ Assert.assertEquals(queriedRealmsSet, expectedRealms);
+ }
+
+ /*
+ * Tests REST endpoints: "/sharding-keys"
+ */
+ @Test
+ public void testGetShardingKeysInNamespace() throws IOException {
+ String responseBody = get("/sharding-keys", null, Response.Status.OK.getStatusCode(), true);
+ // It is safe to cast the object and suppress warnings.
+ @SuppressWarnings("unchecked")
+ Map<String, Collection<String>> queriedShardingKeysMap =
+ OBJECT_MAPPER.readValue(responseBody, Map.class);
+
+ Assert.assertTrue(
+ queriedShardingKeysMap.containsKey(MetadataStoreRoutingConstants.SHARDING_KEYS));
+
+ Set<String> queriedShardingKeys =
+ new HashSet<>(queriedShardingKeysMap.get(MetadataStoreRoutingConstants.SHARDING_KEYS));
+ Set<String> expectedShardingKeys = new HashSet<>();
+ expectedShardingKeys.addAll(TEST_SHARDING_KEYS_1);
+ expectedShardingKeys.addAll(TEST_SHARDING_KEYS_2);
+
+ Assert.assertEquals(queriedShardingKeys, expectedShardingKeys);
+ }
+
+ /*
+ * Tests REST endpoint: "/sharding-keys?realm={realmName}"
+ */
+ @Test
+ public void testGetShardingKeysInRealm() throws IOException {
+ // Test NOT_FOUND response for a non existed realm.
+ new JerseyUriRequestBuilder("/sharding-keys?realm=nonExistedRealm")
+ .expectedReturnStatusCode(Response.Status.NOT_FOUND.getStatusCode()).get(this);
+
+ // Query param realm is set to empty, so NOT_FOUND response is returned.
+ new JerseyUriRequestBuilder("/sharding-keys?realm=")
+ .expectedReturnStatusCode(Response.Status.NOT_FOUND.getStatusCode()).get(this);
+
+ // Success response.
+ String responseBody = new JerseyUriRequestBuilder("/sharding-keys?realm=" + TEST_REALM_1)
+ .isBodyReturnExpected(true).get(this);
+ // It is safe to cast the object and suppress warnings.
+ @SuppressWarnings("unchecked")
+ Map<String, Object> queriedShardingKeysMap = OBJECT_MAPPER.readValue(responseBody, Map.class);
+
+ // Check realm name in json response.
+ Assert.assertTrue(queriedShardingKeysMap
+ .containsKey(MetadataStoreRoutingConstants.SINGLE_METADATA_STORE_REALM));
+ Assert.assertEquals(
+ queriedShardingKeysMap.get(MetadataStoreRoutingConstants.SINGLE_METADATA_STORE_REALM),
+ TEST_REALM_1);
+ Assert.assertTrue(
+ queriedShardingKeysMap.containsKey(MetadataStoreRoutingConstants.SHARDING_KEYS));
+
+ // It is safe to cast the object and suppress warnings.
+ @SuppressWarnings("unchecked")
+ Set<String> queriedShardingKeys = new HashSet<>((Collection<String>) queriedShardingKeysMap
+ .get(MetadataStoreRoutingConstants.SHARDING_KEYS));
+ Set<String> expectedShardingKeys = new HashSet<>(TEST_SHARDING_KEYS_1);
+
+ Assert.assertEquals(queriedShardingKeys, expectedShardingKeys);
+ }
+
+ @AfterClass
+ public void afterClass() {
+ _zkList.forEach(zk -> ZK_SERVER_MAP.get(zk).getZkClient()
+ .deleteRecursive(MetadataStoreRoutingConstants.ROUTING_DATA_PATH));
+ }
+}