You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@helix.apache.org by hu...@apache.org on 2020/04/08 22:53:49 UTC
[helix] 19/50: Add REST read endpoints to helix-rest for metadata
store directory (#761)
This is an automated email from the ASF dual-hosted git repository.
hulee pushed a commit to branch zooscalability_merge
in repository https://gitbox.apache.org/repos/asf/helix.git
commit 8f486b4813581797bf38eb4c1f18d63ceb974c5d
Author: Huizhi Lu <ih...@gmail.com>
AuthorDate: Sat Feb 22 18:12:53 2020 -0800
Add REST read endpoints to helix-rest for metadata store directory (#761)
We need restful metadata store directory service to help scale out zookeeper.
This commit adds REST read endpoints to helix-rest to get sharding keys, realms and namespaces.
---
.../datamodel/MetadataStoreShardingKey.java | 61 +++++
.../MetadataStoreDirectoryAccessor.java | 172 +++++++++++---
.../server/TestMetadataStoreDirectoryAccessor.java | 255 +++++++++++++++++----
.../constant/MetadataStoreRoutingConstants.java | 33 ++-
4 files changed, 442 insertions(+), 79 deletions(-)
diff --git a/helix-rest/src/main/java/org/apache/helix/rest/metadatastore/datamodel/MetadataStoreShardingKey.java b/helix-rest/src/main/java/org/apache/helix/rest/metadatastore/datamodel/MetadataStoreShardingKey.java
new file mode 100644
index 0000000..ac5ca59
--- /dev/null
+++ b/helix-rest/src/main/java/org/apache/helix/rest/metadatastore/datamodel/MetadataStoreShardingKey.java
@@ -0,0 +1,61 @@
+package org.apache.helix.rest.metadatastore.datamodel;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+
+/**
+ * A POJO class that represents a sharding key can be easily converted to JSON
+ * in REST API response. The JSON object for a sharding key looks like:
+ * {
+ * "shardingKey": "/sharding/key/10/abc",
+ * "realm": "realm.github.com"
+ * }
+ */
+@JsonAutoDetect
+public class MetadataStoreShardingKey {
+ private String shardingKey;
+ private String realm;
+
+ @JsonCreator
+ public MetadataStoreShardingKey(@JsonProperty String shardingKey, @JsonProperty String realm) {
+ this.shardingKey = shardingKey;
+ this.realm = realm;
+ }
+
+ @JsonProperty
+ public String getShardingKey() {
+ return shardingKey;
+ }
+
+ @JsonProperty
+ public String getRealm() {
+ return realm;
+ }
+
+ @Override
+ public String toString() {
+ return "MetadataStoreShardingKey{" + "shardingKey='" + shardingKey + '\'' + ", realm='" + realm
+ + '\'' + '}';
+ }
+}
diff --git a/helix-rest/src/main/java/org/apache/helix/rest/server/resources/metadatastore/MetadataStoreDirectoryAccessor.java b/helix-rest/src/main/java/org/apache/helix/rest/server/resources/metadatastore/MetadataStoreDirectoryAccessor.java
index 5d84d8a..bc6571a 100644
--- a/helix-rest/src/main/java/org/apache/helix/rest/server/resources/metadatastore/MetadataStoreDirectoryAccessor.java
+++ b/helix-rest/src/main/java/org/apache/helix/rest/server/resources/metadatastore/MetadataStoreDirectoryAccessor.java
@@ -20,11 +20,12 @@ package org.apache.helix.rest.server.resources.metadatastore;
*/
import java.util.Collection;
-import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
+import java.util.stream.Collectors;
import javax.annotation.PostConstruct;
+import javax.annotation.PreDestroy;
import javax.ws.rs.DELETE;
import javax.ws.rs.GET;
import javax.ws.rs.PUT;
@@ -41,6 +42,7 @@ import org.apache.helix.rest.common.HelixRestNamespace;
import org.apache.helix.rest.common.HelixRestUtils;
import org.apache.helix.rest.metadatastore.MetadataStoreDirectory;
import org.apache.helix.rest.metadatastore.ZkMetadataStoreDirectory;
+import org.apache.helix.rest.metadatastore.datamodel.MetadataStoreShardingKey;
import org.apache.helix.rest.server.resources.AbstractResource;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -65,25 +67,53 @@ public class MetadataStoreDirectoryAccessor extends AbstractResource {
buildMetadataStoreDirectory(_namespace, helixRestNamespace.getMetadataStoreAddress());
}
+ @PreDestroy
+ private void preDestroy() {
+ _metadataStoreDirectory.close();
+ }
+
/**
- * Gets all metadata store realms in a namespace with the endpoint.
+ * Gets all existing namespaces in the routing metadata store at endpoint:
+ * "GET /metadata-store-namespaces"
+ *
+ * @return Json response of all namespaces.
+ */
+ @GET
+ @Path("/metadata-store-namespaces")
+ public Response getAllNamespaces() {
+ Collection<String> namespaces = _metadataStoreDirectory.getAllNamespaces();
+ Map<String, Collection<String>> responseMap =
+ ImmutableMap.of(MetadataStoreRoutingConstants.METADATA_STORE_NAMESPACES, namespaces);
+
+ return JSONRepresentation(responseMap);
+ }
+
+ /**
+ * Gets all metadata store realms in a namespace at path: "GET /metadata-store-realms",
+ * or gets a metadata store realm with the sharding key at path:
+ * "GET /metadata-store-realms?sharding-key={sharding-key}"
*
* @return Json representation of all realms.
*/
@GET
@Path("/metadata-store-realms")
- public Response getAllMetadataStoreRealms() {
- Map<String, Collection<String>> responseMap;
+ public Response getAllMetadataStoreRealms(@QueryParam("sharding-key") String shardingKey) {
try {
- Collection<String> realms = _metadataStoreDirectory.getAllMetadataStoreRealms(_namespace);
+ if (shardingKey == null) {
+ // Get all realms: "GET /metadata-store-realms"
+ Collection<String> realms = _metadataStoreDirectory.getAllMetadataStoreRealms(_namespace);
+ Map<String, Collection<String>> responseMap =
+ ImmutableMap.of(MetadataStoreRoutingConstants.METADATA_STORE_REALMS, realms);
+ return JSONRepresentation(responseMap);
+ }
- responseMap = new HashMap<>(1);
- responseMap.put(MetadataStoreRoutingConstants.METADATA_STORE_REALMS, realms);
+ // Get a single realm filtered by sharding key:
+ // "GET /metadata-store-realms?sharding-key={sharding-key}"
+ String realm = _metadataStoreDirectory.getMetadataStoreRealm(_namespace, shardingKey);
+ return JSONRepresentation(new MetadataStoreShardingKey(shardingKey, realm));
} catch (NoSuchElementException ex) {
return notFound(ex.getMessage());
}
-
- return JSONRepresentation(responseMap);
}
@PUT
@@ -111,41 +141,70 @@ public class MetadataStoreDirectoryAccessor extends AbstractResource {
}
/**
- * Gets sharding keys mapped at path "HTTP GET /sharding-keys" which returns all sharding keys in
- * a namespace, or path "HTTP GET /sharding-keys?realm={realmName}" which returns sharding keys in
- * a realm.
+ * Gets all sharding keys for following requests:
+ * - "HTTP GET /sharding-keys" which returns all sharding keys in a namespace.
+ * - "HTTP GET /sharding-keys?prefix={prefix}" which returns sharding keys that have the prefix.
+ * -- JSON response example for this path:
+ * {
+ * "prefix": "/sharding/key",
+ * "shardingKeys": [{
+ * "realm": "testRealm2",
+ * "shardingKey": "/sharding/key/1/f"
+ * }, {
+ * "realm": "testRealm2",
+ * "shardingKey": "/sharding/key/1/e"
+ * }, {
+ * "realm": "testRealm1",
+ * "shardingKey": "/sharding/key/1/b"
+ * }, {
+ * "realm": "testRealm1",
+ * "shardingKey": "/sharding/key/1/a"
+ * }]
+ * }
*
- * @param realm Query param in endpoint path
- * @return Json representation of a map: shardingKeys -> collection of sharding keys.
+ * @param prefix Query param in endpoint path: prefix substring of sharding key.
+ * @return Json representation for the sharding keys.
*/
@GET
@Path("/sharding-keys")
- public Response getShardingKeys(@QueryParam("realm") String realm) {
- Map<String, Object> responseMap;
- Collection<String> shardingKeys;
+ public Response getShardingKeys(@QueryParam("prefix") String prefix) {
try {
- // If realm is not set in query param, the endpoint is: "/sharding-keys"
- // to get all sharding keys in a namespace.
- if (realm == null) {
- shardingKeys = _metadataStoreDirectory.getAllShardingKeys(_namespace);
- // To avoid allocating unnecessary resource, limit the map's capacity only for
- // SHARDING_KEYS.
- responseMap = new HashMap<>(1);
- } else {
- // For endpoint: "/sharding-keys?realm={realmName}"
- shardingKeys = _metadataStoreDirectory.getAllShardingKeysInRealm(_namespace, realm);
- // To avoid allocating unnecessary resource, limit the map's capacity only for
- // SHARDING_KEYS and SINGLE_METADATA_STORE_REALM.
- responseMap = new HashMap<>(2);
- responseMap.put(MetadataStoreRoutingConstants.SINGLE_METADATA_STORE_REALM, realm);
+ if (prefix == null) {
+ // For endpoint: "/sharding-keys" to get all sharding keys in a namespace.
+ return getAllShardingKeys();
}
+ // For endpoint: "/sharding-keys?prefix={prefix}"
+ return getAllShardingKeysUnderPath(prefix);
} catch (NoSuchElementException ex) {
return notFound(ex.getMessage());
}
+ }
- responseMap.put(MetadataStoreRoutingConstants.SHARDING_KEYS, shardingKeys);
+ /**
+ * Gets all path-based sharding keys for a queried realm at endpoint:
+ * "GET /metadata-store-realms/{realm}/sharding-keys"
+ * <p>
+ * "GET /metadata-store-realms/{realm}/sharding-keys?prefix={prefix}" is also supported,
+ * which is helpful when you want to check what sharding keys have the prefix substring.
+ *
+ * @param realm Queried metadata store realm to get sharding keys.
+ * @param prefix Query param in endpoint path: prefix substring of sharding key.
+ * @return All path-based sharding keys in the queried realm.
+ */
+ @GET
+ @Path("/metadata-store-realms/{realm}/sharding-keys")
+ public Response getRealmShardingKeys(@PathParam("realm") String realm,
+ @QueryParam("prefix") String prefix) {
+ try {
+ if (prefix == null) {
+ return getAllShardingKeysInRealm(realm);
+ }
- return JSONRepresentation(responseMap);
+ // For "GET /metadata-store-realms/{realm}/sharding-keys?prefix={prefix}"
+ return getRealmShardingKeysUnderPath(realm, prefix);
+ } catch (NoSuchElementException ex) {
+ return notFound(ex.getMessage());
+ }
}
@PUT
@@ -209,4 +268,51 @@ public class MetadataStoreDirectoryAccessor extends AbstractResource {
routingZkAddressMap, ex);
}
}
+
+ private Response getAllShardingKeys() {
+ Collection<String> shardingKeys = _metadataStoreDirectory.getAllShardingKeys(_namespace);
+ Map<String, Object> responseMap = ImmutableMap
+ .of(MetadataStoreRoutingConstants.SINGLE_METADATA_STORE_NAMESPACE, _namespace,
+ MetadataStoreRoutingConstants.SHARDING_KEYS, shardingKeys);
+
+ return JSONRepresentation(responseMap);
+ }
+
+ private Response getAllShardingKeysInRealm(String realm) {
+ Collection<String> shardingKeys =
+ _metadataStoreDirectory.getAllShardingKeysInRealm(_namespace, realm);
+
+ Map<String, Object> responseMap = ImmutableMap
+ .of(MetadataStoreRoutingConstants.SINGLE_METADATA_STORE_REALM, realm,
+ MetadataStoreRoutingConstants.SHARDING_KEYS, shardingKeys);
+
+ return JSONRepresentation(responseMap);
+ }
+
+ private Response getAllShardingKeysUnderPath(String prefix) {
+ List<MetadataStoreShardingKey> shardingKeyList =
+ _metadataStoreDirectory.getAllMappingUnderPath(_namespace, prefix).entrySet().stream()
+ .map(entry -> new MetadataStoreShardingKey(entry.getKey(), entry.getValue()))
+ .collect(Collectors.toList());
+
+ Map<String, Object> responseMap = ImmutableMap
+ .of(MetadataStoreRoutingConstants.SHARDING_KEY_PATH_PREFIX, prefix,
+ MetadataStoreRoutingConstants.SHARDING_KEYS, shardingKeyList);
+
+ return JSONRepresentation(responseMap);
+ }
+
+ private Response getRealmShardingKeysUnderPath(String realm, String prefix) {
+ List<String> shardingKeyList =
+ _metadataStoreDirectory.getAllMappingUnderPath(_namespace, prefix).entrySet().stream()
+ .filter(entry -> entry.getValue().equals(realm)).map(Map.Entry::getKey)
+ .collect(Collectors.toList());
+
+ Map<String, Object> responseMap = ImmutableMap
+ .of(MetadataStoreRoutingConstants.SHARDING_KEY_PATH_PREFIX, prefix,
+ MetadataStoreRoutingConstants.SINGLE_METADATA_STORE_REALM, realm,
+ MetadataStoreRoutingConstants.SHARDING_KEYS, shardingKeyList);
+
+ return JSONRepresentation(responseMap);
+ }
}
diff --git a/helix-rest/src/test/java/org/apache/helix/rest/server/TestMetadataStoreDirectoryAccessor.java b/helix-rest/src/test/java/org/apache/helix/rest/server/TestMetadataStoreDirectoryAccessor.java
index 27e2b10..6a9c598 100644
--- a/helix-rest/src/test/java/org/apache/helix/rest/server/TestMetadataStoreDirectoryAccessor.java
+++ b/helix-rest/src/test/java/org/apache/helix/rest/server/TestMetadataStoreDirectoryAccessor.java
@@ -47,6 +47,7 @@ import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
+// TODO: enable asserts and add verify for refreshed MSD once write operations are ready.
public class TestMetadataStoreDirectoryAccessor extends AbstractTestClass {
/*
* The following are constants to be used for testing.
@@ -61,15 +62,14 @@ public class TestMetadataStoreDirectoryAccessor extends AbstractTestClass {
private static final List<String> TEST_SHARDING_KEYS_2 =
Arrays.asList("/sharding/key/1/d", "/sharding/key/1/e", "/sharding/key/1/f");
private static final String TEST_REALM_3 = "testRealm3";
- private static final String TEST_SHARDING_KEY = "/sharding/key/1/x";
+ private static final String TEST_SHARDING_KEY = "/sharding/key/3/x";
// List of all ZK addresses, each of which corresponds to a namespace/routing ZK
private List<String> _zkList;
private MetadataStoreDirectory _metadataStoreDirectory;
@BeforeClass
- public void beforeClass()
- throws Exception {
+ public void beforeClass() throws Exception {
_zkList = new ArrayList<>(ZK_SERVER_MAP.keySet());
deleteRoutingDataPath();
@@ -111,15 +111,39 @@ public class TestMetadataStoreDirectoryAccessor extends AbstractTestClass {
}
@AfterClass
- public void afterClass()
- throws Exception {
+ public void afterClass() throws Exception {
_metadataStoreDirectory.close();
deleteRoutingDataPath();
}
+ /*
+ * Tests REST endpoint: "GET /namespaces/{namespace}/metadata-store-namespaces"
+ */
@Test
- public void testGetAllMetadataStoreRealms()
- throws IOException {
+ public void testGetAllNamespaces() throws IOException {
+ String responseBody = get(TEST_NAMESPACE_URI_PREFIX + "/metadata-store-namespaces", null,
+ Response.Status.OK.getStatusCode(), true);
+
+ // It is safe to cast the object and suppress warnings.
+ @SuppressWarnings("unchecked")
+ Map<String, Collection<String>> queriedNamespacesMap =
+ OBJECT_MAPPER.readValue(responseBody, Map.class);
+
+ Assert.assertEquals(queriedNamespacesMap.keySet(),
+ ImmutableSet.of(MetadataStoreRoutingConstants.METADATA_STORE_NAMESPACES));
+
+ Set<String> queriedNamespacesSet = new HashSet<>(
+ queriedNamespacesMap.get(MetadataStoreRoutingConstants.METADATA_STORE_NAMESPACES));
+ Set<String> expectedNamespaces = ImmutableSet.of(TEST_NAMESPACE);
+
+ Assert.assertEquals(queriedNamespacesSet, expectedNamespaces);
+ }
+
+ /*
+ * Tests REST endpoint: "GET /metadata-store-realms"
+ */
+ @Test(dependsOnMethods = "testGetAllNamespaces")
+ public void testGetAllMetadataStoreRealms() throws IOException {
get(NON_EXISTING_NAMESPACE_URI_PREFIX + "metadata-store-realms", null,
Response.Status.NOT_FOUND.getStatusCode(), false);
@@ -130,8 +154,8 @@ public class TestMetadataStoreDirectoryAccessor extends AbstractTestClass {
Map<String, Collection<String>> queriedRealmsMap =
OBJECT_MAPPER.readValue(responseBody, Map.class);
- Assert.assertTrue(
- queriedRealmsMap.containsKey(MetadataStoreRoutingConstants.METADATA_STORE_REALMS));
+ Assert.assertEquals(queriedRealmsMap.keySet(),
+ ImmutableSet.of(MetadataStoreRoutingConstants.METADATA_STORE_REALMS));
Set<String> queriedRealmsSet =
new HashSet<>(queriedRealmsMap.get(MetadataStoreRoutingConstants.METADATA_STORE_REALMS));
@@ -140,7 +164,36 @@ public class TestMetadataStoreDirectoryAccessor extends AbstractTestClass {
Assert.assertEquals(queriedRealmsSet, expectedRealms);
}
+ /*
+ * Tests REST endpoint: "GET /metadata-store-realms?sharding-key={sharding-key}"
+ */
@Test(dependsOnMethods = "testGetAllMetadataStoreRealms")
+ public void testGetMetadataStoreRealmWithShardingKey() throws IOException {
+ String shardingKey = TEST_SHARDING_KEYS_1.get(0);
+
+ new JerseyUriRequestBuilder(
+ NON_EXISTING_NAMESPACE_URI_PREFIX + "metadata-store-realms?sharding-key=" + shardingKey)
+ .expectedReturnStatusCode(Response.Status.NOT_FOUND.getStatusCode()).get(this);
+
+ String responseBody = new JerseyUriRequestBuilder(
+ TEST_NAMESPACE_URI_PREFIX + "/metadata-store-realms?sharding-key=" + shardingKey)
+ .isBodyReturnExpected(true).get(this);
+
+ // It is safe to cast the object and suppress warnings.
+ @SuppressWarnings("unchecked")
+ Map<String, String> queriedRealmMap = OBJECT_MAPPER.readValue(responseBody, Map.class);
+
+ Map<String, String> expectedRealm = ImmutableMap
+ .of(MetadataStoreRoutingConstants.SINGLE_METADATA_STORE_REALM, TEST_REALM_1,
+ MetadataStoreRoutingConstants.SINGLE_SHARDING_KEY, shardingKey);
+
+ Assert.assertEquals(queriedRealmMap, expectedRealm);
+ }
+
+ /*
+ * Tests REST endpoint: "PUT /metadata-store-realms/{realm}"
+ */
+ @Test(dependsOnMethods = "testGetMetadataStoreRealmWithShardingKey")
public void testAddMetadataStoreRealm() {
Collection<String> previousRealms =
_metadataStoreDirectory.getAllMetadataStoreRealms(TEST_NAMESPACE);
@@ -164,10 +217,12 @@ public class TestMetadataStoreDirectoryAccessor extends AbstractTestClass {
Set<String> updateRealmsSet = new HashSet<>(updatedRealms);
expectedRealmsSet.add(TEST_REALM_3);
- // TODO: enable asserts and add verify for refreshed MSD once write operations are ready.
// Assert.assertEquals(updateRealmsSet, previousRealms);
}
+ /*
+ * Tests REST endpoint: "DELETE /metadata-store-realms/{realm}"
+ */
@Test(dependsOnMethods = "testAddMetadataStoreRealm")
public void testDeleteMetadataStoreRealm() {
Collection<String> previousRealms =
@@ -194,11 +249,10 @@ public class TestMetadataStoreDirectoryAccessor extends AbstractTestClass {
}
/*
- * Tests REST endpoints: "/sharding-keys"
+ * Tests REST endpoint: "GET /sharding-keys"
*/
@Test(dependsOnMethods = "testDeleteMetadataStoreRealm")
- public void testGetShardingKeysInNamespace()
- throws IOException {
+ public void testGetShardingKeysInNamespace() throws IOException {
get(NON_EXISTING_NAMESPACE_URI_PREFIX + "sharding-keys", null,
Response.Status.NOT_FOUND.getStatusCode(), true);
@@ -207,14 +261,19 @@ public class TestMetadataStoreDirectoryAccessor extends AbstractTestClass {
true);
// It is safe to cast the object and suppress warnings.
@SuppressWarnings("unchecked")
- Map<String, Collection<String>> queriedShardingKeysMap =
- OBJECT_MAPPER.readValue(responseBody, Map.class);
+ Map<String, Object> queriedShardingKeysMap = OBJECT_MAPPER.readValue(responseBody, Map.class);
+
+ Assert.assertEquals(queriedShardingKeysMap.keySet(), ImmutableSet
+ .of(MetadataStoreRoutingConstants.SINGLE_METADATA_STORE_NAMESPACE,
+ MetadataStoreRoutingConstants.SHARDING_KEYS));
- Assert.assertTrue(
- queriedShardingKeysMap.containsKey(MetadataStoreRoutingConstants.SHARDING_KEYS));
+ Assert.assertEquals(
+ queriedShardingKeysMap.get(MetadataStoreRoutingConstants.SINGLE_METADATA_STORE_NAMESPACE),
+ TEST_NAMESPACE);
- Set<String> queriedShardingKeys =
- new HashSet<>(queriedShardingKeysMap.get(MetadataStoreRoutingConstants.SHARDING_KEYS));
+ @SuppressWarnings("unchecked")
+ Set<String> queriedShardingKeys = new HashSet<>((Collection<String>) queriedShardingKeysMap
+ .get(MetadataStoreRoutingConstants.SHARDING_KEYS));
Set<String> expectedShardingKeys = new HashSet<>();
expectedShardingKeys.addAll(TEST_SHARDING_KEYS_1);
expectedShardingKeys.addAll(TEST_SHARDING_KEYS_2);
@@ -223,38 +282,125 @@ public class TestMetadataStoreDirectoryAccessor extends AbstractTestClass {
}
/*
- * Tests REST endpoint: "/sharding-keys?realm={realmName}"
+ * Tests REST endpoint: "GET /metadata-store-realms/{realm}/sharding-keys"
*/
@Test(dependsOnMethods = "testGetShardingKeysInNamespace")
- public void testGetShardingKeysInRealm()
- throws IOException {
+ public void testGetShardingKeysInRealm() throws IOException {
// Test NOT_FOUND response for a non existed realm.
- new JerseyUriRequestBuilder(TEST_NAMESPACE_URI_PREFIX + "/sharding-keys?realm=nonExistedRealm")
+ new JerseyUriRequestBuilder(
+ TEST_NAMESPACE_URI_PREFIX + "/metadata-store-realms/nonExistedRealm/sharding-keys")
.expectedReturnStatusCode(Response.Status.NOT_FOUND.getStatusCode()).get(this);
- // Query param realm is set to empty, so NOT_FOUND response is returned.
- new JerseyUriRequestBuilder(TEST_NAMESPACE_URI_PREFIX + "/sharding-keys?realm=")
- .expectedReturnStatusCode(Response.Status.NOT_FOUND.getStatusCode()).get(this);
+ // Success response for "GET /metadata-store-realms/{realm}/sharding-keys"
+ String responseBody = new JerseyUriRequestBuilder(
+ TEST_NAMESPACE_URI_PREFIX + "/metadata-store-realms/" + TEST_REALM_1 + "/sharding-keys")
+ .isBodyReturnExpected(true).get(this);
+
+ verifyRealmShardingKeys(responseBody);
+ }
- // Success response.
+ /*
+ * Tests REST endpoint: "GET /sharding-keys?prefix={prefix}"
+ */
+ @SuppressWarnings("unchecked")
+ @Test(dependsOnMethods = "testGetShardingKeysInRealm")
+ public void testGetShardingKeysUnderPath() throws IOException {
+ // Test non existed prefix and empty sharding keys in response.
String responseBody = new JerseyUriRequestBuilder(
- TEST_NAMESPACE_URI_PREFIX + "/sharding-keys?realm=" + TEST_REALM_1)
+ TEST_NAMESPACE_URI_PREFIX + "/sharding-keys?prefix=/non/Existed/Prefix")
.isBodyReturnExpected(true).get(this);
- // It is safe to cast the object and suppress warnings.
- @SuppressWarnings("unchecked")
+
Map<String, Object> queriedShardingKeysMap = OBJECT_MAPPER.readValue(responseBody, Map.class);
+ Collection<Map<String, String>> emptyKeysList =
+ (Collection<Map<String, String>>) queriedShardingKeysMap
+ .get(MetadataStoreRoutingConstants.SHARDING_KEYS);
+ Assert.assertTrue(emptyKeysList.isEmpty());
+
+ // Success response with non empty sharding keys.
+ String shardingKeyPrefix = "/sharding/key";
+ responseBody = new JerseyUriRequestBuilder(
+ TEST_NAMESPACE_URI_PREFIX + "/sharding-keys?prefix=" + shardingKeyPrefix)
+ .isBodyReturnExpected(true).get(this);
+
+ queriedShardingKeysMap = OBJECT_MAPPER.readValue(responseBody, Map.class);
+
+ // Check fields.
+ Assert.assertEquals(queriedShardingKeysMap.keySet(), ImmutableSet
+ .of(MetadataStoreRoutingConstants.SHARDING_KEY_PATH_PREFIX,
+ MetadataStoreRoutingConstants.SHARDING_KEYS));
+
+ // Check sharding key prefix in json response.
+ Assert.assertEquals(
+ queriedShardingKeysMap.get(MetadataStoreRoutingConstants.SHARDING_KEY_PATH_PREFIX),
+ shardingKeyPrefix);
+
+ Collection<Map<String, String>> queriedShardingKeys =
+ (Collection<Map<String, String>>) queriedShardingKeysMap
+ .get(MetadataStoreRoutingConstants.SHARDING_KEYS);
+ Set<Map<String, String>> queriedShardingKeysSet = new HashSet<>(queriedShardingKeys);
+ Set<Map<String, String>> expectedShardingKeysSet = new HashSet<>();
+
+ TEST_SHARDING_KEYS_1.forEach(key -> expectedShardingKeysSet.add(ImmutableMap
+ .of(MetadataStoreRoutingConstants.SINGLE_SHARDING_KEY, key,
+ MetadataStoreRoutingConstants.SINGLE_METADATA_STORE_REALM, TEST_REALM_1)));
+
+ TEST_SHARDING_KEYS_2.forEach(key -> expectedShardingKeysSet.add(ImmutableMap
+ .of(MetadataStoreRoutingConstants.SINGLE_SHARDING_KEY, key,
+ MetadataStoreRoutingConstants.SINGLE_METADATA_STORE_REALM, TEST_REALM_2)));
+
+ Assert.assertEquals(queriedShardingKeysSet, expectedShardingKeysSet);
+ }
+
+ /*
+ * Tests REST endpoint: "GET /metadata-store-realms/{realm}/sharding-keys?prefix={prefix}"
+ */
+ @SuppressWarnings("unchecked")
+ @Test(dependsOnMethods = "testGetShardingKeysUnderPath")
+ public void testGetRealmShardingKeysUnderPath() throws IOException {
+ // Test non existed prefix and empty sharding keys in response.
+ String responseBody = new JerseyUriRequestBuilder(
+ TEST_NAMESPACE_URI_PREFIX + "/metadata-store-realms/" + TEST_REALM_1
+ + "/sharding-keys?prefix=/non/Existed/Prefix").isBodyReturnExpected(true).get(this);
+
+ Map<String, Object> queriedShardingKeysMap = OBJECT_MAPPER.readValue(responseBody, Map.class);
+ Collection<Map<String, String>> emptyKeysList =
+ (Collection<Map<String, String>>) queriedShardingKeysMap
+ .get(MetadataStoreRoutingConstants.SHARDING_KEYS);
+ Assert.assertTrue(emptyKeysList.isEmpty());
+
+ // Test non existed realm and empty sharding keys in response.
+ responseBody = new JerseyUriRequestBuilder(TEST_NAMESPACE_URI_PREFIX
+ + "/metadata-store-realms/nonExistedRealm/sharding-keys?prefix=/sharding/key")
+ .isBodyReturnExpected(true).get(this);
+
+ queriedShardingKeysMap = OBJECT_MAPPER.readValue(responseBody, Map.class);
+ emptyKeysList = (Collection<Map<String, String>>) queriedShardingKeysMap
+ .get(MetadataStoreRoutingConstants.SHARDING_KEYS);
+ Assert.assertTrue(emptyKeysList.isEmpty());
+
+ // Valid query params and non empty sharding keys.
+ String shardingKeyPrefix = "/sharding/key/1";
+ responseBody = new JerseyUriRequestBuilder(
+ TEST_NAMESPACE_URI_PREFIX + "/metadata-store-realms/" + TEST_REALM_1
+ + "/sharding-keys?prefix=" + shardingKeyPrefix).isBodyReturnExpected(true).get(this);
+
+ queriedShardingKeysMap = OBJECT_MAPPER.readValue(responseBody, Map.class);
+
+ // Check fields.
+ Assert.assertEquals(queriedShardingKeysMap.keySet(), ImmutableSet
+ .of(MetadataStoreRoutingConstants.SHARDING_KEY_PATH_PREFIX,
+ MetadataStoreRoutingConstants.SINGLE_METADATA_STORE_REALM,
+ MetadataStoreRoutingConstants.SHARDING_KEYS));
+
+ // Check sharding key prefix in json response.
+ Assert.assertEquals(
+ queriedShardingKeysMap.get(MetadataStoreRoutingConstants.SHARDING_KEY_PATH_PREFIX),
+ shardingKeyPrefix);
- // Check realm name in json response.
- Assert.assertTrue(queriedShardingKeysMap
- .containsKey(MetadataStoreRoutingConstants.SINGLE_METADATA_STORE_REALM));
Assert.assertEquals(
queriedShardingKeysMap.get(MetadataStoreRoutingConstants.SINGLE_METADATA_STORE_REALM),
TEST_REALM_1);
- Assert.assertTrue(
- queriedShardingKeysMap.containsKey(MetadataStoreRoutingConstants.SHARDING_KEYS));
- // It is safe to cast the object and suppress warnings.
- @SuppressWarnings("unchecked")
Set<String> queriedShardingKeys = new HashSet<>((Collection<String>) queriedShardingKeysMap
.get(MetadataStoreRoutingConstants.SHARDING_KEYS));
Set<String> expectedShardingKeys = new HashSet<>(TEST_SHARDING_KEYS_1);
@@ -262,7 +408,10 @@ public class TestMetadataStoreDirectoryAccessor extends AbstractTestClass {
Assert.assertEquals(queriedShardingKeys, expectedShardingKeys);
}
- @Test(dependsOnMethods = "testGetShardingKeysInRealm")
+ /*
+ * Tests REST endpoint: "PUT /metadata-store-realms/{realm}/sharding-keys/{sharding-key}"
+ */
+ @Test(dependsOnMethods = "testGetRealmShardingKeysUnderPath")
public void testAddShardingKey() {
Set<String> expectedShardingKeysSet = new HashSet<>(
_metadataStoreDirectory.getAllShardingKeysInRealm(TEST_NAMESPACE, TEST_REALM_1));
@@ -287,6 +436,9 @@ public class TestMetadataStoreDirectoryAccessor extends AbstractTestClass {
// Assert.assertEquals(updatedShardingKeysSet, expectedShardingKeysSet);
}
+ /*
+ * Tests REST endpoint: "PUT /metadata-store-realms/{realm}/sharding-keys/{sharding-key}"
+ */
@Test(dependsOnMethods = "testAddShardingKey")
public void testDeleteShardingKey() {
Set<String> expectedShardingKeysSet = new HashSet<>(
@@ -310,8 +462,31 @@ public class TestMetadataStoreDirectoryAccessor extends AbstractTestClass {
// Assert.assertEquals(updatedShardingKeysSet, expectedShardingKeysSet);
}
- private void deleteRoutingDataPath()
- throws Exception {
+ private void verifyRealmShardingKeys(String responseBody) throws IOException {
+ // It is safe to cast the object and suppress warnings.
+ @SuppressWarnings("unchecked")
+ Map<String, Object> queriedShardingKeysMap = OBJECT_MAPPER.readValue(responseBody, Map.class);
+
+ // Check fields in JSON response.
+ Assert.assertEquals(queriedShardingKeysMap.keySet(), ImmutableSet
+ .of(MetadataStoreRoutingConstants.SINGLE_METADATA_STORE_REALM,
+ MetadataStoreRoutingConstants.SHARDING_KEYS));
+
+ // Check realm name in json response.
+ Assert.assertEquals(
+ queriedShardingKeysMap.get(MetadataStoreRoutingConstants.SINGLE_METADATA_STORE_REALM),
+ TEST_REALM_1);
+
+ // It is safe to cast the object and suppress warnings.
+ @SuppressWarnings("unchecked")
+ Set<String> queriedShardingKeys = new HashSet<>((Collection<String>) queriedShardingKeysMap
+ .get(MetadataStoreRoutingConstants.SHARDING_KEYS));
+ Set<String> expectedShardingKeys = new HashSet<>(TEST_SHARDING_KEYS_1);
+
+ Assert.assertEquals(queriedShardingKeys, expectedShardingKeys);
+ }
+
+ private void deleteRoutingDataPath() throws Exception {
Assert.assertTrue(TestHelper.verify(() -> {
_zkList.forEach(zk -> ZK_SERVER_MAP.get(zk).getZkClient()
.deleteRecursively(MetadataStoreRoutingConstants.ROUTING_DATA_PATH));
diff --git a/metadata-store-directory-common/src/main/java/org/apache/helix/msdcommon/constant/MetadataStoreRoutingConstants.java b/metadata-store-directory-common/src/main/java/org/apache/helix/msdcommon/constant/MetadataStoreRoutingConstants.java
index 009e7f3..13e78b0 100644
--- a/metadata-store-directory-common/src/main/java/org/apache/helix/msdcommon/constant/MetadataStoreRoutingConstants.java
+++ b/metadata-store-directory-common/src/main/java/org/apache/helix/msdcommon/constant/MetadataStoreRoutingConstants.java
@@ -28,14 +28,35 @@ public class MetadataStoreRoutingConstants {
// Leader election ZNode for ZkRoutingDataWriter
public static final String LEADER_ELECTION_ZNODE = "/_ZK_ROUTING_DATA_WRITER_LEADER";
- // Field name in JSON REST response of getting metadata store realms in one namespace.
- public static final String METADATA_STORE_REALMS = "metadataStoreRealms";
+ /** Field name in JSON REST response of getting all metadata store namespaces. */
+ public static final String METADATA_STORE_NAMESPACES = "namespaces";
- // Field name in JSON REST response of getting sharding keys in one realm.
- public static final String SINGLE_METADATA_STORE_REALM = "metadataStoreRealm";
+ /** Field name in JSON REST response of getting all sharding keys in a single namespace. */
+ public static final String SINGLE_METADATA_STORE_NAMESPACE = "namespace";
- // Field name in JSON REST response of getting sharding keys.
- public static final String SHARDING_KEYS = "shardingKeys";
+ /** Field name in JSON REST response of getting metadata store realms in one namespace. */
+ public static final String METADATA_STORE_REALMS = "realms";
+
+ /** Field name in JSON REST response of getting sharding keys in one realm. */
+ public static final String SINGLE_METADATA_STORE_REALM = "realm";
+ /** Field name in JSON REST response of getting sharding keys. */
+ public static final String SHARDING_KEYS = "shardingKeys";
+ /** Field name in JSON REST response related to one single sharding key. */
+ public static final String SINGLE_SHARDING_KEY = "shardingKey";
+
+ /**
+ * Field name in JSON response of the REST endpoint getting sharding keys with prefix:
+ * "GET /sharding-keys?prefix={prefix}"
+ * It is used in below response as an example:
+ * {
+ * "prefix": "/sharding/key",
+ * "shardingKeys": [{
+ * "realm": "testRealm2",
+ * "shardingKey": "/sharding/key/1/f"
+ * }]
+ * }
+ */
+ public static final String SHARDING_KEY_PATH_PREFIX = "prefix";
}