[helix] 20/49: [helix-rest] Add endpoint to get namespace routing data (#799)

[hu...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

hulee pushed a commit to branch zooscalability
in repository https://gitbox.apache.org/repos/asf/helix.git

commit 7cc4841ccf6e8bcdd624ba36ca3c405269a950be
Author: Huizhi Lu <ih...@gmail.com>
AuthorDate: Sun Feb 23 15:19:37 2020 -0800

    [helix-rest] Add endpoint to get namespace routing data (#799)
    
    RealmAwareZkClient construction needs a REST endpoint to get routing data in a namespace. This endpoint will help with reducing REST calls down to one single call to read all raw routing data in ZK.
    
    This commit adds Java API getNamespaceRoutingData() and an endpoint "GET /routing-data".
---
 .../rest/metadatastore/MetadataStoreDirectory.java |  9 ++++
 .../metadatastore/ZkMetadataStoreDirectory.java    | 10 ++++
 .../MetadataStoreShardingKeysByRealm.java          | 56 ++++++++++++++++++++++
 .../MetadataStoreDirectoryAccessor.java            | 38 +++++++++++++++
 .../server/TestMetadataStoreDirectoryAccessor.java | 53 +++++++++++++++++++-
 .../constant/MetadataStoreRoutingConstants.java    |  3 ++
 6 files changed, 168 insertions(+), 1 deletion(-)

diff --git a/helix-rest/src/main/java/org/apache/helix/rest/metadatastore/MetadataStoreDirectory.java b/helix-rest/src/main/java/org/apache/helix/rest/metadatastore/MetadataStoreDirectory.java
index 032362a..4630d50 100644
--- a/helix-rest/src/main/java/org/apache/helix/rest/metadatastore/MetadataStoreDirectory.java
+++ b/helix-rest/src/main/java/org/apache/helix/rest/metadatastore/MetadataStoreDirectory.java
@@ -20,6 +20,7 @@ package org.apache.helix.rest.metadatastore;
  */
 
 import java.util.Collection;
+import java.util.List;
 import java.util.Map;
 import java.util.NoSuchElementException;
 
@@ -52,6 +53,14 @@ public interface MetadataStoreDirectory extends AutoCloseable {
   Collection<String> getAllShardingKeys(String namespace);
 
   /**
+   * Returns routing data in the given namespace.
+   *
+   * @param namespace namespace in metadata store directory.
+   * @return Routing data map: realm -> List of sharding keys
+   */
+  Map<String, List<String>> getNamespaceRoutingData(String namespace);
+
+  /**
    * Returns all path-based sharding keys in the given namespace and the realm.
    * @param namespace
    * @param realm
diff --git a/helix-rest/src/main/java/org/apache/helix/rest/metadatastore/ZkMetadataStoreDirectory.java b/helix-rest/src/main/java/org/apache/helix/rest/metadatastore/ZkMetadataStoreDirectory.java
index 7e972ab..5b64f7b 100644
--- a/helix-rest/src/main/java/org/apache/helix/rest/metadatastore/ZkMetadataStoreDirectory.java
+++ b/helix-rest/src/main/java/org/apache/helix/rest/metadatastore/ZkMetadataStoreDirectory.java
@@ -112,6 +112,16 @@ public class ZkMetadataStoreDirectory implements MetadataStoreDirectory, Routing
   }
 
   @Override
+  public Map<String, List<String>> getNamespaceRoutingData(String namespace) {
+    Map<String, List<String>> routingData = _realmToShardingKeysMap.get(namespace);
+    if (routingData == null) {
+      throw new NoSuchElementException("Namespace " + namespace + " does not exist!");
+    }
+
+    return routingData;
+  }
+
+  @Override
   public Collection<String> getAllShardingKeysInRealm(String namespace, String realm) {
     if (!_realmToShardingKeysMap.containsKey(namespace)) {
       throw new NoSuchElementException("Namespace " + namespace + " does not exist!");
diff --git a/helix-rest/src/main/java/org/apache/helix/rest/metadatastore/datamodel/MetadataStoreShardingKeysByRealm.java b/helix-rest/src/main/java/org/apache/helix/rest/metadatastore/datamodel/MetadataStoreShardingKeysByRealm.java
new file mode 100644
index 0000000..0f54350
--- /dev/null
+++ b/helix-rest/src/main/java/org/apache/helix/rest/metadatastore/datamodel/MetadataStoreShardingKeysByRealm.java
@@ -0,0 +1,56 @@
+package org.apache.helix.rest.metadatastore.datamodel;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.util.Collection;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonPropertyOrder;
+
+
+@JsonPropertyOrder({"realm", "shardingKeys"})
+public class MetadataStoreShardingKeysByRealm {
+  private String realm;
+  private Collection<String> shardingKeys;
+
+  @JsonCreator
+  public MetadataStoreShardingKeysByRealm(@JsonProperty String realm,
+      @JsonProperty Collection<String> shardingKeys) {
+    this.realm = realm;
+    this.shardingKeys = shardingKeys;
+  }
+
+  @JsonProperty
+  public String getRealm() {
+    return realm;
+  }
+
+  @JsonProperty
+  public Collection<String> getShardingKeys() {
+    return shardingKeys;
+  }
+
+  @Override
+  public String toString() {
+    return "MetadataStoreShardingKeysByRealm{" + "realm='" + realm + '\'' + ", shardingKeys="
+        + shardingKeys + '}';
+  }
+}
diff --git a/helix-rest/src/main/java/org/apache/helix/rest/server/resources/metadatastore/MetadataStoreDirectoryAccessor.java b/helix-rest/src/main/java/org/apache/helix/rest/server/resources/metadatastore/MetadataStoreDirectoryAccessor.java
index bc6571a..0f22d81 100644
--- a/helix-rest/src/main/java/org/apache/helix/rest/server/resources/metadatastore/MetadataStoreDirectoryAccessor.java
+++ b/helix-rest/src/main/java/org/apache/helix/rest/server/resources/metadatastore/MetadataStoreDirectoryAccessor.java
@@ -43,6 +43,7 @@ import org.apache.helix.rest.common.HelixRestUtils;
 import org.apache.helix.rest.metadatastore.MetadataStoreDirectory;
 import org.apache.helix.rest.metadatastore.ZkMetadataStoreDirectory;
 import org.apache.helix.rest.metadatastore.datamodel.MetadataStoreShardingKey;
+import org.apache.helix.rest.metadatastore.datamodel.MetadataStoreShardingKeysByRealm;
 import org.apache.helix.rest.server.resources.AbstractResource;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -181,6 +182,43 @@ public class MetadataStoreDirectoryAccessor extends AbstractResource {
   }
 
   /**
+   * Gets routing data in current namespace.
+   *
+   * - "HTTP GET /routing-data"
+   * -- Response example:
+   * {
+   *   "namespace" : "my-namespace",
+   *   "routingData" : [ {
+   *     "realm" : "realm-1",
+   *     "shardingKeys" : [ "/sharding/key/1/d", "/sharding/key/1/e", "/sharding/key/1/f" ]
+   *   }, {
+   *     "realm" : "realm-2",
+   *     "shardingKeys" : [ "/sharding/key/1/a", "/sharding/key/1/b", "/sharding/key/1/c" ]
+   *   } ]
+   * }
+   */
+  @GET
+  @Path("/routing-data")
+  public Response getRoutingData() {
+    Map<String, List<String>> rawRoutingData;
+    try {
+      rawRoutingData = _metadataStoreDirectory.getNamespaceRoutingData(_namespace);
+    } catch (NoSuchElementException ex) {
+      return notFound(ex.getMessage());
+    }
+
+    List<MetadataStoreShardingKeysByRealm> shardingKeysByRealm = rawRoutingData.entrySet().stream()
+        .map(entry -> new MetadataStoreShardingKeysByRealm(entry.getKey(), entry.getValue()))
+        .collect(Collectors.toList());
+
+    Map<String, Object> responseMap = ImmutableMap
+        .of(MetadataStoreRoutingConstants.SINGLE_METADATA_STORE_NAMESPACE, _namespace,
+            MetadataStoreRoutingConstants.ROUTING_DATA, shardingKeysByRealm);
+
+    return JSONRepresentation(responseMap);
+  }
+
+  /**
    * Gets all path-based sharding keys for a queried realm at endpoint:
    * "GET /metadata-store-realms/{realm}/sharding-keys"
    * <p>
diff --git a/helix-rest/src/test/java/org/apache/helix/rest/server/TestMetadataStoreDirectoryAccessor.java b/helix-rest/src/test/java/org/apache/helix/rest/server/TestMetadataStoreDirectoryAccessor.java
index 6a9c598..ee49239 100644
--- a/helix-rest/src/test/java/org/apache/helix/rest/server/TestMetadataStoreDirectoryAccessor.java
+++ b/helix-rest/src/test/java/org/apache/helix/rest/server/TestMetadataStoreDirectoryAccessor.java
@@ -282,9 +282,60 @@ public class TestMetadataStoreDirectoryAccessor extends AbstractTestClass {
   }
 
   /*
-   * Tests REST endpoint: "GET /metadata-store-realms/{realm}/sharding-keys"
+   * Tests REST endpoint: "GET /routing-data"
    */
   @Test(dependsOnMethods = "testGetShardingKeysInNamespace")
+  public void testGetRoutingData() throws IOException {
+    /*
+     * responseBody:
+     * {
+     *   "namespace" : "test-namespace",
+     *   "routingData" : [ {
+     *     "realm" : "testRealm2",
+     *     "shardingKeys" : [ "/sharding/key/1/d", "/sharding/key/1/e", "/sharding/key/1/f" ]
+     *   }, {
+     *     "realm" : "testRealm1",
+     *     "shardingKeys" : [ "/sharding/key/1/a", "/sharding/key/1/b", "/sharding/key/1/c" ]
+     *   } ]
+     * }
+     */
+    String responseBody =
+        new JerseyUriRequestBuilder(TEST_NAMESPACE_URI_PREFIX + "/routing-data")
+            .isBodyReturnExpected(true).get(this);
+
+    // It is safe to cast the object and suppress warnings.
+    @SuppressWarnings("unchecked")
+    Map<String, Object> queriedShardingKeysMap = OBJECT_MAPPER.readValue(responseBody, Map.class);
+
+    // Check fields.
+    Assert.assertEquals(queriedShardingKeysMap.keySet(), ImmutableSet
+        .of(MetadataStoreRoutingConstants.SINGLE_METADATA_STORE_NAMESPACE,
+            MetadataStoreRoutingConstants.ROUTING_DATA));
+
+    // Check namespace in json response.
+    Assert.assertEquals(
+        queriedShardingKeysMap.get(MetadataStoreRoutingConstants.SINGLE_METADATA_STORE_NAMESPACE),
+        TEST_NAMESPACE);
+
+    @SuppressWarnings("unchecked")
+    List<Map<String, Object>> queriedShardingKeys =
+        (List<Map<String, Object>>) queriedShardingKeysMap
+            .get(MetadataStoreRoutingConstants.ROUTING_DATA);
+
+    Set<Map<String, Object>> queriedShardingKeysSet = new HashSet<>(queriedShardingKeys);
+    Set<Map<String, Object>> expectedShardingKeysSet = ImmutableSet.of(ImmutableMap
+        .of(MetadataStoreRoutingConstants.SINGLE_METADATA_STORE_REALM, TEST_REALM_1,
+            MetadataStoreRoutingConstants.SHARDING_KEYS, TEST_SHARDING_KEYS_1), ImmutableMap
+        .of(MetadataStoreRoutingConstants.SINGLE_METADATA_STORE_REALM, TEST_REALM_2,
+            MetadataStoreRoutingConstants.SHARDING_KEYS, TEST_SHARDING_KEYS_2));
+
+    Assert.assertEquals(queriedShardingKeysSet, expectedShardingKeysSet);
+  }
+
+  /*
+   * Tests REST endpoint: "GET /metadata-store-realms/{realm}/sharding-keys"
+   */
+  @Test(dependsOnMethods = "testGetRoutingData")
   public void testGetShardingKeysInRealm() throws IOException {
     // Test NOT_FOUND response for a non existed realm.
     new JerseyUriRequestBuilder(
diff --git a/metadata-store-directory-common/src/main/java/org/apache/helix/msdcommon/constant/MetadataStoreRoutingConstants.java b/metadata-store-directory-common/src/main/java/org/apache/helix/msdcommon/constant/MetadataStoreRoutingConstants.java
index 13e78b0..2358bcd 100644
--- a/metadata-store-directory-common/src/main/java/org/apache/helix/msdcommon/constant/MetadataStoreRoutingConstants.java
+++ b/metadata-store-directory-common/src/main/java/org/apache/helix/msdcommon/constant/MetadataStoreRoutingConstants.java
@@ -43,6 +43,9 @@ public class MetadataStoreRoutingConstants {
   /** Field name in JSON REST response of getting sharding keys. */
   public static final String SHARDING_KEYS = "shardingKeys";
 
+  /** Field name in JSON REST response of getting routing data. */
+  public static final String ROUTING_DATA = "routingData";
+
   /** Field name in JSON REST response related to one single sharding key. */
   public static final String SINGLE_SHARDING_KEY = "shardingKey";