You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@helix.apache.org by hu...@apache.org on 2020/04/01 22:47:07 UTC
[helix] 12/49: Implement getAllMappingUnderPath and
getMetadataStoreRealm in ZkMetadataStoreDirectory
This is an automated email from the ASF dual-hosted git repository.
hulee pushed a commit to branch zooscalability
in repository https://gitbox.apache.org/repos/asf/helix.git
commit ba99639c62f7f50e36e96a4b0edeee4148aa1929
Author: Neal Sun <ne...@gmail.com>
AuthorDate: Wed Feb 12 15:16:28 2020 -0800
Implement getAllMappingUnderPath and getMetadataStoreRealm in ZkMetadataStoreDirectory
This PR implements getAllMappingUnderPath and getMetadataStoreRealm in ZkMetadataStoreDirectory. Also, getRoutingData in ZkRoutingDataReader is updated : not having realm addresses and not having sharding keys for some realm addresses no longer raise exceptions. We now allow realms to not have sharding keys and return them as a part of routing data.
---
.../helix/rest/metadatastore/TrieRoutingData.java | 6 +---
.../metadatastore/ZkMetadataStoreDirectory.java | 16 +++++++---
.../accessor/ZkRoutingDataReader.java | 27 ++++++-----------
.../rest/metadatastore/TestTrieRoutingData.java | 13 --------
.../TestZkMetadataStoreDirectory.java | 35 ++++++++++++++++++++++
.../accessor/TestZkRoutingDataReader.java | 18 +++++------
6 files changed, 64 insertions(+), 51 deletions(-)
diff --git a/helix-rest/src/main/java/org/apache/helix/rest/metadatastore/TrieRoutingData.java b/helix-rest/src/main/java/org/apache/helix/rest/metadatastore/TrieRoutingData.java
index 28add4c..923f818 100644
--- a/helix-rest/src/main/java/org/apache/helix/rest/metadatastore/TrieRoutingData.java
+++ b/helix-rest/src/main/java/org/apache/helix/rest/metadatastore/TrieRoutingData.java
@@ -162,15 +162,11 @@ public class TrieRoutingData implements MetadataStoreRoutingData {
* @throws InvalidRoutingDataException - when there is an empty sharding key (edge case that
* always renders the routing data invalid); when there is a sharding key which already
* contains a sharding key (invalid); when there is a sharding key that is a part of
- * another sharding key (invalid)
+ * another sharding key (invalid); when a sharding key doesn't have a leading delimiter
*/
private void constructTrie(Map<String, List<String>> routingData)
throws InvalidRoutingDataException {
for (Map.Entry<String, List<String>> entry : routingData.entrySet()) {
- if (entry.getValue().isEmpty()) {
- throw new InvalidRoutingDataException(
- "Realm address does not have associating sharding keys: " + entry.getKey());
- }
for (String shardingKey : entry.getValue()) {
// Missing leading delimiter is invalid
if (shardingKey.isEmpty() || !shardingKey.substring(0, 1).equals(DELIMITER)) {
diff --git a/helix-rest/src/main/java/org/apache/helix/rest/metadatastore/ZkMetadataStoreDirectory.java b/helix-rest/src/main/java/org/apache/helix/rest/metadatastore/ZkMetadataStoreDirectory.java
index 536d058..3be9b72 100644
--- a/helix-rest/src/main/java/org/apache/helix/rest/metadatastore/ZkMetadataStoreDirectory.java
+++ b/helix-rest/src/main/java/org/apache/helix/rest/metadatastore/ZkMetadataStoreDirectory.java
@@ -79,6 +79,8 @@ public class ZkMetadataStoreDirectory implements MetadataStoreDirectory, Routing
// Populate realmToShardingKeys with ZkRoutingDataReader
_realmToShardingKeysMap.put(routingEntry.getKey(),
_routingDataReaderMap.get(routingEntry.getKey()).getRoutingData());
+ _routingDataMap.put(routingEntry.getKey(),
+ new TrieRoutingData(_realmToShardingKeysMap.get(routingEntry.getKey())));
}
}
@@ -119,14 +121,20 @@ public class ZkMetadataStoreDirectory implements MetadataStoreDirectory, Routing
@Override
public Map<String, String> getAllMappingUnderPath(String namespace, String path) {
- // TODO: get it from routingData
- throw new UnsupportedOperationException();
+ if (!_routingDataMap.containsKey(namespace)) {
+ throw new NoSuchElementException(
+ "Failed to get all mapping under path: Namespace " + namespace + " is not found!");
+ }
+ return _routingDataMap.get(namespace).getAllMappingUnderPath(path);
}
@Override
public String getMetadataStoreRealm(String namespace, String shardingKey) {
- // TODO: get it from routingData
- throw new UnsupportedOperationException();
+ if (!_routingDataMap.containsKey(namespace)) {
+ throw new NoSuchElementException(
+ "Failed to get metadata store realm: Namespace " + namespace + " is not found!");
+ }
+ return _routingDataMap.get(namespace).getMetadataStoreRealm(shardingKey);
}
@Override
diff --git a/helix-rest/src/main/java/org/apache/helix/rest/metadatastore/accessor/ZkRoutingDataReader.java b/helix-rest/src/main/java/org/apache/helix/rest/metadatastore/accessor/ZkRoutingDataReader.java
index 9decf23..44ce110 100644
--- a/helix-rest/src/main/java/org/apache/helix/rest/metadatastore/accessor/ZkRoutingDataReader.java
+++ b/helix-rest/src/main/java/org/apache/helix/rest/metadatastore/accessor/ZkRoutingDataReader.java
@@ -72,36 +72,27 @@ public class ZkRoutingDataReader implements MetadataStoreRoutingDataReader, IZkD
/**
* Returns (realm, list of ZK path sharding keys) mappings.
* @return Map <realm, list of ZK path sharding keys>
- * @throws InvalidRoutingDataException
+ * @throws InvalidRoutingDataException - when the node on
+ * MetadataStoreRoutingConstants.ROUTING_DATA_PATH is missing
*/
- public Map<String, List<String>> getRoutingData()
- throws InvalidRoutingDataException {
+ public Map<String, List<String>> getRoutingData() throws InvalidRoutingDataException {
Map<String, List<String>> routingData = new HashMap<>();
- List<String> children;
+ List<String> allRealmAddresses;
try {
- children = _zkClient.getChildren(MetadataStoreRoutingConstants.ROUTING_DATA_PATH);
+ allRealmAddresses = _zkClient.getChildren(MetadataStoreRoutingConstants.ROUTING_DATA_PATH);
} catch (ZkNoNodeException e) {
throw new InvalidRoutingDataException(
"Routing data directory ZNode " + MetadataStoreRoutingConstants.ROUTING_DATA_PATH
+ " does not exist. Routing ZooKeeper address: " + _zkAddress);
}
- if (children == null || children.isEmpty()) {
- throw new InvalidRoutingDataException(
- "There are no metadata store realms defined. Routing ZooKeeper address: " + _zkAddress);
- }
- for (String child : children) {
+ for (String realmAddress : allRealmAddresses) {
ZNRecord record =
- _zkClient.readData(MetadataStoreRoutingConstants.ROUTING_DATA_PATH + "/" + child);
+ _zkClient.readData(MetadataStoreRoutingConstants.ROUTING_DATA_PATH + "/" + realmAddress);
List<String> shardingKeys =
record.getListField(MetadataStoreRoutingConstants.ZNRECORD_LIST_FIELD_KEY);
- if (shardingKeys == null || shardingKeys.isEmpty()) {
- throw new InvalidRoutingDataException(
- "Realm address ZNode " + MetadataStoreRoutingConstants.ROUTING_DATA_PATH + "/" + child
- + " does not have a value for key "
- + MetadataStoreRoutingConstants.ZNRECORD_LIST_FIELD_KEY
- + ". Routing ZooKeeper address: " + _zkAddress);
+ if (shardingKeys != null) {
+ routingData.put(realmAddress, shardingKeys);
}
- routingData.put(child, shardingKeys);
}
return routingData;
}
diff --git a/helix-rest/src/test/java/org/apache/helix/rest/metadatastore/TestTrieRoutingData.java b/helix-rest/src/test/java/org/apache/helix/rest/metadatastore/TestTrieRoutingData.java
index bf71456..4de68a6 100644
--- a/helix-rest/src/test/java/org/apache/helix/rest/metadatastore/TestTrieRoutingData.java
+++ b/helix-rest/src/test/java/org/apache/helix/rest/metadatastore/TestTrieRoutingData.java
@@ -67,19 +67,6 @@ public class TestTrieRoutingData {
}
@Test
- public void testConstructionEmptyShardingKeys() {
- Map<String, List<String>> routingData = new HashMap<>();
- routingData.put("realmAddress1", Collections.emptyList());
- try {
- new TrieRoutingData(routingData);
- Assert.fail("Expecting InvalidRoutingDataException");
- } catch (InvalidRoutingDataException e) {
- Assert.assertTrue(e.getMessage()
- .contains("Realm address does not have associating sharding keys: realmAddress1"));
- }
- }
-
- @Test
public void testConstructionShardingKeyNoLeadingSlash() {
Map<String, List<String>> routingData = new HashMap<>();
routingData.put("realmAddress1", Arrays.asList("/g", "/h/i", "/h/j"));
diff --git a/helix-rest/src/test/java/org/apache/helix/rest/metadatastore/TestZkMetadataStoreDirectory.java b/helix-rest/src/test/java/org/apache/helix/rest/metadatastore/TestZkMetadataStoreDirectory.java
index c0741ee..68b93b3 100644
--- a/helix-rest/src/test/java/org/apache/helix/rest/metadatastore/TestZkMetadataStoreDirectory.java
+++ b/helix-rest/src/test/java/org/apache/helix/rest/metadatastore/TestZkMetadataStoreDirectory.java
@@ -21,6 +21,7 @@ package org.apache.helix.rest.metadatastore;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
@@ -152,6 +153,40 @@ public class TestZkMetadataStoreDirectory extends AbstractTestClass {
}
@Test(dependsOnMethods = "testGetAllShardingKeysInRealm")
+ public void testGetAllMappingUnderPath() {
+ Map<String, String> mappingFromRoot = new HashMap<>();
+ TEST_SHARDING_KEYS_1.forEach(shardingKey -> {
+ mappingFromRoot.put(shardingKey, TEST_REALM_1);
+ });
+ TEST_SHARDING_KEYS_2.forEach(shardingKey -> {
+ mappingFromRoot.put(shardingKey, TEST_REALM_2);
+ });
+
+ Map<String, String> mappingFromLeaf = new HashMap<>(1);
+ mappingFromLeaf.put("/sharding/key/1/a", TEST_REALM_1);
+
+ for (String namespace : _routingZkAddrMap.keySet()) {
+ Assert.assertEquals(_metadataStoreDirectory.getAllMappingUnderPath(namespace, "/"),
+ mappingFromRoot);
+ Assert.assertEquals(
+ _metadataStoreDirectory.getAllMappingUnderPath(namespace, "/sharding/key/1/a"),
+ mappingFromLeaf);
+ }
+ }
+
+ @Test(dependsOnMethods = "testGetAllMappingUnderPath")
+ public void testGetMetadataStoreRealm() {
+ for (String namespace : _routingZkAddrMap.keySet()) {
+ Assert.assertEquals(
+ _metadataStoreDirectory.getMetadataStoreRealm(namespace, "/sharding/key/1/a/x/y/z"),
+ TEST_REALM_1);
+ Assert.assertEquals(
+ _metadataStoreDirectory.getMetadataStoreRealm(namespace, "/sharding/key/1/d/x/y/z"),
+ TEST_REALM_2);
+ }
+ }
+
+ @Test(dependsOnMethods = "testGetMetadataStoreRealm")
public void testDataChangeCallback()
throws Exception {
// For all namespaces (Routing ZKs), add an extra sharding key to TEST_REALM_1
diff --git a/helix-rest/src/test/java/org/apache/helix/rest/metadatastore/accessor/TestZkRoutingDataReader.java b/helix-rest/src/test/java/org/apache/helix/rest/metadatastore/accessor/TestZkRoutingDataReader.java
index aa46429..5781a85 100644
--- a/helix-rest/src/test/java/org/apache/helix/rest/metadatastore/accessor/TestZkRoutingDataReader.java
+++ b/helix-rest/src/test/java/org/apache/helix/rest/metadatastore/accessor/TestZkRoutingDataReader.java
@@ -104,11 +104,10 @@ public class TestZkRoutingDataReader extends AbstractTestClass {
_baseAccessor.create(MetadataStoreRoutingConstants.ROUTING_DATA_PATH, new ZNRecord("test"),
AccessOption.PERSISTENT);
try {
- _zkRoutingDataReader.getRoutingData();
- Assert.fail("Expecting InvalidRoutingDataException");
+ Map<String, List<String>> routingData = _zkRoutingDataReader.getRoutingData();
+ Assert.assertEquals(routingData.size(), 0);
} catch (InvalidRoutingDataException e) {
- Assert.assertTrue(e.getMessage().contains(
- "There are no metadata store realms defined. Routing ZooKeeper address: " + ZK_ADDR));
+ Assert.fail("Not expecting InvalidRoutingDataException");
}
}
@@ -120,14 +119,11 @@ public class TestZkRoutingDataReader extends AbstractTestClass {
_baseAccessor.create(MetadataStoreRoutingConstants.ROUTING_DATA_PATH + "/testRealmAddress1",
testZnRecord1, AccessOption.PERSISTENT);
try {
- _zkRoutingDataReader.getRoutingData();
- Assert.fail("Expecting InvalidRoutingDataException");
+ Map<String, List<String>> routingData = _zkRoutingDataReader.getRoutingData();
+ Assert.assertEquals(routingData.size(), 1);
+ Assert.assertTrue(routingData.get("testRealmAddress1").isEmpty());
} catch (InvalidRoutingDataException e) {
- Assert.assertTrue(e.getMessage().contains(
- "Realm address ZNode " + MetadataStoreRoutingConstants.ROUTING_DATA_PATH
- + "/testRealmAddress1 does not have a value for key "
- + MetadataStoreRoutingConstants.ZNRECORD_LIST_FIELD_KEY
- + ". Routing ZooKeeper address: " + ZK_ADDR));
+ Assert.fail("Not expecting InvalidRoutingDataException");
}
}
}