You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@helix.apache.org by hz...@apache.org on 2020/09/30 18:22:54 UTC

[helix] branch master updated: Add Metrics for External Custom Health Check Requests (#1383)

This is an automated email from the ASF dual-hosted git repository.

hzlu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/helix.git


The following commit(s) were added to refs/heads/master by this push:
     new 26c026b  Add Metrics for External Custom Health Check Requests (#1383)
26c026b is described below

commit 26c026b83c1377f7f239a2cc6e59332b64f1bf51
Author: Huizhi Lu <ih...@gmail.com>
AuthorDate: Wed Sep 30 11:22:39 2020 -0700

    Add Metrics for External Custom Health Check Requests (#1383)
    
    Custom rest client interacts with participant side to query for its health checks.
    Currently there is no metrics for the query requests. We'd like to have metrics for the external http requests.
    This commit adds http request latency, requests total and error http request total metrics for the custom rest client calls.
---
 .../rest/common/HelixDataAccessorWrapper.java      | 37 +++++++++++++--
 .../apache/helix/rest/server/HelixRestServer.java  | 15 +++---
 .../rest/server/resources/AbstractResource.java    |  8 ++++
 .../server/resources/helix/InstancesAccessor.java  |  5 +-
 .../resources/helix/PerInstanceAccessor.java       |  9 ++--
 .../rest/server/service/InstanceServiceImpl.java   | 54 ++++++++++++++++------
 .../rest/common/TestHelixDataAccessorWrapper.java  |  5 ++
 .../rest/server/service/TestInstanceService.java   | 24 ++++++----
 8 files changed, 113 insertions(+), 44 deletions(-)

diff --git a/helix-rest/src/main/java/org/apache/helix/rest/common/HelixDataAccessorWrapper.java b/helix-rest/src/main/java/org/apache/helix/rest/common/HelixDataAccessorWrapper.java
index 2df2519..cd3cbf4 100644
--- a/helix-rest/src/main/java/org/apache/helix/rest/common/HelixDataAccessorWrapper.java
+++ b/helix-rest/src/main/java/org/apache/helix/rest/common/HelixDataAccessorWrapper.java
@@ -31,12 +31,17 @@ import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 import java.util.stream.Collectors;
+
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.SharedMetricRegistries;
+import com.codahale.metrics.Timer;
 import org.apache.helix.HelixProperty;
 import org.apache.helix.PropertyKey;
 import org.apache.helix.manager.zk.ZKHelixDataAccessor;
 import org.apache.helix.model.RESTConfig;
 import org.apache.helix.rest.client.CustomRestClient;
 import org.apache.helix.rest.client.CustomRestClientFactory;
+import org.apache.helix.rest.server.service.InstanceService;
 import org.apache.helix.zookeeper.datamodel.ZNRecord;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -54,18 +59,38 @@ public class HelixDataAccessorWrapper extends ZKHelixDataAccessor {
   public static final String IS_HEALTHY_KEY = "IS_HEALTHY";
   public static final String EXPIRY_KEY = "EXPIRE";
 
+  // Metric names for custom partition check
+  private static final String CUSTOM_PARTITION_CHECK_HTTP_REQUESTS_ERROR_TOTAL = MetricRegistry
+      .name(InstanceService.class, "custom_partition_check_http_requests_error_total");
+  private static final String CUSTOM_PARTITION_CHECK_HTTP_REQUESTS_DURATION =
+      MetricRegistry.name(InstanceService.class, "custom_partition_check_http_requests_duration");
+
   private final Map<PropertyKey, HelixProperty> _propertyCache = new HashMap<>();
   private final Map<PropertyKey, List<String>> _batchNameCache = new HashMap<>();
+  protected String _namespace;
   protected CustomRestClient _restClient;
 
+  /**
+   * @deprecated Because a namespace is required, please use the other constructors.
+   *
+   * @param dataAccessor Zk Helix data accessor used to access ZK.
+   */
+  @Deprecated
   public HelixDataAccessorWrapper(ZKHelixDataAccessor dataAccessor) {
-    super(dataAccessor);
-    _restClient = CustomRestClientFactory.get();
+    this(dataAccessor, CustomRestClientFactory.get(), HelixRestNamespace.DEFAULT_NAMESPACE_NAME);
+  }
+
+  @Deprecated
+  public HelixDataAccessorWrapper(ZKHelixDataAccessor dataAccessor,
+      CustomRestClient customRestClient) {
+    this(dataAccessor, customRestClient, HelixRestNamespace.DEFAULT_NAMESPACE_NAME);
   }
 
-  public HelixDataAccessorWrapper(ZKHelixDataAccessor dataAccessor, CustomRestClient customRestClient) {
+  public HelixDataAccessorWrapper(ZKHelixDataAccessor dataAccessor,
+      CustomRestClient customRestClient, String namespace) {
     super(dataAccessor);
     _restClient = customRestClient;
+    _namespace = namespace;
   }
 
   public Map<String, Map<String, Boolean>> getAllPartitionsHealthOnLiveInstance(
@@ -166,12 +191,16 @@ public class HelixDataAccessorWrapper extends ZKHelixDataAccessor {
 
   private Map<String, Boolean> getHealthStatusFromRest(String instance, List<String> partitions,
       RESTConfig restConfig, Map<String, String> customPayLoads) {
-    try {
+    MetricRegistry metrics = SharedMetricRegistries.getOrCreate(_namespace);
+    // Total requests metric is included as an attribute(Count) in timers
+    try (final Timer.Context timer = metrics.timer(CUSTOM_PARTITION_CHECK_HTTP_REQUESTS_DURATION)
+        .time()) {
       return _restClient.getPartitionStoppableCheck(restConfig.getBaseUrl(instance), partitions,
           customPayLoads);
     } catch (IOException e) {
       LOG.error("Failed to get partition status on instance {}, partitions: {}", instance,
           partitions, e);
+      metrics.counter(CUSTOM_PARTITION_CHECK_HTTP_REQUESTS_ERROR_TOTAL).inc();
       return Collections.emptyMap();
     }
   }
diff --git a/helix-rest/src/main/java/org/apache/helix/rest/server/HelixRestServer.java b/helix-rest/src/main/java/org/apache/helix/rest/server/HelixRestServer.java
index ac99f48..17e6820 100644
--- a/helix-rest/src/main/java/org/apache/helix/rest/server/HelixRestServer.java
+++ b/helix-rest/src/main/java/org/apache/helix/rest/server/HelixRestServer.java
@@ -48,13 +48,15 @@ import org.eclipse.jetty.servlet.ServletContextHandler;
 import org.eclipse.jetty.servlet.ServletHolder;
 import org.eclipse.jetty.util.ssl.SslContextFactory;
 import org.glassfish.jersey.server.ResourceConfig;
-import org.glassfish.jersey.server.ServerProperties;
 import org.glassfish.jersey.servlet.ServletContainer;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 public class HelixRestServer {
   private static Logger LOG = LoggerFactory.getLogger(HelixRestServer.class);
+
+  private static final String REST_DOMAIN = "org.apache.helix.rest";
+
   // TODO: consider moving the following static context to ServerContext or any other place
   public static SSLContext REST_SERVER_SSL_CONTEXT;
 
@@ -139,6 +141,8 @@ public class HelixRestServer {
     ResourceConfig config = getResourceConfig(namespace, type);
     _resourceConfigMap.put(resourceConfigMapKey, config);
 
+    initMetricRegistry(config, namespace.getName());
+
     // Initialize servlet
     initServlet(config, String.format(type.getServletPathSpecTemplate(), namespace.getName()));
   }
@@ -152,16 +156,13 @@ public class HelixRestServer {
     cfg.packages(type.getServletPackageArray());
     cfg.setApplicationName(namespace.getName());
 
-    // Enable the default statistical monitoring MBean for Jersey server
-    cfg.property(ServerProperties.MONITORING_STATISTICS_MBEANS_ENABLED, true);
     cfg.property(ContextPropertyKeys.SERVER_CONTEXT.name(),
         new ServerContext(namespace.getMetadataStoreAddress(), namespace.isMultiZkEnabled(),
             namespace.getMsdsEndpoint()));
     if (type == ServletType.DEFAULT_SERVLET) {
       cfg.property(ContextPropertyKeys.ALL_NAMESPACES.name(), _helixNamespaces);
-    } else {
-      cfg.property(ContextPropertyKeys.METADATA.name(), namespace);
     }
+    cfg.property(ContextPropertyKeys.METADATA.name(), namespace);
 
     cfg.register(new CORSFilter());
     cfg.register(new AuditLogFilter(_auditLoggers));
@@ -179,7 +180,7 @@ public class HelixRestServer {
     // JmxReporter doesn't have an option to specify namespace for each servlet,
     // we use a customized object name factory to get and insert namespace to object name.
     JmxReporter jmxReporter = JmxReporter.forRegistry(metricRegistry)
-        .inDomain("org.apache.helix.rest")
+        .inDomain(REST_DOMAIN)
         .createsObjectNamesWith(new HelixRestObjectNameFactory(namespace))
         .build();
     jmxReporter.start();
@@ -212,7 +213,7 @@ public class HelixRestServer {
     }
   }
 
-  public void shutdown() {
+  public synchronized void shutdown() {
     if (_server != null) {
       try {
         _server.stop();
diff --git a/helix-rest/src/main/java/org/apache/helix/rest/server/resources/AbstractResource.java b/helix-rest/src/main/java/org/apache/helix/rest/server/resources/AbstractResource.java
index 60a79bb..5101b22 100644
--- a/helix-rest/src/main/java/org/apache/helix/rest/server/resources/AbstractResource.java
+++ b/helix-rest/src/main/java/org/apache/helix/rest/server/resources/AbstractResource.java
@@ -34,6 +34,8 @@ import javax.ws.rs.core.Response;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.SerializationFeature;
 import org.apache.helix.HelixException;
+import org.apache.helix.rest.common.ContextPropertyKeys;
+import org.apache.helix.rest.common.HelixRestNamespace;
 import org.apache.helix.rest.server.auditlog.AuditLog;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -177,4 +179,10 @@ public class AbstractResource {
       throw new HelixException("Unknown command: " + commandStr);
     }
   }
+
+  protected String getNamespace() {
+    HelixRestNamespace namespace =
+        (HelixRestNamespace) _application.getProperties().get(ContextPropertyKeys.METADATA.name());
+    return namespace.getName();
+  }
 }
diff --git a/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/InstancesAccessor.java b/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/InstancesAccessor.java
index 04c24f3..6ebe57a 100644
--- a/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/InstancesAccessor.java
+++ b/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/InstancesAccessor.java
@@ -48,7 +48,6 @@ import org.apache.helix.HelixException;
 import org.apache.helix.manager.zk.ZKHelixDataAccessor;
 import org.apache.helix.model.ClusterConfig;
 import org.apache.helix.model.InstanceConfig;
-import org.apache.helix.rest.common.HelixDataAccessorWrapper;
 import org.apache.helix.rest.common.HttpConstants;
 import org.apache.helix.rest.server.json.cluster.ClusterTopology;
 import org.apache.helix.rest.server.json.instance.StoppableCheck;
@@ -224,8 +223,8 @@ public class InstancesAccessor extends AbstractHelixResource {
       ObjectNode failedStoppableInstances = result.putObject(
           InstancesAccessor.InstancesProperties.instance_not_stoppable_with_reasons.name());
       InstanceService instanceService =
-          new InstanceServiceImpl(new HelixDataAccessorWrapper((ZKHelixDataAccessor) getDataAccssor(clusterId)),
-              getConfigAccessor(), skipZKRead);
+          new InstanceServiceImpl((ZKHelixDataAccessor) getDataAccssor(clusterId),
+              getConfigAccessor(), skipZKRead, getNamespace());
       ClusterService clusterService = new ClusterServiceImpl(getDataAccssor(clusterId), getConfigAccessor());
       ClusterTopology clusterTopology = clusterService.getClusterTopology(clusterId);
       switch (selectionBase) {
diff --git a/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/PerInstanceAccessor.java b/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/PerInstanceAccessor.java
index 474e4fa..8785796 100644
--- a/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/PerInstanceAccessor.java
+++ b/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/PerInstanceAccessor.java
@@ -56,7 +56,6 @@ import org.apache.helix.model.InstanceConfig;
 import org.apache.helix.model.Message;
 import org.apache.helix.model.ParticipantHistory;
 import org.apache.helix.model.builder.HelixConfigScopeBuilder;
-import org.apache.helix.rest.common.HelixDataAccessorWrapper;
 import org.apache.helix.rest.common.HttpConstants;
 import org.apache.helix.rest.server.json.instance.InstanceInfo;
 import org.apache.helix.rest.server.json.instance.StoppableCheck;
@@ -106,8 +105,8 @@ public class PerInstanceAccessor extends AbstractHelixResource {
       HelixDataAccessor dataAccessor = getDataAccssor(clusterId);
       // TODO reduce GC by dependency injection
       InstanceService instanceService =
-          new InstanceServiceImpl(new HelixDataAccessorWrapper((ZKHelixDataAccessor) dataAccessor), getConfigAccessor(),
-              Boolean.valueOf(skipZKRead));
+          new InstanceServiceImpl((ZKHelixDataAccessor) dataAccessor, getConfigAccessor(),
+              Boolean.parseBoolean(skipZKRead), getNamespace());
       InstanceInfo instanceInfo = instanceService.getInstanceInfo(clusterId, instanceName,
           InstanceService.HealthCheck.STARTED_AND_HEALTH_CHECK_LIST);
       String instanceInfoString;
@@ -144,8 +143,8 @@ public class PerInstanceAccessor extends AbstractHelixResource {
     ObjectMapper objectMapper = new ObjectMapper();
     HelixDataAccessor dataAccessor = getDataAccssor(clusterId);
     InstanceService instanceService =
-        new InstanceServiceImpl(new HelixDataAccessorWrapper((ZKHelixDataAccessor) dataAccessor), getConfigAccessor(),
-            Boolean.valueOf(skipZKRead));
+        new InstanceServiceImpl((ZKHelixDataAccessor) dataAccessor, getConfigAccessor(),
+            Boolean.parseBoolean(skipZKRead), getNamespace());
     StoppableCheck stoppableCheck = null;
     try {
       stoppableCheck =
diff --git a/helix-rest/src/main/java/org/apache/helix/rest/server/service/InstanceServiceImpl.java b/helix-rest/src/main/java/org/apache/helix/rest/server/service/InstanceServiceImpl.java
index 5099ec8..2c8458d 100644
--- a/helix-rest/src/main/java/org/apache/helix/rest/server/service/InstanceServiceImpl.java
+++ b/helix-rest/src/main/java/org/apache/helix/rest/server/service/InstanceServiceImpl.java
@@ -21,7 +21,7 @@ package org.apache.helix.rest.server.service;
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -32,12 +32,16 @@ import java.util.concurrent.Future;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.SharedMetricRegistries;
+import com.codahale.metrics.Timer;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableList;
 import org.apache.helix.ConfigAccessor;
 import org.apache.helix.HelixException;
+import org.apache.helix.manager.zk.ZKHelixDataAccessor;
 import org.apache.helix.model.CurrentState;
 import org.apache.helix.model.ExternalView;
 import org.apache.helix.model.InstanceConfig;
@@ -46,6 +50,7 @@ import org.apache.helix.model.RESTConfig;
 import org.apache.helix.rest.client.CustomRestClient;
 import org.apache.helix.rest.client.CustomRestClientFactory;
 import org.apache.helix.rest.common.HelixDataAccessorWrapper;
+import org.apache.helix.rest.common.HelixRestNamespace;
 import org.apache.helix.rest.server.json.instance.InstanceInfo;
 import org.apache.helix.rest.server.json.instance.StoppableCheck;
 import org.apache.helix.util.InstanceValidationUtil;
@@ -58,29 +63,42 @@ public class InstanceServiceImpl implements InstanceService {
   private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
   private static final ExecutorService POOL = Executors.newCachedThreadPool();
 
+  // Metric names for custom instance check
+  private static final String CUSTOM_INSTANCE_CHECK_HTTP_REQUESTS_ERROR_TOTAL =
+      MetricRegistry.name(InstanceService.class, "custom_instance_check_http_requests_error_total");
+  private static final String CUSTOM_INSTANCE_CHECK_HTTP_REQUESTS_DURATION =
+      MetricRegistry.name(InstanceService.class, "custom_instance_check_http_requests_duration");
+
   private final HelixDataAccessorWrapper _dataAccessor;
   private final ConfigAccessor _configAccessor;
   private final CustomRestClient _customRestClient;
+  private String _namespace;
   private boolean _skipZKRead;
 
-  public InstanceServiceImpl(HelixDataAccessorWrapper dataAccessor, ConfigAccessor configAccessor) {
-    _dataAccessor = dataAccessor;
-    _configAccessor = configAccessor;
-    _customRestClient = CustomRestClientFactory.get();
+  @Deprecated
+  public InstanceServiceImpl(ZKHelixDataAccessor dataAccessor, ConfigAccessor configAccessor) {
+    this(dataAccessor, configAccessor, false);
+  }
+
+  @Deprecated
+  public InstanceServiceImpl(ZKHelixDataAccessor dataAccessor, ConfigAccessor configAccessor,
+      boolean skipZKRead) {
+    this(dataAccessor, configAccessor, skipZKRead, HelixRestNamespace.DEFAULT_NAMESPACE_NAME);
   }
 
-  public InstanceServiceImpl(HelixDataAccessorWrapper dataAccessor, ConfigAccessor configAccessor, boolean skipZKRead) {
-    this(dataAccessor,configAccessor);
-    this._skipZKRead = skipZKRead;
+  public InstanceServiceImpl(ZKHelixDataAccessor dataAccessor, ConfigAccessor configAccessor,
+      boolean skipZKRead, String namespace) {
+    this(dataAccessor, configAccessor, CustomRestClientFactory.get(), skipZKRead, namespace);
   }
 
   @VisibleForTesting
-  InstanceServiceImpl(HelixDataAccessorWrapper dataAccessor, ConfigAccessor configAccessor,
-      CustomRestClient customRestClient, boolean skipZKRead) {
-    _dataAccessor = dataAccessor;
+  InstanceServiceImpl(ZKHelixDataAccessor dataAccessor, ConfigAccessor configAccessor,
+      CustomRestClient customRestClient, boolean skipZKRead, String namespace) {
+    _dataAccessor = new HelixDataAccessorWrapper(dataAccessor, customRestClient, namespace);
     _configAccessor = configAccessor;
     _customRestClient = customRestClient;
     _skipZKRead = skipZKRead;
+    _namespace = namespace;
   }
 
   @Override
@@ -231,14 +249,20 @@ public class InstanceServiceImpl implements InstanceService {
   private StoppableCheck performCustomInstanceCheck(String clusterId, String instanceName,
       String baseUrl, Map<String, String> customPayLoads) {
     LOG.info("Perform instance level client side health checks for {}/{}", clusterId, instanceName);
-    try {
-      return new StoppableCheck(
-          _customRestClient.getInstanceStoppableCheck(baseUrl, customPayLoads),
+    MetricRegistry metrics = SharedMetricRegistries.getOrCreate(_namespace);
+
+    // Total requests metric is included as an attribute(Count) in timers
+    try (final Timer.Context timer = metrics.timer(CUSTOM_INSTANCE_CHECK_HTTP_REQUESTS_DURATION)
+        .time()) {
+      Map<String, Boolean> instanceStoppableCheck =
+          _customRestClient.getInstanceStoppableCheck(baseUrl, customPayLoads);
+      return new StoppableCheck(instanceStoppableCheck,
           StoppableCheck.Category.CUSTOM_INSTANCE_CHECK);
     } catch (IOException ex) {
       LOG.error("Custom client side instance level health check for {}/{} failed.", clusterId,
           instanceName, ex);
-      return new StoppableCheck(false, Arrays.asList(instanceName),
+      metrics.counter(CUSTOM_INSTANCE_CHECK_HTTP_REQUESTS_ERROR_TOTAL).inc();
+      return new StoppableCheck(false, Collections.singletonList(instanceName),
           StoppableCheck.Category.CUSTOM_INSTANCE_CHECK);
     }
   }
diff --git a/helix-rest/src/test/java/org/apache/helix/rest/common/TestHelixDataAccessorWrapper.java b/helix-rest/src/test/java/org/apache/helix/rest/common/TestHelixDataAccessorWrapper.java
index df15578..061bdf5 100644
--- a/helix-rest/src/test/java/org/apache/helix/rest/common/TestHelixDataAccessorWrapper.java
+++ b/helix-rest/src/test/java/org/apache/helix/rest/common/TestHelixDataAccessorWrapper.java
@@ -71,6 +71,10 @@ public class TestHelixDataAccessorWrapper {
     void setRestClient(CustomRestClient restClient) {
       _restClient = restClient;
     }
+
+    void setNamespace(String namespace) {
+      _namespace = namespace;
+    }
   }
 
   @BeforeMethod
@@ -80,6 +84,7 @@ public class TestHelixDataAccessorWrapper {
     when(_restConfig.getBaseUrl(anyString())).thenReturn("http://localhost:1000");
     _restClient = mock(CustomRestClient.class);
     _dataAccessor.setRestClient(_restClient);
+    _dataAccessor.setNamespace("test");
 
     when(_restClient.getPartitionStoppableCheck(anyString(), anyList(), anyMap()))
         .thenAnswer((invocationOnMock) -> {
diff --git a/helix-rest/src/test/java/org/apache/helix/rest/server/service/TestInstanceService.java b/helix-rest/src/test/java/org/apache/helix/rest/server/service/TestInstanceService.java
index f785124..ad10e82 100644
--- a/helix-rest/src/test/java/org/apache/helix/rest/server/service/TestInstanceService.java
+++ b/helix-rest/src/test/java/org/apache/helix/rest/server/service/TestInstanceService.java
@@ -40,6 +40,7 @@ import org.apache.helix.model.MasterSlaveSMD;
 import org.apache.helix.model.RESTConfig;
 import org.apache.helix.rest.client.CustomRestClient;
 import org.apache.helix.rest.common.HelixDataAccessorWrapper;
+import org.apache.helix.rest.common.HelixRestNamespace;
 import org.apache.helix.rest.server.json.instance.StoppableCheck;
 import org.apache.helix.zookeeper.datamodel.ZNRecord;
 import org.apache.zookeeper.data.Stat;
@@ -82,7 +83,8 @@ public class TestInstanceService {
   public void testGetInstanceStoppableCheckWhenHelixOwnCheckFail() throws IOException {
     Map<String, Boolean> failedCheck = ImmutableMap.of("FailCheck", false);
     InstanceService service =
-        new InstanceServiceImpl(_dataAccessor, _configAccessor, _customRestClient, false) {
+        new InstanceServiceImpl(_dataAccessor, _configAccessor, _customRestClient, false,
+            HelixRestNamespace.DEFAULT_NAMESPACE_NAME) {
           @Override
           protected Map<String, Boolean> getInstanceHealthStatus(String clusterId,
               String instanceName, List<HealthCheck> healthChecks) {
@@ -103,7 +105,8 @@ public class TestInstanceService {
   @Test
   public void testGetInstanceStoppableCheckWhenCustomInstanceCheckFail() throws IOException {
     InstanceService service =
-        new InstanceServiceImpl(_dataAccessor, _configAccessor, _customRestClient, false) {
+        new InstanceServiceImpl(_dataAccessor, _configAccessor, _customRestClient, false,
+            HelixRestNamespace.DEFAULT_NAMESPACE_NAME) {
           @Override
           protected Map<String, Boolean> getInstanceHealthStatus(String clusterId,
               String instanceName, List<HealthCheck> healthChecks) {
@@ -127,7 +130,8 @@ public class TestInstanceService {
   @Test
   public void testGetInstanceStoppableCheckConnectionRefused() throws IOException {
     InstanceService service =
-        new InstanceServiceImpl(_dataAccessor, _configAccessor, _customRestClient, false) {
+        new InstanceServiceImpl(_dataAccessor, _configAccessor, _customRestClient, false,
+            HelixRestNamespace.DEFAULT_NAMESPACE_NAME) {
           @Override
           protected Map<String, Boolean> getInstanceHealthStatus(String clusterId,
               String instanceName, List<HealthCheck> healthChecks) {
@@ -207,16 +211,14 @@ public class TestInstanceService {
 
     // Valid data only from ZK, pass the check
     InstanceService instanceServiceReadZK =
-        new MockInstanceServiceImpl(new HelixDataAccessorWrapper(zkHelixDataAccessor, _customRestClient),
-            _configAccessor, _customRestClient, false);
+        new MockInstanceServiceImpl(zkHelixDataAccessor, _configAccessor, _customRestClient, false);
     StoppableCheck stoppableCheck =
         instanceServiceReadZK.getInstanceStoppableCheck(TEST_CLUSTER, TEST_INSTANCE, jsonContent);
     Assert.assertTrue(stoppableCheck.isStoppable());
 
     // Even ZK data is valid. Skip ZK read should fail the test.
     InstanceService instanceServiceWithoutReadZK =
-        new MockInstanceServiceImpl(new HelixDataAccessorWrapper(zkHelixDataAccessor, _customRestClient),
-            _configAccessor, _customRestClient, true);
+        new MockInstanceServiceImpl(zkHelixDataAccessor, _configAccessor, _customRestClient, true);
     stoppableCheck = instanceServiceWithoutReadZK.getInstanceStoppableCheck(TEST_CLUSTER, TEST_INSTANCE, jsonContent);
     Assert.assertFalse(stoppableCheck.isStoppable());
   }
@@ -225,7 +227,8 @@ public class TestInstanceService {
   @Test(enabled = false)
   public void testGetInstanceStoppableCheckWhenPartitionsCheckFail() throws IOException {
     InstanceService service =
-        new InstanceServiceImpl(_dataAccessor, _configAccessor, _customRestClient, false) {
+        new InstanceServiceImpl(_dataAccessor, _configAccessor, _customRestClient, false,
+            HelixRestNamespace.DEFAULT_NAMESPACE_NAME) {
           @Override
           protected Map<String, Boolean> getInstanceHealthStatus(String clusterId,
               String instanceName, List<HealthCheck> healthChecks) {
@@ -245,9 +248,10 @@ public class TestInstanceService {
   }
 
   class MockInstanceServiceImpl extends InstanceServiceImpl {
-    MockInstanceServiceImpl(HelixDataAccessorWrapper dataAccessor, ConfigAccessor configAccessor,
+    MockInstanceServiceImpl(ZKHelixDataAccessor dataAccessor, ConfigAccessor configAccessor,
         CustomRestClient customRestClient, boolean skipZKRead) {
-      super(dataAccessor, configAccessor, customRestClient, skipZKRead);
+      super(dataAccessor, configAccessor, customRestClient, skipZKRead,
+          HelixRestNamespace.DEFAULT_NAMESPACE_NAME);
     }
 
     @Override