You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by ma...@apache.org on 2017/01/24 04:21:53 UTC
incubator-atlas git commit: ATLAS-1436: Metrics collection using
gremlin
Repository: incubator-atlas
Updated Branches:
refs/heads/master c8f3184fc -> 143c0f813
ATLAS-1436: Metrics collection using gremlin
Signed-off-by: Madhan Neethiraj <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/143c0f81
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/143c0f81
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/143c0f81
Branch: refs/heads/master
Commit: 143c0f813527c85edc30d3d589837fc20bf9bb04
Parents: c8f3184
Author: apoorvnaik <an...@hortonworks.com>
Authored: Mon Jan 23 12:04:54 2017 -0800
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Mon Jan 23 19:52:58 2017 -0800
----------------------------------------------------------------------
.../java/org/apache/atlas/AtlasAdminClient.java | 14 ++
.../java/org/apache/atlas/AtlasBaseClient.java | 11 ++
distro/src/conf/atlas-application.properties | 14 +-
.../atlas/model/metrics/AtlasMetrics.java | 92 +++++++++++++
.../apache/atlas/services/MetricsService.java | 134 +++++++++++++++++++
.../atlas/web/resources/AdminResource.java | 23 +++-
webapp/src/main/resources/spring-security.xml | 1 +
.../atlas/web/resources/AdminResourceTest.java | 4 +-
8 files changed, 288 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/143c0f81/client/src/main/java/org/apache/atlas/AtlasAdminClient.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/atlas/AtlasAdminClient.java b/client/src/main/java/org/apache/atlas/AtlasAdminClient.java
index 19f9575..b61b2bf 100644
--- a/client/src/main/java/org/apache/atlas/AtlasAdminClient.java
+++ b/client/src/main/java/org/apache/atlas/AtlasAdminClient.java
@@ -18,6 +18,8 @@
package org.apache.atlas;
+import org.apache.atlas.model.metrics.AtlasMetrics;
+import org.apache.atlas.type.AtlasType;
import org.apache.atlas.utils.AuthenticationUtil;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
@@ -44,6 +46,7 @@ import java.util.Arrays;
public class AtlasAdminClient {
private static final Option STATUS = new Option("status", false, "Get the status of an atlas instance");
+ private static final Option STATS = new Option("stats", false, "Get the metrics of an atlas instance");
private static final Options OPTIONS = new Options();
private static final int INVALID_OPTIONS_STATUS = 1;
@@ -51,6 +54,7 @@ public class AtlasAdminClient {
static {
OPTIONS.addOption(STATUS);
+ OPTIONS.addOption(STATS);
}
public static void main(String[] args) throws AtlasException, ParseException {
@@ -88,6 +92,16 @@ public class AtlasAdminClient {
System.err.println("Could not retrieve status of the server at " + Arrays.toString(atlasServerUri));
printStandardHttpErrorDetails(e);
}
+ } else if (commandLine.hasOption(STATS.getOpt())) {
+ try {
+ AtlasMetrics atlasMetrics = atlasClient.getAtlasMetrics();
+ String json = AtlasType.toJson(atlasMetrics);
+ System.out.println(json);
+ cmdStatus = 0;
+ } catch (AtlasServiceException e) {
+ System.err.println("Could not retrieve metrics of the server at " + Arrays.toString(atlasServerUri));
+ printStandardHttpErrorDetails(e);
+ }
} else {
System.err.println("Unsupported option. Refer to usage for valid options.");
printUsage(INVALID_OPTIONS_STATUS);
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/143c0f81/client/src/main/java/org/apache/atlas/AtlasBaseClient.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/atlas/AtlasBaseClient.java b/client/src/main/java/org/apache/atlas/AtlasBaseClient.java
index d055b78..8162900 100644
--- a/client/src/main/java/org/apache/atlas/AtlasBaseClient.java
+++ b/client/src/main/java/org/apache/atlas/AtlasBaseClient.java
@@ -26,6 +26,7 @@ import com.sun.jersey.api.client.config.DefaultClientConfig;
import com.sun.jersey.api.client.filter.HTTPBasicAuthFilter;
import com.sun.jersey.api.json.JSONConfiguration;
import com.sun.jersey.client.urlconnection.URLConnectionClientHandler;
+import org.apache.atlas.model.metrics.AtlasMetrics;
import org.apache.atlas.security.SecureClientUtils;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.utils.AuthenticationUtil;
@@ -54,10 +55,12 @@ public abstract class AtlasBaseClient {
public static final String TYPES = "types";
public static final String ADMIN_VERSION = "admin/version";
public static final String ADMIN_STATUS = "admin/status";
+ public static final String ADMIN_METRICS = "admin/metrics";
public static final String HTTP_AUTHENTICATION_ENABLED = "atlas.http.authentication.enabled";
//Admin operations
public static final APIInfo VERSION = new APIInfo(BASE_URI + ADMIN_VERSION, HttpMethod.GET, Response.Status.OK);
public static final APIInfo STATUS = new APIInfo(BASE_URI + ADMIN_STATUS, HttpMethod.GET, Response.Status.OK);
+ public static final APIInfo METRICS = new APIInfo(BASE_URI + ADMIN_METRICS, HttpMethod.GET, Response.Status.OK);
static final String JSON_MEDIA_TYPE = MediaType.APPLICATION_JSON + "; charset=UTF-8";
static final String UNKNOWN_STATUS = "Unknown status";
static final String ATLAS_CLIENT_HA_RETRIES_KEY = "atlas.client.ha.retries";
@@ -366,6 +369,14 @@ public abstract class AtlasBaseClient {
return result;
}
+ /**
+ * @return Return metrics of the service instance the client is pointing to
+ * @throws AtlasServiceException
+ */
+ public AtlasMetrics getAtlasMetrics () throws AtlasServiceException {
+ return callAPI(METRICS, AtlasMetrics.class, null);
+ }
+
boolean isRetryableException(ClientHandlerException che) {
return che.getCause().getClass().equals(IOException.class)
|| che.getCause().getClass().equals(ConnectException.class);
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/143c0f81/distro/src/conf/atlas-application.properties
----------------------------------------------------------------------
diff --git a/distro/src/conf/atlas-application.properties b/distro/src/conf/atlas-application.properties
index 6fa066b..7f79ad7 100755
--- a/distro/src/conf/atlas-application.properties
+++ b/distro/src/conf/atlas-application.properties
@@ -205,5 +205,15 @@ atlas.feature.taxonomy.enable=true
#atlas.sso.knox.providerurl=https://<knox gateway ip>:8443/gateway/knoxsso/api/v1/websso
#atlas.sso.knox.publicKey=
-
-
+############ Atlas Metric/Stats configs ################
+# Format: atlas.metric.query.<key>.<name>
+#atlas.metric.query.general.typeCount=
+#atlas.metric.query.general.typeUnusedCount=
+#atlas.metric.query.general.entityCount=
+#atlas.metric.query.general.tagCount=
+#atlas.metric.query.general.entityDeleted=
+#
+#atlas.metric.query.entity.typeEntities=
+#atlas.metric.query.entity.entityTagged=
+#
+#atlas.metric.query.tags.entityTags=
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/143c0f81/intg/src/main/java/org/apache/atlas/model/metrics/AtlasMetrics.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/model/metrics/AtlasMetrics.java b/intg/src/main/java/org/apache/atlas/model/metrics/AtlasMetrics.java
new file mode 100644
index 0000000..602cdb4
--- /dev/null
+++ b/intg/src/main/java/org/apache/atlas/model/metrics/AtlasMetrics.java
@@ -0,0 +1,92 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.model.metrics;
+
+
+import org.codehaus.jackson.annotate.JsonAutoDetect;
+import org.codehaus.jackson.annotate.JsonIgnore;
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.codehaus.jackson.annotate.JsonAutoDetect.Visibility.NONE;
+import static org.codehaus.jackson.annotate.JsonAutoDetect.Visibility.PUBLIC_ONLY;
+
+/**
+ * Atlas metrics
+ */
+@JsonAutoDetect(getterVisibility=PUBLIC_ONLY, setterVisibility=PUBLIC_ONLY, fieldVisibility=NONE)
+@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL)
+@JsonIgnoreProperties(ignoreUnknown=true)
+public class AtlasMetrics {
+ private Map<String, Map<String, Number>> data;
+
+ public AtlasMetrics() {
+ setData(null);
+ }
+
+ public AtlasMetrics(Map<String, Map<String, Number>> data) {
+ setData(data);
+ }
+
+ public AtlasMetrics(AtlasMetrics other) {
+ if (other != null) {
+ setData(other.getData());
+ }
+ }
+
+ public Map<String, Map<String, Number>> getData() {
+ return data;
+ }
+
+ public void setData(Map<String, Map<String, Number>> data) {
+ this.data = data;
+ }
+
+ @JsonIgnore
+ public void addData(String groupKey, String key, Integer value) {
+ Map<String, Map<String, Number>> data = this.data;
+ if (data == null) {
+ data = new HashMap<>();
+ }
+ Map<String, Number> metricMap = data.get(groupKey);
+ if (metricMap == null) {
+ metricMap = new HashMap<>();
+ data.put(groupKey, metricMap);
+ }
+ metricMap.put(key, value);
+ setData(data);
+ }
+
+ @JsonIgnore
+ public Number getMetric(String groupKey, String key) {
+ Map<String, Map<String, Number>> data = this.data;
+ if (data == null) {
+ return null;
+ } else {
+ Map<String, Number> metricMap = data.get(groupKey);
+ if (metricMap == null || metricMap.isEmpty()) {
+ return null;
+ } else {
+ return metricMap.get(key);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/143c0f81/repository/src/main/java/org/apache/atlas/services/MetricsService.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/services/MetricsService.java b/repository/src/main/java/org/apache/atlas/services/MetricsService.java
new file mode 100644
index 0000000..855e402
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/services/MetricsService.java
@@ -0,0 +1,134 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.services;
+
+import com.google.inject.Singleton;
+import org.apache.atlas.ApplicationProperties;
+import org.apache.atlas.AtlasException;
+import org.apache.atlas.model.metrics.AtlasMetrics;
+import org.apache.atlas.repository.graph.AtlasGraphProvider;
+import org.apache.atlas.repository.graphdb.AtlasGraph;
+import org.apache.commons.configuration.Configuration;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.script.ScriptException;
+import java.util.List;
+import java.util.Map;
+
+@Singleton
+public class MetricsService {
+ private static final Logger LOG = LoggerFactory.getLogger(MetricsService.class);
+
+ public static final String METRIC_QUERY_PREFIX = "atlas.metric.query.";
+
+ public static final String TYPE = "type";
+ public static final String ENTITY = "entity";
+ public static final String TAG = "tag";
+ public static final String GENERAL = "general";
+
+ public static final String METRIC_TYPE_COUNT = TYPE + "Count";
+ public static final String METRIC_TYPE_UNUSED_COUNT = TYPE + "UnusedCount";
+ public static final String METRIC_TYPE_ENTITIES = TYPE + "Entities";
+
+ public static final String METRIC_ENTITY_COUNT = ENTITY + "Count";
+ public static final String METRIC_ENTITY_DELETED = ENTITY + "Deleted";
+ public static final String METRIC_TAGGED_ENTITIES = ENTITY + "Tagged";
+ public static final String METRIC_TAGS_PER_ENTITY = ENTITY + "Tags";
+
+ public static final String METRIC_TAG_COUNT = TAG + "Count";
+ public static final String METRIC_ENTITIES_PER_TAG = TAG + "Entities";
+
+ private static AtlasGraph atlasGraph;
+ private static Configuration configuration;
+
+ public MetricsService() throws AtlasException {
+ atlasGraph = AtlasGraphProvider.getGraphInstance();
+ configuration = ApplicationProperties.get();
+ }
+
+ @SuppressWarnings("unchecked")
+ public AtlasMetrics getMetrics() {
+ AtlasMetrics metrics = new AtlasMetrics();
+ for (MetricQuery metricQuery : MetricQuery.values()) {
+ try {
+ Object result = atlasGraph.executeGremlinScript(metricQuery.query, false);
+ if (result instanceof Number) {
+ metrics.addData(metricQuery.type, metricQuery.name, ((Number) result).intValue());
+ } else if (result instanceof List) {
+ for (Map resultMap : (List<Map>) result) {
+ metrics.addData(metricQuery.type, (String) resultMap.get("key"), ((Number) resultMap.get("value")).intValue());
+ }
+ } else {
+ LOG.warn("Unhandled return type {} for {}. Ignoring", result.getClass().getSimpleName(), metricQuery);
+ }
+ } catch (ScriptException e) {
+ LOG.error("Gremlin execution failed for metric {}", metricQuery, e);
+ }
+ }
+
+ return metrics;
+ }
+
+ /**
+ * MetricQuery enum has the capability of reading the queries from the externalized config.
+ *
+ * The default behavior is to read from the properties and override the statically type query if the configured
+ * query is not blank/empty.
+ */
+ enum MetricQuery {
+ TYPE_COUNT(GENERAL, METRIC_TYPE_COUNT, "g.V().has('__type', 'typeSystem').filter({it.'__type.category'.name() != 'TRAIT'}).count()"),
+ UNUSED_TYPE_COUNT(GENERAL, METRIC_TYPE_UNUSED_COUNT, "g.V('__type', 'typeSystem').filter({ it.'__type.category'.name() != 'TRAIT' && it.inE.count() == 0}).count()"),
+ ENTITY_COUNT(GENERAL, METRIC_ENTITY_COUNT, "g.V().has('__superTypeNames', T.in, ['Referenceable']).count()"),
+ TAGS_COUNT(GENERAL, METRIC_TAG_COUNT, "g.V().has('__type', 'typeSystem').filter({it.'__type.category'.name() == 'TRAIT'}).count()"),
+ DELETED_ENTITY_COUNT(GENERAL, METRIC_ENTITY_DELETED, "g.V().has('__superTypeNames', T.in, ['Referenceable']).has('__status', 'DELETED').count()"),
+
+ ENTITIES_PER_TYPE(ENTITY, METRIC_TYPE_ENTITIES, "g.V().has('__type', 'typeSystem').has('__type.name').filter({it.'__type.category'.name() != 'TRAIT'}).transform{[key: it.'__type.name', value: it.inE.count()]}.dedup().toList()"),
+ TAGGED_ENTITIES(ENTITY, METRIC_TAGGED_ENTITIES, "g.V().has('__superTypeNames', T.in, ['Referenceable']).has('__traitNames').count()"),
+
+ TAGS_PER_ENTITY(TAG, METRIC_TAGS_PER_ENTITY, "g.V().has('__superTypeNames', T.in, ['Referenceable']).has('__traitNames').transform{[ key: it.'Referenceable.qualifiedName', value: it.'__traitNames'.size()]}.dedup().toList()"),
+ ;
+ private String type;
+ private String name;
+ private String query;
+
+ private static String getQuery(String type, String name) {
+ String metricQueryKey = METRIC_QUERY_PREFIX + type + "." + name;
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Looking for configured query {}", metricQueryKey);
+ }
+ return configuration.getString(metricQueryKey, "");
+ }
+
+ MetricQuery(String type, String name, String query) {
+ this.type = type;
+ this.name = name;
+ String configuredQuery = getQuery(type, name);
+ this.query = StringUtils.isNotEmpty(configuredQuery) ? configuredQuery : query;
+ }
+
+ @Override
+ public String toString() {
+ return "MetricQuery{" + "type='" + type + '\'' +
+ ", name='" + name + '\'' +
+ ", query='" + query + '\'' +
+ '}';
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/143c0f81/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java b/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java
index 3b4155c..54c46a8 100755
--- a/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java
+++ b/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java
@@ -20,6 +20,8 @@ package org.apache.atlas.web.resources;
import com.google.inject.Inject;
import org.apache.atlas.AtlasClient;
+import org.apache.atlas.model.metrics.AtlasMetrics;
+import org.apache.atlas.services.MetricsService;
import org.apache.atlas.authorize.AtlasActionTypes;
import org.apache.atlas.authorize.AtlasResourceTypes;
import org.apache.atlas.authorize.simple.AtlasAuthorizationUtils;
@@ -65,10 +67,12 @@ public class AdminResource {
private static final String isEntityCreateAllowed = "atlas.entity.create.allowed";
private Response version;
private ServiceState serviceState;
+ private MetricsService metricsService;
@Inject
- public AdminResource(ServiceState serviceState) {
+ public AdminResource(ServiceState serviceState, MetricsService metricsService) {
this.serviceState = serviceState;
+ this.metricsService = metricsService;
}
/**
@@ -223,4 +227,21 @@ public class AdminResource {
return response;
}
+
+ @GET
+ @Path("metrics")
+ @Produces(Servlets.JSON_MEDIA_TYPE)
+ public AtlasMetrics getMetrics() {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> AdminResource.getMetrics()");
+ }
+
+ AtlasMetrics metrics = metricsService.getMetrics();
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== AdminResource.getMetrics()");
+ }
+
+ return metrics;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/143c0f81/webapp/src/main/resources/spring-security.xml
----------------------------------------------------------------------
diff --git a/webapp/src/main/resources/spring-security.xml b/webapp/src/main/resources/spring-security.xml
index 4ed88ec..208c325 100644
--- a/webapp/src/main/resources/spring-security.xml
+++ b/webapp/src/main/resources/spring-security.xml
@@ -33,6 +33,7 @@
<security:http pattern="/js/**" security="none" />
<security:http pattern="/ieerror.html" security="none" />
<security:http pattern="/api/atlas/admin/status" security="none" />
+ <security:http pattern="/api/atlas/admin/metrics" security="none" />
<security:http disable-url-rewriting="true"
use-expressions="true" create-session="always"
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/143c0f81/webapp/src/test/java/org/apache/atlas/web/resources/AdminResourceTest.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/atlas/web/resources/AdminResourceTest.java b/webapp/src/test/java/org/apache/atlas/web/resources/AdminResourceTest.java
index eb2b2f6..d73190e 100644
--- a/webapp/src/test/java/org/apache/atlas/web/resources/AdminResourceTest.java
+++ b/webapp/src/test/java/org/apache/atlas/web/resources/AdminResourceTest.java
@@ -48,7 +48,7 @@ public class AdminResourceTest {
when(serviceState.getState()).thenReturn(ServiceState.ServiceStateValue.ACTIVE);
- AdminResource adminResource = new AdminResource(serviceState);
+ AdminResource adminResource = new AdminResource(serviceState, null);
Response response = adminResource.getStatus();
assertEquals(response.getStatus(), HttpServletResponse.SC_OK);
JSONObject entity = (JSONObject) response.getEntity();
@@ -59,7 +59,7 @@ public class AdminResourceTest {
public void testResourceGetsValueFromServiceState() throws JSONException {
when(serviceState.getState()).thenReturn(ServiceState.ServiceStateValue.PASSIVE);
- AdminResource adminResource = new AdminResource(serviceState);
+ AdminResource adminResource = new AdminResource(serviceState, null);
Response response = adminResource.getStatus();
verify(serviceState).getState();