You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ka...@apache.org on 2017/09/04 13:44:18 UTC
[07/50] [abbrv] kylin git commit: KYLIN-2776 Using dropwizard as
default metric framework
KYLIN-2776 Using dropwizard as default metric framework
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8e078847
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8e078847
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8e078847
Branch: refs/heads/2622-2764
Commit: 8e078847a42b4f7853a2c1a75afa405c09c4ea7b
Parents: d85f6ab
Author: yiming.xu <10...@qq.com>
Authored: Wed Aug 23 13:12:58 2017 +0800
Committer: Hongbin Ma <ma...@kyligence.io>
Committed: Wed Aug 23 13:49:23 2017 +0800
----------------------------------------------------------------------
core-common/pom.xml | 17 +
.../apache/kylin/common/KylinConfigBase.java | 20 +
.../kylin/common/metrics/common/Metrics.java | 162 ++++++
.../common/metrics/common/MetricsConstant.java | 38 ++
.../common/metrics/common/MetricsFactory.java | 62 +++
.../common/metrics/common/MetricsScope.java | 33 ++
.../common/metrics/common/MetricsVariable.java | 27 +
.../kylin/common/metrics/common/Metricss.java | 40 ++
.../metrics/metrics2/CodahaleMetrics.java | 508 +++++++++++++++++++
.../metrics/metrics2/CodahaleReporter.java | 31 ++
.../metrics2/ConsoleMetricsReporter.java | 51 ++
.../metrics/metrics2/JmxMetricsReporter.java | 55 ++
.../metrics2/JsonFileMetricsReporter.java | 136 +++++
.../metrics2/KylinObjectNameFactory.java | 68 +++
.../metrics2/MetricVariableRatioGauge.java | 47 ++
.../metrics/metrics2/Metrics2Reporter.java | 60 +++
.../metrics/metrics2/MetricsReporting.java | 26 +
.../common/metrics/perflog/IPerfLogger.java | 48 ++
.../common/metrics/perflog/PerfLogger.java | 158 ++++++
.../metrics/perflog/PerfLoggerFactory.java | 56 ++
.../metrics/perflog/SimplePerfLogger.java | 73 +++
pom.xml | 3 +-
.../kylin/rest/init/InitialTaskManager.java | 4 +-
.../kylin/rest/metrics/QueryMetrics2Facade.java | 106 ++++
.../apache/kylin/rest/service/QueryService.java | 2 +
.../kylin/rest/metrics/QueryMetrics2Test.java | 128 +++++
26 files changed, 1957 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/8e078847/core-common/pom.xml
----------------------------------------------------------------------
diff --git a/core-common/pom.xml b/core-common/pom.xml
index 8852743..d58de36 100644
--- a/core-common/pom.xml
+++ b/core-common/pom.xml
@@ -68,6 +68,23 @@
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
+
+ <dependency>
+ <groupId>io.dropwizard.metrics</groupId>
+ <artifactId>metrics-jvm</artifactId>
+ <version>${dropwizard.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>io.dropwizard.metrics</groupId>
+ <artifactId>metrics-json</artifactId>
+ <version>${dropwizard.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.github.joshelser</groupId>
+ <artifactId>dropwizard-metrics-hadoop-metrics2-reporter</artifactId>
+ <version>0.1.2</version>
+ </dependency>
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/kylin/blob/8e078847/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index b622825..99b9bb2 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -1173,4 +1173,24 @@ abstract public class KylinConfigBase implements Serializable {
public boolean isWebCrossDomainEnabled() {
return Boolean.parseBoolean(getOptional("kylin.web.cross-domain-enabled", "true"));
}
+
+ /**
+ * metric
+ */
+ public String getCoadhaleMetricReportClassesName() {
+ return getOptional("kylin.metric.codahale-metric-report-classes",
+ "org.apache.kylin.common.metrics.metrics2.JsonFileMetricsReporter,org.apache.kylin.common.metrics.metrics2.JmxMetricsReporter");
+ }
+
+ public String getMetricFileLocation() {
+ return getOptional("kylin.metric.file.location", "/tmp/report.json");
+ }
+
+ public Long getJsonFileMetricsReporterInterval() {
+ return Long.parseLong(getOptional("kylin.metric.json-file-metric-reporter.interval", "5000"));
+ }
+
+ public String getPerfLoggerClassName() {
+ return getOptional("kylin.metric.perf-logger.class", "org.apache.kylin.common.metrics.perflog.PerfLogger");
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/8e078847/core-common/src/main/java/org/apache/kylin/common/metrics/common/Metrics.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/metrics/common/Metrics.java b/core-common/src/main/java/org/apache/kylin/common/metrics/common/Metrics.java
new file mode 100644
index 0000000..dcda3cd
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/metrics/common/Metrics.java
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+package org.apache.kylin.common.metrics.common;
+
+import com.codahale.metrics.Counter;
+import com.codahale.metrics.Histogram;
+import com.codahale.metrics.Meter;
+import com.codahale.metrics.Timer;
+
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Generic Metics interface.
+ */
+public interface Metrics {
+
+ /**
+ * Deinitializes the Metrics system.
+ */
+ public void close() throws Exception;
+
+ /**
+ *
+ * @param name starts a scope of a given name. Scopes is stored as thread-local variable.
+ */
+ public void startStoredScope(String name);
+
+ /**
+ * Closes the stored scope of a given name.
+ * Note that this must be called on the same thread as where the scope was started.
+ * @param name
+ */
+ public void endStoredScope(String name);
+
+ /**
+ * Create scope with given name and returns it.
+ * @param name
+ * @return
+ */
+ public MetricsScope createScope(String name);
+
+ /**
+ * Close the given scope.
+ * @param scope
+ */
+ public void endScope(MetricsScope scope);
+
+ //Counter-related methods
+
+ /**
+ * Increments a counter of the given name by 1.
+ * @param name
+ * @return
+ */
+ public Long incrementCounter(String name);
+
+ /**
+ * Increments a counter of the given name by "increment"
+ * @param name
+ * @param increment
+ * @return
+ */
+ public Long incrementCounter(String name, long increment);
+
+ /**
+ * Decrements a counter of the given name by 1.
+ * @param name
+ * @return
+ */
+ public Long decrementCounter(String name);
+
+ /**
+ * Decrements a counter of the given name by "decrement"
+ * @param name
+ * @param decrement
+ * @return
+ */
+ public Long decrementCounter(String name, long decrement);
+
+ /**
+ * Adds a metrics-gauge to track variable. For example, number of open database connections.
+ * @param name name of gauge
+ * @param variable variable to track.
+ */
+ public void addGauge(String name, final MetricsVariable variable);
+
+ /**
+ * Add a ratio metric to track the correlation between two variables
+ * @param name name of the ratio gauge
+ * @param numerator numerator of the ratio
+ * @param denominator denominator of the ratio
+ */
+ public void addRatio(String name, MetricsVariable<Integer> numerator, MetricsVariable<Integer> denominator);
+
+ /**
+ * Mark an event occurance for a meter. Meters measure the rate of an event and track
+ * 1/5/15 minute moving averages
+ * @param name name of the meter
+ */
+ public void markMeter(String name);
+
+ /**
+ *
+ * @param name name of the Timer
+ * @param duration
+ * @param unit
+ */
+
+ public void updateTimer(String name, long duration, TimeUnit unit);
+
+
+ /**
+ *
+ * @param name name of the histogram
+ * @param count
+ */
+ public void updateHistogram(String name, long count);
+
+ /**
+ *
+ * @param name name of the time
+ * @return
+ */
+ public Timer getTimer(String name);
+
+ /**
+ *
+ * @param name name of the counter
+ * @return
+ */
+ public Counter getCounter(String name);
+
+ /**
+ *
+ * @param name name of the time
+ * @return
+ */
+ public Histogram getHistogram(String name);
+
+ /**
+ *
+ * @param name name of the meter
+ * @return
+ */
+ public Meter getMeter(String name);
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/8e078847/core-common/src/main/java/org/apache/kylin/common/metrics/common/MetricsConstant.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/metrics/common/MetricsConstant.java b/core-common/src/main/java/org/apache/kylin/common/metrics/common/MetricsConstant.java
new file mode 100644
index 0000000..49c8d8e
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/metrics/common/MetricsConstant.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+package org.apache.kylin.common.metrics.common;
+
+/**
+ * This class defines some metrics generated by Hive processes.
+ */
+public class MetricsConstant {
+
+ public static final String API_PREFIX = "api_";
+ public static final String ACTIVE_CALLS = "metrics:name=active_calls,method=";
+ public static final String CALLS = "metrics:name=calls,method=";
+
+ public static final String QUERY_SUCCESS_COUNT = "QuerySuccessCount";
+ public static final String QUERY_FAIL_COUNT = "QueryFailCount";
+ public static final String QUERY_CACHE_COUNT = "QueryCacheCount";
+ public static final String QUERY_COUNT = "QueryCount";
+ public static final String QUERY_DURATION = "QueryDuration";
+ public static final String QUERY_RESULT_ROWCOUNT = "QueryResultRowcount";
+ public static final String QUERY_SCAN_ROWCOUNT = "QueryScanRowcount";
+ public static final String TOTAL = "total";
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/8e078847/core-common/src/main/java/org/apache/kylin/common/metrics/common/MetricsFactory.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/metrics/common/MetricsFactory.java b/core-common/src/main/java/org/apache/kylin/common/metrics/common/MetricsFactory.java
new file mode 100644
index 0000000..ff0ab7d
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/metrics/common/MetricsFactory.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common.metrics.common;
+
+
+import org.apache.kylin.common.metrics.metrics2.CodahaleMetrics;
+
+/**
+ * Class that manages a static Metric instance for this process.
+ */
+public class MetricsFactory {
+
+ //Volatile ensures that static access returns Metrics instance in fully-initialized state.
+ //Alternative is to synchronize static access, which has performance penalties.
+ private volatile static Metrics metrics;
+ static {
+ MetricsFactory.init();
+ }
+
+ /**
+ * Initializes static Metrics instance.
+ */
+ public synchronized static void init() {
+ if (metrics == null) {
+ Class metricsClass = MetricsFactory.class;
+ metrics = new CodahaleMetrics();
+ }
+ }
+
+ /**
+ * Returns static Metrics instance, null if not initialized or closed.
+ */
+ public static Metrics getInstance() {
+ return metrics;
+ }
+
+ /**
+ * Closes and removes static Metrics instance.
+ */
+ public synchronized static void close() throws Exception {
+ if (metrics != null) {
+ metrics.close();
+ metrics = null;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/8e078847/core-common/src/main/java/org/apache/kylin/common/metrics/common/MetricsScope.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/metrics/common/MetricsScope.java b/core-common/src/main/java/org/apache/kylin/common/metrics/common/MetricsScope.java
new file mode 100644
index 0000000..8a72073
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/metrics/common/MetricsScope.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+package org.apache.kylin.common.metrics.common;
+
+/**
+ * Metrics Scope to represent duration of an event.
+ *
+ * Implementation can capture information like the average duration of open scopes,
+ * number of open scopes, number of completed scopes.
+ *
+ * Scopes are created via the Metrics framework (see Metrics#createScope or Metrics$createStoredScope)
+ *
+ * Scope may be stored by the Metrics framework via 'storedScope' concept for further reference.
+ *
+ * In either case, it is the caller's responsibility to end the scope via the Metrics framework (see Metrics#endScope)
+ */
+public interface MetricsScope {
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/8e078847/core-common/src/main/java/org/apache/kylin/common/metrics/common/MetricsVariable.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/metrics/common/MetricsVariable.java b/core-common/src/main/java/org/apache/kylin/common/metrics/common/MetricsVariable.java
new file mode 100644
index 0000000..3273c7f
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/metrics/common/MetricsVariable.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common.metrics.common;
+
+/**
+ * Interface for metrics variables. For example a the database service could expose the number of
+ * currently active connections.
+ */
+public interface MetricsVariable<T> {
+ public T getValue();
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/8e078847/core-common/src/main/java/org/apache/kylin/common/metrics/common/Metricss.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/metrics/common/Metricss.java b/core-common/src/main/java/org/apache/kylin/common/metrics/common/Metricss.java
new file mode 100644
index 0000000..105ab5d
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/metrics/common/Metricss.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common.metrics.common;
+
+public final class Metricss {
+ public final static String METRICS = "metrics:";
+ public final static String PROJECT_TEMPLATE = METRICS + "project=%s";
+ public final static String CUBE_TEMPLATE = METRICS + "project=%s,cube=%s";
+
+
+ public static String buildMetricName(String prefix, String name) {
+ return String.format(prefix+",name=%s", name);
+ }
+
+ public static String buildCubeMetricPrefix(String project) {
+ return String.format(PROJECT_TEMPLATE, project);
+ }
+
+ public static String buildCubeMetricPrefix(String project, String cube) {
+ return String.format(CUBE_TEMPLATE, project, cube);
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/8e078847/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/CodahaleMetrics.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/CodahaleMetrics.java b/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/CodahaleMetrics.java
new file mode 100644
index 0000000..d7e2f8c
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/CodahaleMetrics.java
@@ -0,0 +1,508 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common.metrics.metrics2;
+
+import java.io.Closeable;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+
+import org.apache.commons.lang3.ClassUtils;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.metrics.common.Metrics;
+import org.apache.kylin.common.metrics.common.MetricsConstant;
+import org.apache.kylin.common.metrics.common.MetricsScope;
+import org.apache.kylin.common.metrics.common.MetricsVariable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.codahale.metrics.Counter;
+import com.codahale.metrics.ExponentiallyDecayingReservoir;
+import com.codahale.metrics.Gauge;
+import com.codahale.metrics.Histogram;
+import com.codahale.metrics.Meter;
+import com.codahale.metrics.Metric;
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.MetricSet;
+import com.codahale.metrics.Timer;
+import com.codahale.metrics.json.MetricsModule;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Splitter;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.google.common.collect.Lists;
+
+/**
+ * Codahale-backed Metrics implementation.
+ */
+public class CodahaleMetrics implements Metrics {
+
+ public static final Logger LOGGER = LoggerFactory.getLogger(CodahaleMetrics.class);
+
+ public final MetricRegistry metricRegistry = new MetricRegistry();
+ private final Lock timersLock = new ReentrantLock();
+ private final Lock countersLock = new ReentrantLock();
+ private final Lock gaugesLock = new ReentrantLock();
+ private final Lock metersLock = new ReentrantLock();
+ private final Lock histogramLock = new ReentrantLock();
+ private final Set<Closeable> reporters = new HashSet<Closeable>();
+ private final ThreadLocal<HashMap<String, CodahaleMetricsScope>> threadLocalScopes = new ThreadLocal<HashMap<String, CodahaleMetricsScope>>() {
+ @Override
+ protected HashMap<String, CodahaleMetricsScope> initialValue() {
+ return new HashMap<String, CodahaleMetricsScope>();
+ }
+ };
+ private LoadingCache<String, Timer> timers;
+ private LoadingCache<String, Counter> counters;
+ private LoadingCache<String, Meter> meters;
+ private LoadingCache<String, Histogram> histograms;
+ private ConcurrentHashMap<String, Gauge> gauges;
+ private KylinConfig conf;
+
+ public CodahaleMetrics() {
+ this.conf = conf;
+ //Codahale artifacts are lazily-created.
+ timers = CacheBuilder.newBuilder().build(new CacheLoader<String, Timer>() {
+ @Override
+ public Timer load(String key) {
+ Timer timer = new Timer(new ExponentiallyDecayingReservoir());
+ metricRegistry.register(key, timer);
+ return timer;
+ }
+ });
+ counters = CacheBuilder.newBuilder().build(new CacheLoader<String, Counter>() {
+ @Override
+ public Counter load(String key) {
+ Counter counter = new Counter();
+ metricRegistry.register(key, counter);
+ return counter;
+ }
+ });
+ meters = CacheBuilder.newBuilder().build(new CacheLoader<String, Meter>() {
+ @Override
+ public Meter load(String key) {
+ Meter meter = new Meter();
+ metricRegistry.register(key, meter);
+ return meter;
+ }
+ });
+ histograms = CacheBuilder.newBuilder().build(new CacheLoader<String, Histogram>() {
+ @Override
+ public Histogram load(String key) {
+ Histogram histogram = new Histogram(new ExponentiallyDecayingReservoir());
+ metricRegistry.register(key, histogram);
+ return histogram;
+ }
+ });
+ gauges = new ConcurrentHashMap<String, Gauge>();
+ //register JVM metrics
+ // registerAll("gc", new GarbageCollectorMetricSet());
+ // registerAll("buffers", new BufferPoolMetricSet(ManagementFactory.getPlatformMBeanServer()));
+ // registerAll("memory", new MemoryUsageGaugeSet());
+ // registerAll("threads", new ThreadStatesGaugeSet());
+ // registerAll("classLoadingz", new ClassLoadingGaugeSet());
+
+ //initialize reporters
+ initReporting();
+ }
+
+ @Override
+ public void close() throws Exception {
+ if (reporters != null) {
+ for (Closeable reporter : reporters) {
+ reporter.close();
+ }
+ }
+ for (Map.Entry<String, Metric> metric : metricRegistry.getMetrics().entrySet()) {
+ metricRegistry.remove(metric.getKey());
+ }
+ timers.invalidateAll();
+ counters.invalidateAll();
+ meters.invalidateAll();
+ }
+
+ @Override
+ public void startStoredScope(String name) {
+ if (threadLocalScopes.get().containsKey(name)) {
+ threadLocalScopes.get().get(name).open();
+ } else {
+ threadLocalScopes.get().put(name, new CodahaleMetricsScope(name));
+ }
+ }
+
+ public MetricsScope getStoredScope(String name) throws IllegalArgumentException {
+ if (threadLocalScopes.get().containsKey(name)) {
+ return threadLocalScopes.get().get(name);
+ } else {
+ throw new IllegalArgumentException("No metrics scope named " + name);
+ }
+ }
+
+ @Override
+ public void endStoredScope(String name) {
+ if (threadLocalScopes.get().containsKey(name)) {
+ threadLocalScopes.get().get(name).close();
+ threadLocalScopes.get().remove(name);
+ }
+ }
+
+ public MetricsScope createScope(String name) {
+ return new CodahaleMetricsScope(name);
+ }
+
+ public void endScope(MetricsScope scope) {
+ ((CodahaleMetricsScope) scope).close();
+ }
+
+ @Override
+ public Long incrementCounter(String name) {
+ return incrementCounter(name, 1L);
+ }
+
+ @Override
+ public Long incrementCounter(String name, long increment) {
+ String key = name;
+ try {
+ countersLock.lock();
+ counters.get(key).inc(increment);
+ return counters.get(key).getCount();
+ } catch (ExecutionException ee) {
+ throw new IllegalStateException("Error retrieving counter from the metric registry ", ee);
+ } finally {
+ countersLock.unlock();
+ }
+ }
+
+ @Override
+ public Long decrementCounter(String name) {
+ return decrementCounter(name, 1L);
+ }
+
+ @Override
+ public Long decrementCounter(String name, long decrement) {
+ String key = name;
+ try {
+ countersLock.lock();
+ counters.get(key).dec(decrement);
+ return counters.get(key).getCount();
+ } catch (ExecutionException ee) {
+ throw new IllegalStateException("Error retrieving counter from the metric registry ", ee);
+ } finally {
+ countersLock.unlock();
+ }
+ }
+
+ @Override
+ public void addGauge(String name, final MetricsVariable variable) {
+ Gauge gauge = new Gauge() {
+ @Override
+ public Object getValue() {
+ return variable.getValue();
+ }
+ };
+ addGaugeInternal(name, gauge);
+ }
+
+ @Override
+ public void addRatio(String name, MetricsVariable<Integer> numerator, MetricsVariable<Integer> denominator) {
+ Preconditions.checkArgument(numerator != null, "Numerator must not be null");
+ Preconditions.checkArgument(denominator != null, "Denominator must not be null");
+
+ MetricVariableRatioGauge gauge = new MetricVariableRatioGauge(numerator, denominator);
+ addGaugeInternal(name, gauge);
+ }
+
+ private void addGaugeInternal(String name, Gauge gauge) {
+ try {
+ gaugesLock.lock();
+ gauges.put(name, gauge);
+ // Metrics throws an Exception if we don't do this when the key already exists
+ if (metricRegistry.getGauges().containsKey(name)) {
+ LOGGER.warn("A Gauge with name [" + name + "] already exists. "
+ + " The old gauge will be overwritten, but this is not recommended");
+ metricRegistry.remove(name);
+ }
+ metricRegistry.register(name, gauge);
+ } finally {
+ gaugesLock.unlock();
+ }
+ }
+
+ @Override
+ public void markMeter(String name) {
+ String key = name;
+ try {
+ metersLock.lock();
+ Meter meter = meters.get(name);
+ meter.mark();
+ } catch (ExecutionException e) {
+ throw new IllegalStateException("Error retrieving meter " + name + " from the metric registry ", e);
+ } finally {
+ metersLock.unlock();
+ }
+ }
+
+ @Override
+ public void updateHistogram(String name, long count) {
+ try {
+ histogramLock.lock();
+ Histogram histogram = histograms.get(name);
+ histogram.update(count);
+ } catch (ExecutionException e) {
+ throw new IllegalStateException("Error retrieving meter " + name + " from the metric registry ", e);
+ } finally {
+ histogramLock.unlock();
+ }
+ }
+
+ @Override
+ public void updateTimer(String name, long duration, TimeUnit unit) {
+ String key = name;
+ try {
+ timersLock.lock();
+ Timer timer = timers.get(key);
+ timer.update(duration, unit);
+ } catch (ExecutionException e) {
+ throw new IllegalStateException("Error retrieving timer " + name + " from the metric registry ", e);
+ } finally {
+ timersLock.unlock();
+ }
+ }
+
+ // This method is necessary to synchronize lazy-creation to the timers.
+ public Timer getTimer(String name) {
+ try {
+ timersLock.lock();
+ return timers.get(name);
+ } catch (ExecutionException e) {
+ throw new IllegalStateException("Error retrieving timer " + name + " from the metric registry ", e);
+ } finally {
+ timersLock.unlock();
+ }
+ }
+
+ public Counter getCounter(String name) {
+ try {
+ countersLock.lock();
+ return counters.get(name);
+ } catch (ExecutionException e) {
+ throw new IllegalStateException("Error retrieving counter " + name + " from the metric registry ", e);
+ } finally {
+ countersLock.unlock();
+ }
+ }
+
+ public Histogram getHistogram(String name) {
+ try {
+ histogramLock.lock();
+ return histograms.get(name);
+ } catch (ExecutionException e) {
+ throw new IllegalStateException("Error retrieving histogram " + name + " from the metric registry ", e);
+ } finally {
+ histogramLock.unlock();
+ }
+ }
+
+ public Meter getMeter(String name) {
+ try {
+ metersLock.lock();
+ return meters.get(name);
+ } catch (ExecutionException e) {
+ throw new IllegalStateException("Error retrieving meter " + name + " from the metric registry ", e);
+ } finally {
+ metersLock.unlock();
+ }
+ }
+
+ private void registerAll(String prefix, MetricSet metricSet) {
+ for (Map.Entry<String, Metric> entry : metricSet.getMetrics().entrySet()) {
+ if (entry.getValue() instanceof MetricSet) {
+ registerAll(prefix + "." + entry.getKey(), (MetricSet) entry.getValue());
+ } else {
+ metricRegistry.register(prefix + "." + entry.getKey(), entry.getValue());
+ }
+ }
+ }
+
+ @VisibleForTesting
+ public MetricRegistry getMetricRegistry() {
+ return metricRegistry;
+ }
+
+ @VisibleForTesting
+ public String dumpJson() throws Exception {
+ ObjectMapper jsonMapper = new ObjectMapper()
+ .registerModule(new MetricsModule(TimeUnit.MILLISECONDS, TimeUnit.MILLISECONDS, false));
+ return jsonMapper.writerWithDefaultPrettyPrinter().writeValueAsString(metricRegistry);
+ }
+
+ /**
+ * Initializes reporters from HIVE_CODAHALE_METRICS_REPORTER_CLASSES or HIVE_METRICS_REPORTER if the former is not defined.
+ * Note: if both confs are defined, only HIVE_CODAHALE_METRICS_REPORTER_CLASSES will be used.
+ */
+ private void initReporting() {
+
+ if (!(initCodahaleMetricsReporterClasses() || initMetricsReporter())) {
+ LOGGER.warn("Unable to initialize metrics reporting");
+ }
+ if (reporters.isEmpty()) {
+ // log a warning incase no reporters were successfully added
+ LOGGER.warn("No reporters configured for codahale metrics!");
+ }
+ }
+
+ /**
+ * Initializes reporting using HIVE_CODAHALE_METRICS_REPORTER_CLASSES.
+ * @return whether initialization was successful or not
+ */
+ private boolean initCodahaleMetricsReporterClasses() {
+
+ List<String> reporterClasses = Lists.newArrayList(Splitter.on(",").trimResults().omitEmptyStrings()
+ .split(KylinConfig.getInstanceFromEnv().getCoadhaleMetricReportClassesName()));
+ if (reporterClasses.isEmpty()) {
+ return false;
+ }
+
+ for (String reporterClass : reporterClasses) {
+ Class name = null;
+ try {
+ name = ClassUtils.getClass(reporterClass);
+ } catch (ClassNotFoundException e) {
+ LOGGER.error("Unable to instantiate metrics reporter class " + reporterClass
+ + " from conf HIVE_CODAHALE_METRICS_REPORTER_CLASSES", e);
+ throw new IllegalArgumentException(e);
+ }
+ try {
+ Constructor constructor = name.getConstructor(MetricRegistry.class, KylinConfig.class);
+ CodahaleReporter reporter = (CodahaleReporter) constructor.newInstance(metricRegistry, conf);
+ reporter.start();
+ reporters.add(reporter);
+ } catch (NoSuchMethodException | InstantiationException | IllegalAccessException
+ | InvocationTargetException e) {
+ LOGGER.error("Unable to instantiate using constructor(MetricRegistry, HiveConf) for" + " reporter "
+ + reporterClass + " from conf HIVE_CODAHALE_METRICS_REPORTER_CLASSES", e);
+ throw new IllegalArgumentException(e);
+ }
+ }
+ return true;
+ }
+
+ /**
+ * Initializes reporting using HIVE_METRICS+REPORTER.
+ * @return whether initialization was successful or not
+ */
+ private boolean initMetricsReporter() {
+
+ List<String> metricsReporterNames = Lists.newArrayList(Splitter.on(",").trimResults().omitEmptyStrings()
+ .split(KylinConfig.getInstanceFromEnv().getCoadhaleMetricReportClassesName()));
+ if (metricsReporterNames.isEmpty()) {
+ return false;
+ }
+
+ MetricsReporting reporter = null;
+ for (String metricsReportingName : metricsReporterNames) {
+ try {
+ reporter = MetricsReporting.valueOf(metricsReportingName.trim().toUpperCase());
+ } catch (IllegalArgumentException e) {
+ LOGGER.error("Invalid reporter name " + metricsReportingName, e);
+ throw e;
+ }
+ CodahaleReporter codahaleReporter = null;
+ switch (reporter) {
+ case CONSOLE:
+ codahaleReporter = new ConsoleMetricsReporter(metricRegistry, conf);
+ break;
+ case JMX:
+ codahaleReporter = new JmxMetricsReporter(metricRegistry, conf);
+ break;
+ case JSON_FILE:
+ codahaleReporter = new JsonFileMetricsReporter(metricRegistry, conf);
+ break;
+ case HADOOP2:
+ codahaleReporter = new Metrics2Reporter(metricRegistry, conf);
+ break;
+ default:
+ LOGGER.warn("Unhandled reporter " + reporter + " provided.");
+ }
+ if (codahaleReporter != null) {
+ codahaleReporter.start();
+ reporters.add(codahaleReporter);
+ }
+ }
+ return true;
+ }
+
+ public class CodahaleMetricsScope implements MetricsScope {
+
+ private final String name;
+ private final Timer timer;
+ private Timer.Context timerContext;
+
+ private boolean isOpen = false;
+
+ /**
+ * Instantiates a named scope - intended to only be called by Metrics, so locally scoped.
+ * @param name - name of the variable
+ */
+ private CodahaleMetricsScope(String name) {
+ this.name = name;
+ this.timer = CodahaleMetrics.this.getTimer(MetricsConstant.CALLS + name);
+ open();
+ }
+
+ /**
+ * Opens scope, and makes note of the time started, increments run counter
+ *
+ */
+ public void open() {
+ if (!isOpen) {
+ isOpen = true;
+ this.timerContext = timer.time();
+ CodahaleMetrics.this.incrementCounter(MetricsConstant.ACTIVE_CALLS + name);
+ } else {
+ LOGGER.warn("Scope named " + name + " is not closed, cannot be opened.");
+ }
+ }
+
+ /**
+ * Closes scope, and records the time taken
+ */
+ public void close() {
+ if (isOpen) {
+ timerContext.close();
+ CodahaleMetrics.this.decrementCounter(MetricsConstant.ACTIVE_CALLS + name);
+ } else {
+ LOGGER.warn("Scope named " + name + " is not open, cannot be closed.");
+ }
+ isOpen = false;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/8e078847/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/CodahaleReporter.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/CodahaleReporter.java b/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/CodahaleReporter.java
new file mode 100644
index 0000000..354f427
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/CodahaleReporter.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common.metrics.metrics2;
+
+import java.io.Closeable;
+
+import com.codahale.metrics.Reporter;
+
+public interface CodahaleReporter extends Closeable, Reporter {
+
+ /**
+ * Start the reporter.
+ */
+ public void start();
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/8e078847/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/ConsoleMetricsReporter.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/ConsoleMetricsReporter.java b/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/ConsoleMetricsReporter.java
new file mode 100644
index 0000000..b292443
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/ConsoleMetricsReporter.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common.metrics.metrics2;
+
+import java.util.concurrent.TimeUnit;
+
+import org.apache.kylin.common.KylinConfig;
+
+import com.codahale.metrics.ConsoleReporter;
+import com.codahale.metrics.MetricRegistry;
+
+/**
+ * A wrapper around Codahale ConsoleReporter to make it a pluggable/configurable Hive Metrics reporter.
+ */
+public class ConsoleMetricsReporter implements CodahaleReporter {
+
+ private final ConsoleReporter reporter;
+
+ public ConsoleMetricsReporter(MetricRegistry registry, KylinConfig conf) {
+
+ reporter = ConsoleReporter.forRegistry(registry).convertRatesTo(TimeUnit.SECONDS)
+ .convertDurationsTo(TimeUnit.MILLISECONDS).build();
+
+ }
+
+ @Override
+ public void start() {
+ reporter.start(10, TimeUnit.SECONDS);
+ }
+
+ @Override
+ public void close() {
+ reporter.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/8e078847/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/JmxMetricsReporter.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/JmxMetricsReporter.java b/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/JmxMetricsReporter.java
new file mode 100644
index 0000000..2b18f2e
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/JmxMetricsReporter.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common.metrics.metrics2;
+
+import java.util.concurrent.TimeUnit;
+
+import org.apache.kylin.common.KylinConfig;
+
+import com.codahale.metrics.JmxReporter;
+import com.codahale.metrics.MetricRegistry;
+
+/**
+ * A wrapper around Codahale JmxReporter to make it a pluggable/configurable Hive Metrics reporter.
+ */
+public class JmxMetricsReporter implements CodahaleReporter {
+
+ private final MetricRegistry registry;
+ private final KylinConfig conf;
+ private final JmxReporter jmxReporter;
+
+ public JmxMetricsReporter(MetricRegistry registry, KylinConfig conf) {
+ this.registry = registry;
+ this.conf = conf;
+
+ jmxReporter = JmxReporter.forRegistry(registry).convertRatesTo(TimeUnit.SECONDS)
+ .createsObjectNamesWith(new KylinObjectNameFactory()).convertDurationsTo(TimeUnit.MILLISECONDS).build();
+ }
+
+ @Override
+ public void start() {
+ jmxReporter.start();
+ }
+
+ @Override
+ public void close() {
+ jmxReporter.close();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/8e078847/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/JsonFileMetricsReporter.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/JsonFileMetricsReporter.java b/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/JsonFileMetricsReporter.java
new file mode 100644
index 0000000..6e8ad76
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/JsonFileMetricsReporter.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common.metrics.metrics2;
+
+import java.io.BufferedWriter;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.net.URI;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.kylin.common.KylinConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.json.MetricsModule;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectWriter;
+
+/**
+ * A metrics reporter for CodahaleMetrics that dumps metrics periodically into a file in JSON format.
+ */
+
+public class JsonFileMetricsReporter implements CodahaleReporter {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(JsonFileMetricsReporter.class);
+ private final MetricRegistry metricRegistry;
+ private final ObjectWriter jsonWriter;
+ private final ScheduledExecutorService executorService;
+ private final KylinConfig conf;
+ private final long interval;
+ private final String pathString;
+ private final Path path;
+
+ public JsonFileMetricsReporter(MetricRegistry registry, KylinConfig conf) {
+ this.metricRegistry = registry;
+ this.jsonWriter = new ObjectMapper()
+ .registerModule(new MetricsModule(TimeUnit.MILLISECONDS, TimeUnit.MILLISECONDS, false))
+ .writerWithDefaultPrettyPrinter();
+ executorService = Executors.newSingleThreadScheduledExecutor();
+ this.conf = conf;
+
+ interval = KylinConfig.getInstanceFromEnv().getJsonFileMetricsReporterInterval();
+ pathString = KylinConfig.getInstanceFromEnv().getMetricFileLocation();
+ path = new Path(pathString);
+ }
+
+ @Override
+ public void start() {
+
+ final Path tmpPath = new Path(pathString + ".tmp");
+ URI tmpPathURI = tmpPath.toUri();
+ final FileSystem fs;
+ try {
+ if (tmpPathURI.getScheme() == null && tmpPathURI.getAuthority() == null) {
+ //default local
+ fs = FileSystem.getLocal(new Configuration());
+ } else {
+ fs = FileSystem.get(tmpPathURI, new Configuration());
+ }
+ } catch (IOException e) {
+ LOGGER.error("Unable to access filesystem for path " + tmpPath + ". Aborting reporting", e);
+ return;
+ }
+
+ Runnable task = new Runnable() {
+ public void run() {
+ try {
+ String json = null;
+ try {
+ json = jsonWriter.writeValueAsString(metricRegistry);
+ } catch (JsonProcessingException e) {
+ LOGGER.error("Unable to convert json to string ", e);
+ return;
+ }
+
+ BufferedWriter bw = null;
+ try {
+ fs.delete(tmpPath, true);
+ bw = new BufferedWriter(new OutputStreamWriter(fs.create(tmpPath, true)));
+ bw.write(json);
+ fs.setPermission(tmpPath, FsPermission.createImmutable((short) 0644));
+ } catch (IOException e) {
+ LOGGER.error("Unable to write to temp file " + tmpPath, e);
+ return;
+ } finally {
+ if (bw != null) {
+ bw.close();
+ }
+ }
+
+ try {
+ fs.rename(tmpPath, path);
+ fs.setPermission(path, FsPermission.createImmutable((short) 0644));
+ } catch (IOException e) {
+ LOGGER.error("Unable to rename temp file " + tmpPath + " to " + pathString, e);
+ return;
+ }
+ } catch (Throwable t) {
+ // catch all errors (throwable and execptions to prevent subsequent tasks from being suppressed)
+ LOGGER.error("Error executing scheduled task ", t);
+ }
+ }
+ };
+
+ executorService.scheduleWithFixedDelay(task, 0, interval, TimeUnit.MILLISECONDS);
+ }
+
+ @Override
+ public void close() {
+ executorService.shutdown();
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/8e078847/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/KylinObjectNameFactory.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/KylinObjectNameFactory.java b/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/KylinObjectNameFactory.java
new file mode 100644
index 0000000..3348999
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/KylinObjectNameFactory.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common.metrics.metrics2;
+
+import java.util.Hashtable;
+
+import javax.management.MalformedObjectNameException;
+import javax.management.ObjectName;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.codahale.metrics.ObjectNameFactory;
+
+public class KylinObjectNameFactory implements ObjectNameFactory {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(KylinObjectNameFactory.class);
+
+ @Override
+ public ObjectName createName(String type, String domain, String name) {
+ try {
+ if (name.startsWith(domain)) {
+ ObjectName objectName = new ObjectName(name);
+ return objectName;
+ }
+
+ ObjectName objectName = new ObjectName(domain, "name", name);
+ if (objectName.isPattern()) {
+ objectName = new ObjectName(domain, "name", ObjectName.quote(name));
+ }
+ return objectName;
+ } catch (MalformedObjectNameException e) {
+ try {
+ return new ObjectName(domain, "name", ObjectName.quote(name));
+ } catch (MalformedObjectNameException e1) {
+ LOGGER.warn("Unable to register {} {}", type, name, e1);
+ throw new RuntimeException(e1);
+ }
+ }
+ }
+
+ public ObjectName process(String domain, String name) throws MalformedObjectNameException {
+ String[] kvArry = name.split(",");
+ Hashtable<String, String> hashTable = new Hashtable<>();
+ for (int i = 0; i < kvArry.length; i++) {
+ String[] split = kvArry[i].split("=");
+ hashTable.put(split[0], split[1]);
+ }
+ ObjectName objectName = new ObjectName(domain, hashTable);
+ return objectName;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/8e078847/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/MetricVariableRatioGauge.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/MetricVariableRatioGauge.java b/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/MetricVariableRatioGauge.java
new file mode 100644
index 0000000..b49093d
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/MetricVariableRatioGauge.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common.metrics.metrics2;
+
+import com.codahale.metrics.RatioGauge;
+import org.apache.kylin.common.metrics.common.MetricsVariable;
+
+
+/**
+ * Combines two numeric metric variables into one gauge type metric displaying their ratio
+ */
+public class MetricVariableRatioGauge extends RatioGauge {
+
+ private final MetricsVariable<Integer> numerator;
+ private final MetricsVariable<Integer> denominator;
+
+ public MetricVariableRatioGauge(MetricsVariable<Integer> numerator, MetricsVariable<Integer> denominator) {
+ this.numerator = numerator;
+ this.denominator = denominator;
+ }
+
+ @Override
+ protected Ratio getRatio() {
+ Integer numValue = numerator.getValue();
+ Integer denomValue = denominator.getValue();
+ if (numValue != null && denomValue != null) {
+ return Ratio.of(numValue.doubleValue(), denomValue.doubleValue());
+ }
+ return Ratio.of(0d, 0d);
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/8e078847/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/Metrics2Reporter.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/Metrics2Reporter.java b/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/Metrics2Reporter.java
new file mode 100644
index 0000000..d1c3c7f
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/Metrics2Reporter.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common.metrics.metrics2;
+
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.kylin.common.KylinConfig;
+
+import com.codahale.metrics.MetricRegistry;
+import com.github.joshelser.dropwizard.metrics.hadoop.HadoopMetrics2Reporter;
+
+/**
+ * A wrapper around Codahale HadoopMetrics2Reporter to make it a pluggable/configurable Hive Metrics reporter.
+ */
+public class Metrics2Reporter implements CodahaleReporter {
+
+ private final MetricRegistry metricRegistry;
+ private final KylinConfig conf;
+ private final HadoopMetrics2Reporter reporter;
+
+ public Metrics2Reporter(MetricRegistry registry, KylinConfig conf) {
+ this.metricRegistry = registry;
+ this.conf = conf;
+ String applicationName = "kylin";
+
+ reporter = HadoopMetrics2Reporter.forRegistry(metricRegistry).convertRatesTo(TimeUnit.SECONDS)
+ .convertDurationsTo(TimeUnit.MILLISECONDS).build(DefaultMetricsSystem.initialize(applicationName), // The application-level name
+ applicationName, // Component name
+ applicationName, // Component description
+ "General"); // Name for each metric record
+ }
+
+ @Override
+ public void start() {
+ long reportingInterval = 30;
+ reporter.start(reportingInterval, TimeUnit.SECONDS);
+ }
+
+ @Override
+ public void close() {
+ reporter.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/8e078847/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/MetricsReporting.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/MetricsReporting.java b/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/MetricsReporting.java
new file mode 100644
index 0000000..fc1b663
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/MetricsReporting.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common.metrics.metrics2;
+
+/**
+ * Reporting options for org.apache.hadoop.hive.common.metrics.metrics2.Metrics.
+ */
+public enum MetricsReporting {
+ JMX, CONSOLE, JSON_FILE, HADOOP2
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/8e078847/core-common/src/main/java/org/apache/kylin/common/metrics/perflog/IPerfLogger.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/metrics/perflog/IPerfLogger.java b/core-common/src/main/java/org/apache/kylin/common/metrics/perflog/IPerfLogger.java
new file mode 100644
index 0000000..83a4210
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/metrics/perflog/IPerfLogger.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common.metrics.perflog;
+
+public interface IPerfLogger {
+
+ /**
+ * Call this function when you start to measure time spent by a piece of code.
+ *
+ * @param callerName the logging object to be used.
+ * @param method method or ID that identifies this perf log element.
+ */
+ public void perfLogBegin(String callerName, String method);
+
+ /**
+ * Call this function in correspondence of perfLogBegin to mark the end of the measurement.
+ *
+ * @param callerName
+ * @param method
+ * @return long duration the difference between now and startTime, or -1 if startTime is null
+ */
+ public long perfLogEnd(String callerName, String method);
+
+ /**
+ * Call this function in correspondence of perfLogBegin to mark the end of the measurement.
+ *
+ * @param callerName
+ * @param method
+ * @return long duration the difference between now and startTime, or -1 if startTime is null
+ */
+ public long perfLogEnd(String callerName, String method, String additionalInfo);
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/8e078847/core-common/src/main/java/org/apache/kylin/common/metrics/perflog/PerfLogger.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/metrics/perflog/PerfLogger.java b/core-common/src/main/java/org/apache/kylin/common/metrics/perflog/PerfLogger.java
new file mode 100644
index 0000000..cf235bb
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/metrics/perflog/PerfLogger.java
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common.metrics.perflog;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.kylin.common.metrics.common.Metrics;
+import org.apache.kylin.common.metrics.common.MetricsFactory;
+import org.apache.kylin.common.metrics.common.MetricsScope;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.ImmutableMap;
+
+
+/**
+ * PerfLogger.
+ * <p>
+ * Can be used to measure and log the time spent by a piece of code.
+ */
+public class PerfLogger implements IPerfLogger {
+
+ static final private Logger LOG = LoggerFactory.getLogger(PerfLogger.class.getName());
+ protected final Map<String, Long> startTimes = new HashMap<String, Long>();
+ protected final Map<String, Long> endTimes = new HashMap<String, Long>();
+ //Methods for metrics integration. Each thread-local PerfLogger will open/close scope during each perf-log method.
+ transient Map<String, MetricsScope> openScopes = new HashMap<String, MetricsScope>();
+
+ public void perfLogBegin(String callerName, String method) {
+ long startTime = System.currentTimeMillis();
+ startTimes.put(method, new Long(startTime));
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<PERFLOG method=" + method + " from=" + callerName + ">");
+ }
+ beginMetrics(callerName + "." + method);
+ }
+
+ public long perfLogEnd(String callerName, String method) {
+ return perfLogEnd(callerName, method, null);
+ }
+
+ public long perfLogEnd(String callerName, String method, String additionalInfo) {
+ Long startTime = startTimes.get(method);
+ long endTime = System.currentTimeMillis();
+ endTimes.put(method, new Long(endTime));
+ long duration = startTime == null ? -1 : endTime - startTime.longValue();
+
+ if (LOG.isDebugEnabled()) {
+ StringBuilder sb = new StringBuilder("</PERFLOG method=").append(method);
+ if (startTime != null) {
+ sb.append(" start=").append(startTime);
+ }
+ sb.append(" end=").append(endTime);
+ if (startTime != null) {
+ sb.append(" duration=").append(duration);
+ }
+ sb.append(" from=").append(callerName);
+ if (additionalInfo != null) {
+ sb.append(" ").append(additionalInfo);
+ }
+ sb.append(">");
+ LOG.debug(sb.toString());
+ }
+ endMetrics(callerName + "." + method);
+ return duration;
+ }
+
+ public Long getStartTime(String method) {
+ long startTime = 0L;
+
+ if (startTimes.containsKey(method)) {
+ startTime = startTimes.get(method);
+ }
+ return startTime;
+ }
+
+ public Long getEndTime(String method) {
+ long endTime = 0L;
+
+ if (endTimes.containsKey(method)) {
+ endTime = endTimes.get(method);
+ }
+ return endTime;
+ }
+
+ public boolean startTimeHasMethod(String method) {
+ return startTimes.containsKey(method);
+ }
+
+ public boolean endTimeHasMethod(String method) {
+ return endTimes.containsKey(method);
+ }
+
+ public Long getDuration(String method) {
+ long duration = 0;
+ if (startTimes.containsKey(method) && endTimes.containsKey(method)) {
+ duration = endTimes.get(method) - startTimes.get(method);
+ }
+ return duration;
+ }
+
+ public ImmutableMap<String, Long> getStartTimes() {
+ return ImmutableMap.copyOf(startTimes);
+ }
+
+ public ImmutableMap<String, Long> getEndTimes() {
+ return ImmutableMap.copyOf(endTimes);
+ }
+
+ private void beginMetrics(String method) {
+ Metrics metrics = MetricsFactory.getInstance();
+ if (metrics != null) {
+ MetricsScope scope = metrics.createScope(method);
+ openScopes.put(method, scope);
+ }
+
+ }
+
+ private void endMetrics(String method) {
+ Metrics metrics = MetricsFactory.getInstance();
+ if (metrics != null) {
+ MetricsScope scope = openScopes.remove(method);
+ if (scope != null) {
+ metrics.endScope(scope);
+ }
+ }
+ }
+
+ /**
+ * Cleans up any dangling perfLog metric call scopes.
+ */
+ public void cleanupPerfLogMetrics() {
+ Metrics metrics = MetricsFactory.getInstance();
+ if (metrics != null) {
+ for (MetricsScope openScope : openScopes.values()) {
+ metrics.endScope(openScope);
+ }
+ }
+ openScopes.clear();
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/8e078847/core-common/src/main/java/org/apache/kylin/common/metrics/perflog/PerfLoggerFactory.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/metrics/perflog/PerfLoggerFactory.java b/core-common/src/main/java/org/apache/kylin/common/metrics/perflog/PerfLoggerFactory.java
new file mode 100644
index 0000000..4fc43a8
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/metrics/perflog/PerfLoggerFactory.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common.metrics.perflog;
+
+import org.apache.commons.lang3.ClassUtils;
+import org.apache.kylin.common.KylinConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class PerfLoggerFactory {
+
+ protected static final ThreadLocal<IPerfLogger> perfLogger = new ThreadLocal<IPerfLogger>();
+ static final private Logger LOG = LoggerFactory.getLogger(PerfLoggerFactory.class.getName());
+
+ public static IPerfLogger getPerfLogger() {
+ return getPerfLogger(false);
+ }
+
+ public static void setPerfLogger(IPerfLogger iPerfLogger) {
+ perfLogger.set(iPerfLogger);
+ }
+
+ public static IPerfLogger getPerfLogger(boolean resetPerfLogger) {
+ IPerfLogger result = perfLogger.get();
+ if (resetPerfLogger || result == null) {
+ try {
+ result = (IPerfLogger) ClassUtils.getClass(KylinConfig.getInstanceFromEnv().getPerfLoggerClassName())
+ .newInstance();
+ } catch (ClassNotFoundException e) {
+ LOG.error("Performance Logger Class not found:" + e.getMessage());
+ result = new SimplePerfLogger();
+ } catch (IllegalAccessException | InstantiationException e) {
+ e.printStackTrace();
+ }
+ perfLogger.set(result);
+ }
+ return result;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/8e078847/core-common/src/main/java/org/apache/kylin/common/metrics/perflog/SimplePerfLogger.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/metrics/perflog/SimplePerfLogger.java b/core-common/src/main/java/org/apache/kylin/common/metrics/perflog/SimplePerfLogger.java
new file mode 100644
index 0000000..cd88c65
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/metrics/perflog/SimplePerfLogger.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common.metrics.perflog;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class SimplePerfLogger implements IPerfLogger {
+
+ static final private Logger LOG = LoggerFactory.getLogger(SimplePerfLogger.class.getName());
+ protected final Map<String, Long> startTimes = new HashMap<String, Long>();
+ protected final Map<String, Long> endTimes = new HashMap<String, Long>();
+
+ protected SimplePerfLogger() {
+ }
+
+ public void perfLogBegin(String callerName, String method) {
+ long startTime = System.currentTimeMillis();
+ startTimes.put(method, new Long(startTime));
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<PERFLOG method=" + method + " from=" + callerName + ">");
+ }
+ }
+
+ public long perfLogEnd(String callerName, String method) {
+ return perfLogEnd(callerName, method, null);
+ }
+
+ public long perfLogEnd(String callerName, String method, String additionalInfo) {
+ Long startTime = startTimes.get(method);
+ long endTime = System.currentTimeMillis();
+ endTimes.put(method, new Long(endTime));
+ long duration = startTime == null ? -1 : endTime - startTime.longValue();
+
+ if (LOG.isDebugEnabled()) {
+ StringBuilder sb = new StringBuilder("</PERFLOG method=").append(method);
+ if (startTime != null) {
+ sb.append(" start=").append(startTime);
+ }
+ sb.append(" end=").append(endTime);
+ if (startTime != null) {
+ sb.append(" duration=").append(duration);
+ }
+ sb.append(" from=").append(callerName);
+ if (additionalInfo != null) {
+ sb.append(" ").append(additionalInfo);
+ }
+ sb.append(">");
+ LOG.debug(sb.toString());
+ }
+ return duration;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/8e078847/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index f7444cd..5b00798 100644
--- a/pom.xml
+++ b/pom.xml
@@ -114,7 +114,8 @@
<cors.version>2.5</cors.version>
<tomcat.version>7.0.69</tomcat.version>
<t-digest.version>3.1</t-digest.version>
-
+ <!--metric-->
+ <dropwizard.version>3.1.2</dropwizard.version>
<!-- REST Service, ref https://github.com/spring-projects/spring-boot/blob/v1.3.8.RELEASE/spring-boot-dependencies/pom.xml -->
<spring.boot.version>1.3.8.RELEASE</spring.boot.version>
<spring.framework.version>4.2.8.RELEASE</spring.framework.version>
http://git-wip-us.apache.org/repos/asf/kylin/blob/8e078847/server-base/src/main/java/org/apache/kylin/rest/init/InitialTaskManager.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/init/InitialTaskManager.java b/server-base/src/main/java/org/apache/kylin/rest/init/InitialTaskManager.java
index 8bba674..0194334 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/init/InitialTaskManager.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/init/InitialTaskManager.java
@@ -20,6 +20,7 @@ package org.apache.kylin.rest.init;
import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.rest.metrics.QueryMetrics2Facade;
import org.apache.kylin.rest.metrics.QueryMetricsFacade;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -42,7 +43,8 @@ public class InitialTaskManager implements InitializingBean {
private void runInitialTasks() {
// init metrics system for kylin
QueryMetricsFacade.init();
-
+ QueryMetrics2Facade.init();
+
KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
String initTasks = kylinConfig.getInitTasks();
if (!StringUtils.isEmpty(initTasks)) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/8e078847/server-base/src/main/java/org/apache/kylin/rest/metrics/QueryMetrics2Facade.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/metrics/QueryMetrics2Facade.java b/server-base/src/main/java/org/apache/kylin/rest/metrics/QueryMetrics2Facade.java
new file mode 100644
index 0000000..b42e65c
--- /dev/null
+++ b/server-base/src/main/java/org/apache/kylin/rest/metrics/QueryMetrics2Facade.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.rest.metrics;
+
+import static org.apache.kylin.common.metrics.common.MetricsConstant.TOTAL;
+import static org.apache.kylin.common.metrics.common.Metricss.buildCubeMetricPrefix;
+
+import java.util.concurrent.TimeUnit;
+
+import javax.annotation.concurrent.ThreadSafe;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.metrics.common.Metrics;
+import org.apache.kylin.common.metrics.common.MetricsConstant;
+import org.apache.kylin.common.metrics.common.MetricsFactory;
+import org.apache.kylin.common.metrics.common.Metricss;
+import org.apache.kylin.rest.request.SQLRequest;
+import org.apache.kylin.rest.response.SQLResponse;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * The entrance of metrics features.
+ */
+@ThreadSafe
+public class QueryMetrics2Facade {
+
+ private static final Logger logger = LoggerFactory.getLogger(QueryMetrics2Facade.class);
+ private static Metrics metrics;
+ private static boolean enabled = false;
+
+ public static void init() {
+ enabled = KylinConfig.getInstanceFromEnv().getQueryMetricsEnabled();
+ }
+
+ public static void updateMetrics(SQLRequest sqlRequest, SQLResponse sqlResponse) {
+ if (!enabled) {
+ return;
+ }
+ if (metrics == null) {
+ metrics = MetricsFactory.getInstance();
+ }
+ String projectName = sqlRequest.getProject();
+ String cube = sqlResponse.getCube();
+ if (cube == null) {
+ return;
+ }
+ String cubeName = cube.replace("=", "->");
+
+ // update(getQueryMetrics("Server_Total"), sqlResponse);
+ update(buildCubeMetricPrefix(TOTAL), sqlResponse);
+ update(buildCubeMetricPrefix(projectName), sqlResponse);
+ String cubeMetricName = buildCubeMetricPrefix(projectName, cubeName);
+ update(cubeMetricName, sqlResponse);
+ }
+
+ private static void update(String name, SQLResponse sqlResponse) {
+ try {
+ incrQueryCount(name, sqlResponse);
+ incrCacheHitCount(name, sqlResponse);
+ if (!sqlResponse.getIsException()) {
+ metrics.updateTimer(Metricss.buildMetricName(name, MetricsConstant.QUERY_DURATION),
+ sqlResponse.getDuration(), TimeUnit.MILLISECONDS);
+ metrics.updateHistogram(Metricss.buildMetricName(name, MetricsConstant.QUERY_RESULT_ROWCOUNT),
+ sqlResponse.getResults().size());
+ metrics.updateHistogram(Metricss.buildMetricName(name, MetricsConstant.QUERY_SCAN_ROWCOUNT),
+ sqlResponse.getTotalScanCount());
+ }
+ } catch (Exception e) {
+ logger.error(e.getMessage());
+ }
+
+ }
+
+ private static void incrQueryCount(String name, SQLResponse sqlResponse) {
+ if (!sqlResponse.isHitExceptionCache() && !sqlResponse.getIsException()) {
+ metrics.incrementCounter(Metricss.buildMetricName(name, MetricsConstant.QUERY_SUCCESS_COUNT));
+ } else {
+ metrics.incrementCounter(Metricss.buildMetricName(name, MetricsConstant.QUERY_FAIL_COUNT));
+ }
+ metrics.incrementCounter(Metricss.buildMetricName(name, MetricsConstant.QUERY_COUNT));
+ }
+
+ private static void incrCacheHitCount(String name, SQLResponse sqlResponse) {
+ if (sqlResponse.isStorageCacheUsed()) {
+ metrics.incrementCounter(Metricss.buildMetricName(name, MetricsConstant.QUERY_CACHE_COUNT));
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/8e078847/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 24ac5a0..533b93d 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -87,6 +87,7 @@ import org.apache.kylin.query.util.QueryUtil;
import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.exception.BadRequestException;
import org.apache.kylin.rest.exception.InternalErrorException;
+import org.apache.kylin.rest.metrics.QueryMetrics2Facade;
import org.apache.kylin.rest.metrics.QueryMetricsFacade;
import org.apache.kylin.rest.model.Query;
import org.apache.kylin.rest.msg.Message;
@@ -436,6 +437,7 @@ public class QueryService extends BasicService {
logQuery(sqlRequest, sqlResponse);
QueryMetricsFacade.updateMetrics(sqlRequest, sqlResponse);
+ QueryMetrics2Facade.updateMetrics(sqlRequest, sqlResponse);
if (sqlResponse.getIsException())
throw new InternalErrorException(sqlResponse.getExceptionMessage());