You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by cw...@apache.org on 2017/04/03 17:11:17 UTC
hive git commit: HIVE-16206: Make Codahale metrics reporters
pluggable (Sunitha Beeram via Carl Steinbach)
Repository: hive
Updated Branches:
refs/heads/master 6a82a898e -> bf98700a7
HIVE-16206: Make Codahale metrics reporters pluggable (Sunitha Beeram via Carl Steinbach)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/bf98700a
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/bf98700a
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/bf98700a
Branch: refs/heads/master
Commit: bf98700a760ea1cca30f236e7dce10272fffdd48
Parents: 6a82a89
Author: Carl Steinbach <cw...@apache.org>
Authored: Mon Apr 3 10:09:53 2017 -0700
Committer: Carl Steinbach <cw...@apache.org>
Committed: Mon Apr 3 10:09:53 2017 -0700
----------------------------------------------------------------------
.../metrics/metrics2/CodahaleMetrics.java | 192 +++++++++----------
.../metrics/metrics2/CodahaleReporter.java | 29 +++
.../metrics2/ConsoleMetricsReporter.java | 55 ++++++
.../metrics/metrics2/JmxMetricsReporter.java | 56 ++++++
.../metrics2/JsonFileMetricsReporter.java | 136 +++++++++++++
.../metrics/metrics2/Metrics2Reporter.java | 62 ++++++
.../org/apache/hadoop/hive/conf/HiveConf.java | 22 ++-
.../metrics/metrics2/TestCodahaleMetrics.java | 7 +-
.../metrics2/TestCodahaleReportersConf.java | 145 ++++++++++++++
9 files changed, 589 insertions(+), 115 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/bf98700a/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/CodahaleMetrics.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/CodahaleMetrics.java b/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/CodahaleMetrics.java
index e8abf6c..2d6c1b4 100644
--- a/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/CodahaleMetrics.java
+++ b/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/CodahaleMetrics.java
@@ -44,6 +44,8 @@ import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.google.common.collect.Lists;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
@@ -190,22 +192,8 @@ public class CodahaleMetrics implements org.apache.hadoop.hive.common.metrics.co
registerAll("threads", new ThreadStatesGaugeSet());
registerAll("classLoading", new ClassLoadingGaugeSet());
- //Metrics reporter
- Set<MetricsReporting> finalReporterList = new HashSet<MetricsReporting>();
- List<String> metricsReporterNames = Lists.newArrayList(
- Splitter.on(",").trimResults().omitEmptyStrings().split(conf.getVar(HiveConf.ConfVars.HIVE_METRICS_REPORTER)));
-
- if(metricsReporterNames != null) {
- for (String metricsReportingName : metricsReporterNames) {
- try {
- MetricsReporting reporter = MetricsReporting.valueOf(metricsReportingName.trim().toUpperCase());
- finalReporterList.add(reporter);
- } catch (IllegalArgumentException e) {
- LOGGER.warn("Metrics reporter skipped due to invalid configured reporter: " + metricsReportingName);
- }
- }
- }
- initReporting(finalReporterList);
+ //initialize reporters
+ initReporting();
}
@@ -385,107 +373,99 @@ public class CodahaleMetrics implements org.apache.hadoop.hive.common.metrics.co
}
/**
- * Should be only called once to initialize the reporters
+ * Initializes reporters from HIVE_CODAHALE_METRICS_REPORTER_CLASSES or HIVE_METRICS_REPORTER if the former is not defined.
+ * Note: if both confs are defined, only HIVE_CODAHALE_METRICS_REPORTER_CLASSES will be used.
*/
- private void initReporting(Set<MetricsReporting> reportingSet) {
- for (MetricsReporting reporting : reportingSet) {
- switch(reporting) {
- case CONSOLE:
- final ConsoleReporter consoleReporter = ConsoleReporter.forRegistry(metricRegistry)
- .convertRatesTo(TimeUnit.SECONDS)
- .convertDurationsTo(TimeUnit.MILLISECONDS)
- .build();
- consoleReporter.start(1, TimeUnit.SECONDS);
- reporters.add(consoleReporter);
- break;
- case JMX:
- final JmxReporter jmxReporter = JmxReporter.forRegistry(metricRegistry)
- .convertRatesTo(TimeUnit.SECONDS)
- .convertDurationsTo(TimeUnit.MILLISECONDS)
- .build();
- jmxReporter.start();
- reporters.add(jmxReporter);
- break;
- case JSON_FILE:
- final JsonFileReporter jsonFileReporter = new JsonFileReporter();
- jsonFileReporter.start();
- reporters.add(jsonFileReporter);
- break;
- case HADOOP2:
- String applicationName = conf.get(HiveConf.ConfVars.HIVE_METRICS_HADOOP2_COMPONENT_NAME.varname);
- long reportingInterval = HiveConf.toTime(
- conf.get(HiveConf.ConfVars.HIVE_METRICS_HADOOP2_INTERVAL.varname),
- TimeUnit.SECONDS, TimeUnit.SECONDS);
- final HadoopMetrics2Reporter metrics2Reporter = HadoopMetrics2Reporter.forRegistry(metricRegistry)
- .convertRatesTo(TimeUnit.SECONDS)
- .convertDurationsTo(TimeUnit.MILLISECONDS)
- .build(DefaultMetricsSystem.initialize(applicationName), // The application-level name
- applicationName, // Component name
- applicationName, // Component description
- "General"); // Name for each metric record
- metrics2Reporter.start(reportingInterval, TimeUnit.SECONDS);
- break;
- }
+ private void initReporting() {
+
+ if (!(initCodahaleMetricsReporterClasses() || initMetricsReporter())) {
+ LOGGER.warn("Unable to initialize metrics reporting");
+ }
+ if (reporters.isEmpty()) {
+ // log a warning incase no reporters were successfully added
+ LOGGER.warn("No reporters configured for codahale metrics!");
}
}
- class JsonFileReporter implements Closeable {
- private ObjectMapper jsonMapper = null;
- private java.util.Timer timer = null;
-
- public void start() {
- this.jsonMapper = new ObjectMapper().registerModule(new MetricsModule(TimeUnit.MILLISECONDS, TimeUnit.MILLISECONDS, false));
- this.timer = new java.util.Timer(true);
+ /**
+ * Initializes reporting using HIVE_CODAHALE_METRICS_REPORTER_CLASSES.
+ * @return whether initialization was successful or not
+ */
+ private boolean initCodahaleMetricsReporterClasses() {
- long time = conf.getTimeVar(HiveConf.ConfVars.HIVE_METRICS_JSON_FILE_INTERVAL, TimeUnit.MILLISECONDS);
- final String pathString = conf.getVar(HiveConf.ConfVars.HIVE_METRICS_JSON_FILE_LOCATION);
+ List<String> reporterClasses = Lists.newArrayList(Splitter.on(",").trimResults().
+ omitEmptyStrings().split(conf.getVar(HiveConf.ConfVars.HIVE_CODAHALE_METRICS_REPORTER_CLASSES)));
+ if (reporterClasses.isEmpty()) {
+ return false;
+ }
- timer.schedule(new TimerTask() {
- @Override
- public void run() {
- BufferedWriter bw = null;
- try {
- String json = jsonMapper.writerWithDefaultPrettyPrinter().writeValueAsString(metricRegistry);
- Path tmpPath = new Path(pathString + ".tmp");
- URI tmpPathURI = tmpPath.toUri();
- FileSystem fs = null;
- if (tmpPathURI.getScheme() == null && tmpPathURI.getAuthority() == null) {
- //default local
- fs = FileSystem.getLocal(conf);
- } else {
- fs = FileSystem.get(tmpPathURI, conf);
- }
- fs.delete(tmpPath, true);
- bw = new BufferedWriter(new OutputStreamWriter(fs.create(tmpPath, true)));
- bw.write(json);
- bw.close();
- fs.setPermission(tmpPath, FsPermission.createImmutable((short) 0644));
-
- Path path = new Path(pathString);
- fs.rename(tmpPath, path);
- fs.setPermission(path, FsPermission.createImmutable((short) 0644));
- } catch (Exception e) {
- LOGGER.warn("Error writing JSON Metrics to file", e);
- } finally {
- try {
- if (bw != null) {
- bw.close();
- }
- } catch (IOException e) {
- //Ignore.
- }
- }
+ for (String reporterClass : reporterClasses) {
+ Class name = null;
+ try {
+ name = conf.getClassByName(reporterClass);
+ } catch (ClassNotFoundException e) {
+ LOGGER.error("Unable to instantiate metrics reporter class " + reporterClass +
+ " from conf HIVE_CODAHALE_METRICS_REPORTER_CLASSES", e);
+ throw new IllegalArgumentException(e);
+ }
+ try {
+ Constructor constructor = name.getConstructor(MetricRegistry.class, HiveConf.class);
+ CodahaleReporter reporter = (CodahaleReporter) constructor.newInstance(metricRegistry, conf);
+ reporter.start();
+ reporters.add(reporter);
+ } catch (NoSuchMethodException | InstantiationException |
+ IllegalAccessException | InvocationTargetException e) {
+ LOGGER.error("Unable to instantiate using constructor(MetricRegistry, HiveConf) for"
+ + " reporter " + reporterClass + " from conf HIVE_CODAHALE_METRICS_REPORTER_CLASSES",
+ e);
+ throw new IllegalArgumentException(e);
+ }
+ }
+ return true;
+ }
+ /**
+ * Initializes reporting using HIVE_METRICS+REPORTER.
+ * @return whether initialization was successful or not
+ */
+ private boolean initMetricsReporter() {
- }
- }, 0, time);
+ List<String> metricsReporterNames = Lists.newArrayList(Splitter.on(",").trimResults().
+ omitEmptyStrings().split(conf.getVar(HiveConf.ConfVars.HIVE_METRICS_REPORTER)));
+ if (metricsReporterNames.isEmpty()) {
+ return false;
}
- @Override
- public void close() {
- if (timer != null) {
- this.timer.cancel();
+ MetricsReporting reporter = null;
+ for (String metricsReportingName : metricsReporterNames) {
+ try {
+ reporter = MetricsReporting.valueOf(metricsReportingName.trim().toUpperCase());
+ } catch (IllegalArgumentException e) {
+ LOGGER.error("Invalid reporter name " + metricsReportingName, e);
+ throw e;
+ }
+ CodahaleReporter codahaleReporter = null;
+ switch (reporter) {
+ case CONSOLE:
+ codahaleReporter = new ConsoleMetricsReporter(metricRegistry, conf);
+ break;
+ case JMX:
+ codahaleReporter = new JmxMetricsReporter(metricRegistry, conf);
+ break;
+ case JSON_FILE:
+ codahaleReporter = new JsonFileMetricsReporter(metricRegistry, conf);
+ break;
+ case HADOOP2:
+ codahaleReporter = new Metrics2Reporter(metricRegistry, conf);
+ break;
+ default:
+ LOGGER.warn("Unhandled reporter " + reporter + " provided.");
+ }
+ if (codahaleReporter != null) {
+ codahaleReporter.start();
+ reporters.add(codahaleReporter);
}
}
+ return true;
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/bf98700a/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/CodahaleReporter.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/CodahaleReporter.java b/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/CodahaleReporter.java
new file mode 100644
index 0000000..ba4d8e4
--- /dev/null
+++ b/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/CodahaleReporter.java
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.common.metrics.metrics2;
+
+import com.codahale.metrics.Reporter;
+import java.io.Closeable;
+
+interface CodahaleReporter extends Closeable, Reporter {
+
+ /**
+ * Start the reporter.
+ */
+ public void start();
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/bf98700a/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/ConsoleMetricsReporter.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/ConsoleMetricsReporter.java b/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/ConsoleMetricsReporter.java
new file mode 100644
index 0000000..dea1848
--- /dev/null
+++ b/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/ConsoleMetricsReporter.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.common.metrics.metrics2;
+
+import com.codahale.metrics.ConsoleReporter;
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.Reporter;
+import java.io.Closeable;
+import java.util.concurrent.TimeUnit;
+import org.apache.hadoop.hive.conf.HiveConf;
+
+
+/**
+ * A wrapper around Codahale ConsoleReporter to make it a pluggable/configurable Hive Metrics reporter.
+ */
+public class ConsoleMetricsReporter implements CodahaleReporter {
+
+ private final ConsoleReporter reporter;
+
+ public ConsoleMetricsReporter(MetricRegistry registry, HiveConf conf) {
+
+ reporter = ConsoleReporter.forRegistry(registry)
+ .convertRatesTo(TimeUnit.SECONDS)
+ .convertDurationsTo(TimeUnit.MILLISECONDS)
+ .build();
+
+ }
+
+ @Override
+ public void start() {
+ reporter.start(1, TimeUnit.SECONDS);
+ }
+
+ @Override
+ public void close() {
+ reporter.close();
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/hive/blob/bf98700a/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/JmxMetricsReporter.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/JmxMetricsReporter.java b/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/JmxMetricsReporter.java
new file mode 100644
index 0000000..f12adf9
--- /dev/null
+++ b/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/JmxMetricsReporter.java
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.common.metrics.metrics2;
+
+import com.codahale.metrics.JmxReporter;
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.Reporter;
+import java.io.Closeable;
+import java.util.concurrent.TimeUnit;
+import org.apache.hadoop.hive.conf.HiveConf;
+
+/**
+ * A wrapper around Codahale JmxReporter to make it a pluggable/configurable Hive Metrics reporter.
+ */
+public class JmxMetricsReporter implements CodahaleReporter {
+
+ private final MetricRegistry registry;
+ private final HiveConf conf;
+ private final JmxReporter jmxReporter;
+
+ public JmxMetricsReporter(MetricRegistry registry, HiveConf conf) {
+ this.registry = registry;
+ this.conf = conf;
+
+ jmxReporter = JmxReporter.forRegistry(registry)
+ .convertRatesTo(TimeUnit.SECONDS)
+ .convertDurationsTo(TimeUnit.MILLISECONDS)
+ .build();
+ }
+
+ @Override
+ public void start() {
+ jmxReporter.start();
+ }
+
+ @Override
+ public void close() {
+ jmxReporter.close();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/bf98700a/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/JsonFileMetricsReporter.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/JsonFileMetricsReporter.java b/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/JsonFileMetricsReporter.java
new file mode 100644
index 0000000..c07517a
--- /dev/null
+++ b/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/JsonFileMetricsReporter.java
@@ -0,0 +1,136 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.common.metrics.metrics2;
+
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.json.MetricsModule;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectWriter;
+import java.io.BufferedWriter;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.net.URI;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * A metrics reporter for CodahaleMetrics that dumps metrics periodically into a file in JSON format.
+ */
+
+public class JsonFileMetricsReporter implements CodahaleReporter {
+
+ private final MetricRegistry metricRegistry;
+ private final ObjectWriter jsonWriter;
+ private final ScheduledExecutorService executorService;
+ private final HiveConf conf;
+ private final long interval;
+ private final String pathString;
+ private final Path path;
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(JsonFileMetricsReporter.class);
+
+ public JsonFileMetricsReporter(MetricRegistry registry, HiveConf conf) {
+ this.metricRegistry = registry;
+ this.jsonWriter =
+ new ObjectMapper().registerModule(new MetricsModule(TimeUnit.MILLISECONDS,
+ TimeUnit.MILLISECONDS, false)).writerWithDefaultPrettyPrinter();
+ executorService = Executors.newSingleThreadScheduledExecutor();
+ this.conf = conf;
+
+ interval = conf.getTimeVar(HiveConf.ConfVars.HIVE_METRICS_JSON_FILE_INTERVAL, TimeUnit.MILLISECONDS);
+ pathString = conf.getVar(HiveConf.ConfVars.HIVE_METRICS_JSON_FILE_LOCATION);
+ path = new Path(pathString);
+ }
+
+ @Override
+ public void start() {
+
+ final Path tmpPath = new Path(pathString + ".tmp");
+ URI tmpPathURI = tmpPath.toUri();
+ final FileSystem fs;
+ try {
+ if (tmpPathURI.getScheme() == null && tmpPathURI.getAuthority() == null) {
+ //default local
+ fs = FileSystem.getLocal(conf);
+ } else {
+ fs = FileSystem.get(tmpPathURI, conf);
+ }
+ }
+ catch (IOException e) {
+ LOGGER.error("Unable to access filesystem for path " + tmpPath + ". Aborting reporting", e);
+ return;
+ }
+
+ Runnable task = new Runnable() {
+ public void run() {
+ try {
+ String json = null;
+ try {
+ json = jsonWriter.writeValueAsString(metricRegistry);
+ } catch (JsonProcessingException e) {
+ LOGGER.error("Unable to convert json to string ", e);
+ return;
+ }
+
+ BufferedWriter bw = null;
+ try {
+ fs.delete(tmpPath, true);
+ bw = new BufferedWriter(new OutputStreamWriter(fs.create(tmpPath, true)));
+ bw.write(json);
+ fs.setPermission(tmpPath, FsPermission.createImmutable((short) 0644));
+ } catch (IOException e) {
+ LOGGER.error("Unable to write to temp file " + tmpPath, e);
+ return;
+ } finally {
+ if (bw != null) {
+ bw.close();
+ }
+ }
+
+ try {
+ fs.rename(tmpPath, path);
+ fs.setPermission(path, FsPermission.createImmutable((short) 0644));
+ } catch (IOException e) {
+ LOGGER.error("Unable to rename temp file " + tmpPath + " to " + pathString, e);
+ return;
+ }
+ } catch (Throwable t) {
+ // catch all errors (throwable and execptions to prevent subsequent tasks from being suppressed)
+ LOGGER.error("Error executing scheduled task ", t);
+ }
+ }
+ };
+
+ executorService.scheduleWithFixedDelay(task,0, interval, TimeUnit.MILLISECONDS);
+ }
+
+ @Override
+ public void close() {
+ executorService.shutdown();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/bf98700a/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/Metrics2Reporter.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/Metrics2Reporter.java b/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/Metrics2Reporter.java
new file mode 100644
index 0000000..3b402d8
--- /dev/null
+++ b/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/Metrics2Reporter.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.common.metrics.metrics2;
+
+import com.codahale.metrics.MetricRegistry;
+import com.github.joshelser.dropwizard.metrics.hadoop.HadoopMetrics2Reporter;
+import java.io.Closeable;
+import java.util.concurrent.TimeUnit;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import com.codahale.metrics.Reporter;
+
+/**
+ * A wrapper around Codahale HadoopMetrics2Reporter to make it a pluggable/configurable Hive Metrics reporter.
+ */
+public class Metrics2Reporter implements CodahaleReporter {
+
+ private final MetricRegistry metricRegistry;
+ private final HiveConf conf;
+ private final HadoopMetrics2Reporter reporter;
+
+ public Metrics2Reporter(MetricRegistry registry, HiveConf conf) {
+ this.metricRegistry = registry;
+ this.conf = conf;
+ String applicationName = conf.get(HiveConf.ConfVars.HIVE_METRICS_HADOOP2_COMPONENT_NAME.varname);
+
+ reporter = HadoopMetrics2Reporter.forRegistry(metricRegistry)
+ .convertRatesTo(TimeUnit.SECONDS)
+ .convertDurationsTo(TimeUnit.MILLISECONDS)
+ .build(DefaultMetricsSystem.initialize(applicationName), // The application-level name
+ applicationName, // Component name
+ applicationName, // Component description
+ "General"); // Name for each metric record
+ }
+
+ @Override
+ public void start() {
+ long reportingInterval =
+ HiveConf.toTime(conf.get(HiveConf.ConfVars.HIVE_METRICS_HADOOP2_INTERVAL.varname), TimeUnit.SECONDS, TimeUnit.SECONDS);
+ reporter.start(reportingInterval, TimeUnit.SECONDS);
+ }
+
+ @Override
+ public void close() {
+ reporter.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/bf98700a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index b9a72d1..6de6ed6 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -2210,19 +2210,29 @@ public class HiveConf extends Configuration {
"org.apache.hadoop.hive.common.metrics.metrics2.CodahaleMetrics",
"org.apache.hadoop.hive.common.metrics.LegacyMetrics"),
"Hive metrics subsystem implementation class."),
- HIVE_METRICS_REPORTER("hive.service.metrics.reporter", "JSON_FILE, JMX",
- "Reporter type for metric class org.apache.hadoop.hive.common.metrics.metrics2.CodahaleMetrics, " +
- "comma separated list of JMX, CONSOLE, JSON_FILE, HADOOP2"),
+ HIVE_CODAHALE_METRICS_REPORTER_CLASSES("hive.service.metrics.codahale.reporter.classes",
+ "org.apache.hadoop.hive.common.metrics.metrics2.JsonFileMetricsReporter, " +
+ "org.apache.hadoop.hive.common.metrics.metrics2.JmxMetricsReporter",
+ "Comma separated list of reporter implementation classes for metric class "
+ + "org.apache.hadoop.hive.common.metrics.metrics2.CodahaleMetrics. Overrides "
+ + "HIVE_METRICS_REPORTER conf if present"),
+ @Deprecated
+ HIVE_METRICS_REPORTER("hive.service.metrics.reporter", "",
+ "Reporter implementations for metric class "
+ + "org.apache.hadoop.hive.common.metrics.metrics2.CodahaleMetrics;" +
+ "Deprecated, use HIVE_CODAHALE_METRICS_REPORTER_CLASSES instead. This configuraiton will be"
+ + " overridden by HIVE_CODAHALE_METRICS_REPORTER_CLASSES if present. " +
+ "Comma separated list of JMX, CONSOLE, JSON_FILE, HADOOP2"),
HIVE_METRICS_JSON_FILE_LOCATION("hive.service.metrics.file.location", "/tmp/report.json",
"For metric class org.apache.hadoop.hive.common.metrics.metrics2.CodahaleMetrics JSON_FILE reporter, the location of local JSON metrics file. " +
"This file will get overwritten at every interval."),
- HIVE_METRICS_JSON_FILE_INTERVAL("hive.service.metrics.file.frequency", "5s",
+ HIVE_METRICS_JSON_FILE_INTERVAL("hive.service.metrics.file.frequency", "5000ms",
new TimeValidator(TimeUnit.MILLISECONDS),
- "For metric class org.apache.hadoop.hive.common.metrics.metrics2.CodahaleMetrics JSON_FILE reporter, " +
+ "For metric class org.apache.hadoop.hive.common.metrics.metrics2.JsonFileMetricsReporter, " +
"the frequency of updating JSON metrics file."),
HIVE_METRICS_HADOOP2_INTERVAL("hive.service.metrics.hadoop2.frequency", "30s",
new TimeValidator(TimeUnit.SECONDS),
- "For metric class org.apache.hadoop.hive.common.metrics.metrics2.CodahaleMetrics HADOOP2 reporter, " +
+ "For metric class org.apache.hadoop.hive.common.metrics.metrics2.Metrics2Reporter, " +
"the frequency of updating the HADOOP2 metrics system."),
HIVE_METRICS_HADOOP2_COMPONENT_NAME("hive.service.metrics.hadoop2.component",
"hive",
http://git-wip-us.apache.org/repos/asf/hive/blob/bf98700a/common/src/test/org/apache/hadoop/hive/common/metrics/metrics2/TestCodahaleMetrics.java
----------------------------------------------------------------------
diff --git a/common/src/test/org/apache/hadoop/hive/common/metrics/metrics2/TestCodahaleMetrics.java b/common/src/test/org/apache/hadoop/hive/common/metrics/metrics2/TestCodahaleMetrics.java
index aa4e75f..67f81d6 100644
--- a/common/src/test/org/apache/hadoop/hive/common/metrics/metrics2/TestCodahaleMetrics.java
+++ b/common/src/test/org/apache/hadoop/hive/common/metrics/metrics2/TestCodahaleMetrics.java
@@ -33,8 +33,6 @@ import org.junit.Before;
import org.junit.Test;
import java.io.File;
-import java.nio.file.Files;
-import java.nio.file.Paths;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@@ -57,9 +55,12 @@ public class TestCodahaleMetrics {
jsonReportFile = new File(workDir, "json_reporting");
jsonReportFile.delete();
+
conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, "local");
conf.setVar(HiveConf.ConfVars.HIVE_METRICS_CLASS, CodahaleMetrics.class.getCanonicalName());
- conf.setVar(HiveConf.ConfVars.HIVE_METRICS_REPORTER, MetricsReporting.JSON_FILE.name() + "," + MetricsReporting.JMX.name());
+ conf.setVar(HiveConf.ConfVars.HIVE_CODAHALE_METRICS_REPORTER_CLASSES,
+ "org.apache.hadoop.hive.common.metrics.metrics2.JsonFileMetricsReporter, "
+ + "org.apache.hadoop.hive.common.metrics.metrics2.JmxMetricsReporter");
conf.setVar(HiveConf.ConfVars.HIVE_METRICS_JSON_FILE_LOCATION, jsonReportFile.toString());
conf.setVar(HiveConf.ConfVars.HIVE_METRICS_JSON_FILE_INTERVAL, "100ms");
http://git-wip-us.apache.org/repos/asf/hive/blob/bf98700a/common/src/test/org/apache/hadoop/hive/common/metrics/metrics2/TestCodahaleReportersConf.java
----------------------------------------------------------------------
diff --git a/common/src/test/org/apache/hadoop/hive/common/metrics/metrics2/TestCodahaleReportersConf.java b/common/src/test/org/apache/hadoop/hive/common/metrics/metrics2/TestCodahaleReportersConf.java
new file mode 100644
index 0000000..d694e26
--- /dev/null
+++ b/common/src/test/org/apache/hadoop/hive/common/metrics/metrics2/TestCodahaleReportersConf.java
@@ -0,0 +1,145 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.common.metrics.metrics2;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import java.lang.reflect.InvocationTargetException;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.hive.common.metrics.MetricsTestUtils;
+import org.apache.hadoop.hive.common.metrics.common.MetricsFactory;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.File;
+
+/**
+ * Unit tests for Codahale reporter config backward compatibility
+ */
+public class TestCodahaleReportersConf {
+
+ private static File workDir = new File(System.getProperty("test.tmp.dir"));
+ private static File jsonReportFile;
+
+ @After
+ public void after() throws Exception {
+ MetricsFactory.close();
+ }
+
+ /**
+ * Tests that the deprecated HIVE_METRICS_REPORTER config is used if the HIVE_CODAHALE_METRICS_REPORTER_CLASSES is missing.
+ */
+ @Test
+ public void testFallbackToDeprecatedConfig() throws Exception {
+
+ HiveConf conf = new HiveConf();
+
+ jsonReportFile = new File(workDir, "json_reporting");
+ jsonReportFile.delete();
+
+ conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, "local");
+ conf.setVar(HiveConf.ConfVars.HIVE_METRICS_CLASS, CodahaleMetrics.class.getCanonicalName());
+ conf.setVar(HiveConf.ConfVars.HIVE_METRICS_REPORTER, "JMX, JSON");
+ conf.setVar(HiveConf.ConfVars.HIVE_METRICS_JSON_FILE_LOCATION, jsonReportFile.toString());
+ conf.setVar(HiveConf.ConfVars.HIVE_METRICS_JSON_FILE_INTERVAL, "100ms");
+
+ MetricsFactory.init(conf);
+
+ int runs = 5;
+ for (int i = 0; i < runs; i++) {
+ MetricsFactory.getInstance().incrementCounter("count2");
+ }
+
+ // we expect json file to be updated
+ byte[] jsonData = MetricsTestUtils.getFileData(jsonReportFile.getAbsolutePath(), 2000, 3);
+ ObjectMapper objectMapper = new ObjectMapper();
+
+ JsonNode rootNode = objectMapper.readTree(jsonData);
+ JsonNode countersNode = rootNode.path("counters");
+ JsonNode methodCounterNode = countersNode.path("count2");
+ JsonNode countNode = methodCounterNode.path("count");
+ Assert.assertEquals(countNode.asInt(), 5);
+ }
+
+ /**
+ * Tests that the deprecated HIVE_METRICS_REPORTER config is not used if
+ * HIVE_CODAHALE_METRICS_REPORTER_CLASSES is present.
+ *
+ * The deprecated config specifies json reporters whereas the newer one doesn't. Validates that
+ * the JSON file is not created.
+ */
+ @Test
+ public void testNoFallback() throws Exception {
+
+ HiveConf conf = new HiveConf();
+
+ jsonReportFile = new File(workDir, "json_reporting");
+ jsonReportFile.delete();
+
+ conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, "local");
+ conf.setVar(HiveConf.ConfVars.HIVE_METRICS_CLASS, CodahaleMetrics.class.getCanonicalName());
+ conf.setVar(HiveConf.ConfVars.HIVE_METRICS_REPORTER, "JMX, JSON");
+ conf.setVar(HiveConf.ConfVars.HIVE_CODAHALE_METRICS_REPORTER_CLASSES,
+ "org.apache.hadoop.hive.common.metrics.metrics2.JmxMetricsReporter");
+ conf.setVar(HiveConf.ConfVars.HIVE_METRICS_JSON_FILE_LOCATION, jsonReportFile.toString());
+ conf.setVar(HiveConf.ConfVars.HIVE_METRICS_JSON_FILE_INTERVAL, "100ms");
+
+ MetricsFactory.init(conf);
+
+ int runs = 5;
+ for (int i = 0; i < runs; i++) {
+ MetricsFactory.getInstance().incrementCounter("count2");
+ }
+
+ Assert.assertFalse(jsonReportFile.exists());
+ }
+
+ /**
+ * Tests that the deprecated HIVE_METRICS_REPORTER config is not used if
+ * HIVE_CODAHALE_METRICS_REPORTER_CLASSES is present but incorrect.
+ *
+ * The deprecated config specifies json reporters whereas the newer one doesn't. Validates that
+ * the JSON file is not created.
+ */
+ @Test
+ public void testNoFallbackOnIncorrectConf() throws Exception {
+
+ HiveConf conf = new HiveConf();
+
+ jsonReportFile = new File(workDir, "json_reporting");
+ jsonReportFile.delete();
+
+ conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, "local");
+ conf.setVar(HiveConf.ConfVars.HIVE_METRICS_CLASS, CodahaleMetrics.class.getCanonicalName());
+ conf.setVar(HiveConf.ConfVars.HIVE_METRICS_REPORTER, "JMX, JSON");
+ conf.setVar(HiveConf.ConfVars.HIVE_CODAHALE_METRICS_REPORTER_CLASSES,
+ "org.apache.hadoop.hive.common.metrics.NonExistentReporter");
+ conf.setVar(HiveConf.ConfVars.HIVE_METRICS_JSON_FILE_LOCATION, jsonReportFile.toString());
+ conf.setVar(HiveConf.ConfVars.HIVE_METRICS_JSON_FILE_INTERVAL, "100ms");
+
+ try {
+ MetricsFactory.init(conf);
+ } catch (InvocationTargetException expectedException) {
+
+ }
+
+ Assert.assertFalse(jsonReportFile.exists());
+ }
+}