You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@heron.apache.org by nw...@apache.org on 2019/06/18 15:22:49 UTC
[incubator-heron] branch master updated: Bug: Read override config
in Metrics Manager (#3293)
This is an automated email from the ASF dual-hosted git repository.
nwang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-heron.git
The following commit(s) were added to refs/heads/master by this push:
new de1df48 Bug: Read override config in Metrics Manager (#3293)
de1df48 is described below
commit de1df488856c61269b8a7bd032401a8a6ce5a89a
Author: Rohan Agarwal <ro...@gmail.com>
AuthorDate: Tue Jun 18 08:22:43 2019 -0700
Bug: Read override config in Metrics Manager (#3293)
* read override config in metrics mgr
* add file name to exception message
* refactor MetricsSinksConfig, fix unit tests
* retrigger ci for flaky integration test
---
.../heron/metricscachemgr/MetricsCacheManager.java | 3 +-
.../metricscache/MetricsCacheTest.java | 2 +-
.../apache/heron/metricsmgr/MetricsManager.java | 5 ++-
.../heron/metricsmgr/MetricsSinksConfig.java | 46 +++++++++++++---------
.../heron/scheduler/nomad/NomadScheduler.java | 6 +--
5 files changed, 36 insertions(+), 26 deletions(-)
diff --git a/heron/metricscachemgr/src/java/org/apache/heron/metricscachemgr/MetricsCacheManager.java b/heron/metricscachemgr/src/java/org/apache/heron/metricscachemgr/MetricsCacheManager.java
index dc05f70..2c637f7 100644
--- a/heron/metricscachemgr/src/java/org/apache/heron/metricscachemgr/MetricsCacheManager.java
+++ b/heron/metricscachemgr/src/java/org/apache/heron/metricscachemgr/MetricsCacheManager.java
@@ -342,7 +342,8 @@ public class MetricsCacheManager {
LOG.info("System Config: " + systemConfig);
// read sink config file
- MetricsSinksConfig sinksConfig = new MetricsSinksConfig(metricsSinksConfigFilename);
+ MetricsSinksConfig sinksConfig = new MetricsSinksConfig(metricsSinksConfigFilename,
+ overrideConfigFilename);
LOG.info("Sinks Config: " + sinksConfig.toString());
// build config from cli
diff --git a/heron/metricscachemgr/tests/java/org/apache/heron/metricscachemgr/metricscache/MetricsCacheTest.java b/heron/metricscachemgr/tests/java/org/apache/heron/metricscachemgr/metricscache/MetricsCacheTest.java
index 5875731..dbfdc48 100644
--- a/heron/metricscachemgr/tests/java/org/apache/heron/metricscachemgr/metricscache/MetricsCacheTest.java
+++ b/heron/metricscachemgr/tests/java/org/apache/heron/metricscachemgr/metricscache/MetricsCacheTest.java
@@ -42,7 +42,7 @@ public class MetricsCacheTest {
SystemConfig systemConfig = SystemConfig.newBuilder(true)
.putAll(CONFIG_SYSTEM_PATH, true)
.build();
- MetricsSinksConfig sinksConfig = new MetricsSinksConfig(CONFIG_SINK_PATH);
+ MetricsSinksConfig sinksConfig = new MetricsSinksConfig(CONFIG_SINK_PATH, null);
// initialize metric cache, except looper
MetricsCache mc = new MetricsCache(systemConfig, sinksConfig, new NIOLooper());
diff --git a/heron/metricsmgr/src/java/org/apache/heron/metricsmgr/MetricsManager.java b/heron/metricsmgr/src/java/org/apache/heron/metricsmgr/MetricsManager.java
index b2076cd..8d4ab5e 100644
--- a/heron/metricsmgr/src/java/org/apache/heron/metricsmgr/MetricsManager.java
+++ b/heron/metricsmgr/src/java/org/apache/heron/metricsmgr/MetricsManager.java
@@ -382,14 +382,15 @@ public class MetricsManager {
LoggingHelper.addLoggingHandler(new ErrorReportLoggingHandler());
LOG.info(String.format("Starting Metrics Manager for topology %s with topologyId %s with "
- + "Metrics Manager Id %s, Merics Manager Port: %d, for cluster/role/env %s.",
+ + "Metrics Manager Id %s, Metrics Manager Port: %d, for cluster/role/env %s.",
topologyName, topologyId, metricsmgrId, metricsPort,
String.format("%s/%s/%s", cluster, role, environment)));
LOG.info("System Config: " + systemConfig);
// Populate the config
- MetricsSinksConfig sinksConfig = new MetricsSinksConfig(metricsSinksConfigFilename);
+ MetricsSinksConfig sinksConfig = new MetricsSinksConfig(metricsSinksConfigFilename,
+ overrideConfigFilename);
LOG.info("Sinks Config:" + sinksConfig.toString());
diff --git a/heron/metricsmgr/src/java/org/apache/heron/metricsmgr/MetricsSinksConfig.java b/heron/metricsmgr/src/java/org/apache/heron/metricsmgr/MetricsSinksConfig.java
index 28601b7..ff989cd 100644
--- a/heron/metricsmgr/src/java/org/apache/heron/metricsmgr/MetricsSinksConfig.java
+++ b/heron/metricsmgr/src/java/org/apache/heron/metricsmgr/MetricsSinksConfig.java
@@ -19,19 +19,20 @@
package org.apache.heron.metricsmgr;
-import java.io.File;
import java.io.FileInputStream;
-import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.yaml.snakeyaml.Yaml;
-import org.apache.heron.common.basics.SysUtils;
import org.apache.heron.common.basics.TypeUtils;
+@SuppressWarnings("unchecked")
public class MetricsSinksConfig {
public static final String CONFIG_KEY_METRICS_SINKS = "sinks";
public static final String CONFIG_KEY_CLASSNAME = "class";
@@ -41,22 +42,29 @@ public class MetricsSinksConfig {
private final Map<String, Map<String, Object>> sinksConfigs = new HashMap<>();
- @SuppressWarnings("unchecked")
- public MetricsSinksConfig(String filename) throws FileNotFoundException {
- FileInputStream fin = new FileInputStream(new File(filename));
- try {
- Yaml yaml = new Yaml();
- Map<Object, Object> ret = (Map<Object, Object>) yaml.load(fin);
-
- if (ret == null) {
- throw new RuntimeException("Could not parse metrics-sinks config file");
- } else {
- for (String sinkId : TypeUtils.getListOfStrings(ret.get(CONFIG_KEY_METRICS_SINKS))) {
- sinksConfigs.put(sinkId, (Map<String, Object>) ret.get(sinkId));
- }
- }
- } finally {
- SysUtils.closeIgnoringExceptions(fin);
+ public MetricsSinksConfig(String metricsSinksConfigFilename, String overrideConfigFilename)
+ throws IOException {
+ Map<Object, Object> allConfig = new HashMap<>();
+ allConfig.putAll(readConfig(metricsSinksConfigFilename));
+ allConfig.putAll(readConfig(overrideConfigFilename));
+
+ if (allConfig.isEmpty()) {
+ throw new RuntimeException("Missing required config 'sinks' for metrics manager");
+ }
+
+ for (String sinkId : TypeUtils.getListOfStrings(allConfig.get(CONFIG_KEY_METRICS_SINKS))) {
+ sinksConfigs.put(sinkId, (Map<String, Object>) allConfig.get(sinkId));
+ }
+ }
+
+ private Map<Object, Object> readConfig(String configFile) throws IOException {
+ if (configFile == null) {
+ return Collections.emptyMap();
+ }
+
+ Yaml yaml = new Yaml();
+ try (InputStream inputStream = new FileInputStream(configFile)) {
+ return (Map<Object, Object>) yaml.load(inputStream);
}
}
diff --git a/heron/schedulers/src/java/org/apache/heron/scheduler/nomad/NomadScheduler.java b/heron/schedulers/src/java/org/apache/heron/scheduler/nomad/NomadScheduler.java
index 7f7e1c5..b1f80c5 100644
--- a/heron/schedulers/src/java/org/apache/heron/scheduler/nomad/NomadScheduler.java
+++ b/heron/schedulers/src/java/org/apache/heron/scheduler/nomad/NomadScheduler.java
@@ -19,7 +19,6 @@
package org.apache.heron.scheduler.nomad;
-import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
@@ -581,8 +580,9 @@ public class NomadScheduler implements IScheduler {
static String getPrometheusMetricsFile(Config config) {
MetricsSinksConfig metricsSinksConfig;
try {
- metricsSinksConfig = new MetricsSinksConfig(Context.metricsSinksFile(config));
- } catch (FileNotFoundException e) {
+ metricsSinksConfig = new MetricsSinksConfig(Context.metricsSinksFile(config),
+ Context.overrideFile(config));
+ } catch (IOException e) {
return null;
}