You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nemo.apache.org by wy...@apache.org on 2019/04/05 14:56:51 UTC
[incubator-nemo] branch master updated: [NEMO-375] Add option to
turn off metric collection to DB (#211)
This is an automated email from the ASF dual-hosted git repository.
wylee pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-nemo.git
The following commit(s) were added to refs/heads/master by this push:
new 6eea2aa [NEMO-375] Add option to turn off metric collection to DB (#211)
6eea2aa is described below
commit 6eea2aa500d335a86aa8610bd194a9275ef9bf9c
Author: Won Wook SONG <wo...@apache.org>
AuthorDate: Fri Apr 5 23:56:46 2019 +0900
[NEMO-375] Add option to turn off metric collection to DB (#211)
JIRA: [NEMO-375: Add option to turn off metric collection to DB](https://issues.apache.org/jira/projects/NEMO/issues/NEMO-375)
**Major changes:**
- Adds configuration for the DB to turn off metric collection to DB by default.
- Adds proper README docs
**Minor changes to note:**
- None
**Tests for the changes:**
- None
**Other comments:**
- None
Closes #211
---
README.md | 7 +++++++
conf/src/main/java/org/apache/nemo/conf/JobConf.java | 8 ++++++++
.../main/java/org/apache/nemo/runtime/master/RuntimeMaster.java | 8 +++++++-
3 files changed, 22 insertions(+), 1 deletion(-)
diff --git a/README.md b/README.md
index 887f05f..f429b27 100644
--- a/README.md
+++ b/README.md
@@ -155,6 +155,13 @@ Nemo Compiler and Engine can store JSON representation of intermediate DAGs.
-user_args "`pwd`/examples/resources/inputs/test_input_als 10 3"
```
+## Options for writing metric results to databases.
+
+* `-db_enabled`: Whether or not to turn on the DB (`true` or `false`).
+* `-db_address`: Address of the DB. (ex. PostgreSQL DB starts with `jdbc:postgresql://...`)
+* `-db_id` : ID of the DB from the given address.
+* `-db_password`: Credentials for the DB from the given address.
+
## Speeding up builds
* To exclude Spark related packages: mvn clean install -T 2C -DskipTests -pl \\!compiler/frontend/spark,\\!examples/spark
* To exclude Beam related packages: mvn clean install -T 2C -DskipTests -pl \\!compiler/frontend/beam,\\!examples/beam
diff --git a/conf/src/main/java/org/apache/nemo/conf/JobConf.java b/conf/src/main/java/org/apache/nemo/conf/JobConf.java
index 83d8957..23cf597 100644
--- a/conf/src/main/java/org/apache/nemo/conf/JobConf.java
+++ b/conf/src/main/java/org/apache/nemo/conf/JobConf.java
@@ -83,6 +83,14 @@ public final class JobConf extends ConfigurationModuleBuilder {
public final class EnvironmentType implements Name<String> {
}
+ ///////////////////////// DB Configurations
+ /**
+ * Specified whether or not to enable writing metrics to DB or not.
+ */
+ @NamedParameter(doc = "Boolean flag for enabling DB metrics", short_name = "db_enabled", default_value = "false")
+ public final class DBEnabled implements Name<Boolean> {
+ }
+
/**
* Address pointing to the DB for saving metrics.
*/
diff --git a/runtime/master/src/main/java/org/apache/nemo/runtime/master/RuntimeMaster.java b/runtime/master/src/main/java/org/apache/nemo/runtime/master/RuntimeMaster.java
index ea6c31c..2309490 100644
--- a/runtime/master/src/main/java/org/apache/nemo/runtime/master/RuntimeMaster.java
+++ b/runtime/master/src/main/java/org/apache/nemo/runtime/master/RuntimeMaster.java
@@ -104,6 +104,7 @@ public final class RuntimeMaster {
private final ObjectMapper objectMapper;
private final String jobId;
private final String dagDirectory;
+ private final Boolean dbEnabled;
private final String dbAddress;
private final String dbId;
private final String dbPassword;
@@ -125,6 +126,7 @@ public final class RuntimeMaster {
* @param clientRPC the RPC channel to communicate with the client.
* @param planStateManager the manager that keeps track of the plan state.
* @param jobId the Job ID, provided by the user.
+ * @param dbEnabled whether or not the DB is enabled, provided by the user.
* @param dbAddress the DB Address, provided by the user.
* @param dbId the ID for the given DB.
* @param dbPassword the password for the given DB.
@@ -139,6 +141,7 @@ public final class RuntimeMaster {
final ClientRPC clientRPC,
final PlanStateManager planStateManager,
@Parameter(JobConf.JobId.class) final String jobId,
+ @Parameter(JobConf.DBEnabled.class) final Boolean dbEnabled,
@Parameter(JobConf.DBAddress.class) final String dbAddress,
@Parameter(JobConf.DBId.class) final String dbId,
@Parameter(JobConf.DBPasswd.class) final String dbPassword,
@@ -169,6 +172,7 @@ public final class RuntimeMaster {
this.metricManagerMaster = metricManagerMaster;
this.jobId = jobId;
this.dagDirectory = dagDirectory;
+ this.dbEnabled = dbEnabled;
this.dbAddress = dbAddress;
this.dbId = dbId;
this.dbPassword = dbPassword;
@@ -225,7 +229,9 @@ public final class RuntimeMaster {
metricStore.dumpAllMetricToFile(Paths.get(dagDirectory,
"Metric_" + jobId + "_" + System.currentTimeMillis() + ".json").toString());
- metricStore.saveOptimizationMetricsToDB(dbAddress, jobId, dbId, dbPassword);
+ if (this.dbEnabled) {
+ metricStore.saveOptimizationMetricsToDB(dbAddress, jobId, dbId, dbPassword);
+ }
}
/**