You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@submarine.apache.org by zh...@apache.org on 2019/11/16 07:32:37 UTC
[submarine] branch master updated: SUBMARINE-276. Submarine runtime
class config should be defined in the new configuration class.
This is an automated email from the ASF dual-hosted git repository.
zhouquan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/submarine.git
The following commit(s) were added to refs/heads/master by this push:
new 7a04419 SUBMARINE-276. Submarine runtime class config should be defined in the new configuration class.
7a04419 is described below
commit 7a04419a96e403548326a58c16f98184f496492e
Author: Zac Zhou <zh...@apache.org>
AuthorDate: Wed Nov 13 18:15:26 2019 +0800
SUBMARINE-276. Submarine runtime class config should be defined in the new configuration class.
### What is this PR for?
There are two SubmarineConfiguration classes in the submarine project.
Let's combine them into the new configuration class org.apache.submarine.commons.utils.SubmarineConfiguration.
### What type of PR is it?
Improvement
### What is the Jira issue?
https://issues.apache.org/jira/browse/SUBMARINE-276
### How should this be tested?
https://travis-ci.org/yuanzac/hadoop-submarine/builds/611726441?utm_medium=notification&utm_source=github_status
### Screenshots (if appropriate)
### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? Yes
* Does this needs documentation? Yes
Author: Zac Zhou <zh...@apache.org>
Closes #92 from yuanzac/topic/SUBMARINE-276 and squashes the following commits:
f48ff17 [Zac Zhou] SUBMARINE-276. Submarine runtime class config should be defined in the new configuration class.
---
conf/submarine-site.xml | 6 ++
conf/submarine-site.xml.template | 6 ++
dev-support/mini-submarine/README.md | 2 +-
dev-support/mini-submarine/conf/bootstrap.sh | 2 +-
.../conf/{submarine.xml => submarine-site.xml} | 0
docs/helper/QuickStart.md | 4 +-
pom.xml | 10 ++++
.../apache/submarine/client/cli/AbstractCli.java | 4 +-
.../java/org/apache/submarine/client/cli/Cli.java | 2 +-
.../submarine/commons/cluster/ClusterManager.java | 2 +-
.../submarine/commons/cluster/ClusterMonitor.java | 2 +-
.../commons/cluster/ClusterMultiNodeTest.java | 2 +-
submarine-commons/commons-runtime/pom.xml | 11 ++++
.../submarine/commons/runtime/ClientContext.java | 12 ++--
.../submarine/commons/runtime/RuntimeFactory.java | 10 ++--
.../runtime/conf/SubmarineConfiguration.java | 69 ----------------------
submarine-commons/commons-utils/pom.xml | 7 ---
.../commons/utils/SubmarineConfiguration.java | 48 ++++++++++-----
.../org/apache/submarine/jobserver/JobServer.java | 2 +-
.../submarine/server/submitter/yarn/YarnUtils.java | 2 +-
.../yarnservice/FileSystemOperations.java | 10 ++--
.../TestYarnServiceRunJobCliLocalization.java | 11 ++--
.../submarine/interpreter/InterpreterProcess.java | 2 +-
.../interpreter/InterpreterClusterTest.java | 2 +-
.../org/apache/submarine/database/MyBatisUtil.java | 4 +-
.../apache/submarine/server/WorkbenchServer.java | 10 +++-
.../server/WorkbenchClusterServerTest.java | 2 +-
27 files changed, 113 insertions(+), 131 deletions(-)
diff --git a/conf/submarine-site.xml b/conf/submarine-site.xml
index 8f77a54..a8e7794 100755
--- a/conf/submarine-site.xml
+++ b/conf/submarine-site.xml
@@ -20,6 +20,12 @@
<configuration>
<property>
+ <name>submarine.runtime.class</name>
+ <value>org.apache.submarine.server.submitter.yarn.YarnRuntimeFactory</value>
+ <description>RuntimeFactory for Submarine jobs</description>
+ </property>
+
+ <property>
<name>workbench.server.addr</name>
<value>0.0.0.0</value>
<description>Server address</description>
diff --git a/conf/submarine-site.xml.template b/conf/submarine-site.xml.template
index 8f77a54..a8e7794 100755
--- a/conf/submarine-site.xml.template
+++ b/conf/submarine-site.xml.template
@@ -20,6 +20,12 @@
<configuration>
<property>
+ <name>submarine.runtime.class</name>
+ <value>org.apache.submarine.server.submitter.yarn.YarnRuntimeFactory</value>
+ <description>RuntimeFactory for Submarine jobs</description>
+ </property>
+
+ <property>
<name>workbench.server.addr</name>
<value>0.0.0.0</value>
<description>Server address</description>
diff --git a/dev-support/mini-submarine/README.md b/dev-support/mini-submarine/README.md
index faaae88..185e54f 100644
--- a/dev-support/mini-submarine/README.md
+++ b/dev-support/mini-submarine/README.md
@@ -247,7 +247,7 @@ cd && cd spark-script && ./run_spark.sh
1. Submarine package name error
- Because the package name of submarine 0.3.0 or higher has been changed from `apache.hadoop.yarn.submarine` to `apache.submarine`, So you need to set the Runtime settings in the `/usr/local/hadoop/etc/hadoop/submarine.xml` file.
+ Because the package name of submarine 0.3.0 or higher has been changed from `apache.hadoop.yarn.submarine` to `apache.submarine`, So you need to set the Runtime settings in the `/usr/local/hadoop/etc/hadoop/submarine-site.xml` file.
```
<configuration>
diff --git a/dev-support/mini-submarine/conf/bootstrap.sh b/dev-support/mini-submarine/conf/bootstrap.sh
index f07c63d..68b2f11 100755
--- a/dev-support/mini-submarine/conf/bootstrap.sh
+++ b/dev-support/mini-submarine/conf/bootstrap.sh
@@ -27,7 +27,7 @@ CONFIG_DIR="/tmp/hadoop-config"
# Copy config files from volume mount
-for f in slaves core-site.xml hdfs-site.xml mapred-site.xml yarn-site.xml container-executor.cfg capacity-scheduler.xml node-resources.xml resource-types.xml submarine.xml; do
+for f in slaves core-site.xml hdfs-site.xml mapred-site.xml yarn-site.xml container-executor.cfg capacity-scheduler.xml node-resources.xml resource-types.xml submarine-site.xml; do
if [[ -e ${CONFIG_DIR}/$f ]]; then
cp ${CONFIG_DIR}/$f $HADOOP_PREFIX/etc/hadoop/$f
else
diff --git a/dev-support/mini-submarine/conf/submarine.xml b/dev-support/mini-submarine/conf/submarine-site.xml
similarity index 100%
rename from dev-support/mini-submarine/conf/submarine.xml
rename to dev-support/mini-submarine/conf/submarine-site.xml
diff --git a/docs/helper/QuickStart.md b/docs/helper/QuickStart.md
index 4507475..dd5c6ac 100644
--- a/docs/helper/QuickStart.md
+++ b/docs/helper/QuickStart.md
@@ -47,7 +47,7 @@ Note that if you want to quickly try Submarine on new or existing YARN cluster,
For the environment setup, please check [InstallationGuide](InstallationGuide.md) or [InstallationGuideCN](InstallationGuideChineseVersion.md)
-Once the applicable runtime is chosen and environment is ready, a `submarine.xml` need to be created under `$HADOOP_CONF_DIR`. To use the TonY runtime, please set below value in the submarine configuration.
+Once the applicable runtime is chosen and environment is ready, a `submarine-site.xml` need to be created under `$HADOOP_CONF_DIR`. To use the TonY runtime, please set below value in the submarine configuration.
|Configuration Name | Description |
|:---- |:---- |
@@ -55,7 +55,7 @@ Once the applicable runtime is chosen and environment is ready, a `submarine.xml
<br />
-A sample `submarine.xml` is here:
+A sample `submarine-site.xml` is here:
```java
<?xml version="1.0"?>
<configuration>
diff --git a/pom.xml b/pom.xml
index 8936bae..82234c7 100644
--- a/pom.xml
+++ b/pom.xml
@@ -247,6 +247,16 @@
<version>${commons-io.version}</version>
</dependency>
<dependency>
+ <groupId>commons-configuration</groupId>
+ <artifactId>commons-configuration</artifactId>
+ <version>${commons-configuration.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ <version>${commons-lang.version}</version>
+ </dependency>
+ <dependency>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
<version>${zookeeper.version}</version>
diff --git a/submarine-client/src/main/java/org/apache/submarine/client/cli/AbstractCli.java b/submarine-client/src/main/java/org/apache/submarine/client/cli/AbstractCli.java
index a15397b..177bf16 100644
--- a/submarine-client/src/main/java/org/apache/submarine/client/cli/AbstractCli.java
+++ b/submarine-client/src/main/java/org/apache/submarine/client/cli/AbstractCli.java
@@ -42,11 +42,11 @@ public abstract class AbstractCli implements Tool {
@Override
public void setConf(Configuration conf) {
- clientContext.setSubmarineConfig(conf);
+ clientContext.setYarnConfig(conf);
}
@Override
public Configuration getConf() {
- return clientContext.getSubmarineConfig();
+ return clientContext.getYarnConfig();
}
}
diff --git a/submarine-client/src/main/java/org/apache/submarine/client/cli/Cli.java b/submarine-client/src/main/java/org/apache/submarine/client/cli/Cli.java
index f13d208..4b8d0ad 100644
--- a/submarine-client/src/main/java/org/apache/submarine/client/cli/Cli.java
+++ b/submarine-client/src/main/java/org/apache/submarine/client/cli/Cli.java
@@ -47,7 +47,7 @@ public class Cli {
private static ClientContext getClientContext() {
Configuration conf = new YarnConfiguration();
ClientContext clientContext = new ClientContext();
- clientContext.setConfiguration(conf);
+ clientContext.setYarnConfig(conf);
RuntimeFactory runtimeFactory = RuntimeFactory.getRuntimeFactory(
clientContext);
clientContext.setRuntimeFactory(runtimeFactory);
diff --git a/submarine-commons/commons-cluster/src/main/java/org/apache/submarine/commons/cluster/ClusterManager.java b/submarine-commons/commons-cluster/src/main/java/org/apache/submarine/commons/cluster/ClusterManager.java
index e303ccb..851130e 100644
--- a/submarine-commons/commons-cluster/src/main/java/org/apache/submarine/commons/cluster/ClusterManager.java
+++ b/submarine-commons/commons-cluster/src/main/java/org/apache/submarine/commons/cluster/ClusterManager.java
@@ -125,7 +125,7 @@ import static org.apache.submarine.commons.cluster.meta.ClusterMetaOperation.GET
public abstract class ClusterManager {
private static Logger LOG = LoggerFactory.getLogger(ClusterManager.class);
- public final SubmarineConfiguration sconf = SubmarineConfiguration.create();
+ public final SubmarineConfiguration sconf = SubmarineConfiguration.getInstance();
protected Collection<Node> clusterNodes = new ArrayList<>();
diff --git a/submarine-commons/commons-cluster/src/main/java/org/apache/submarine/commons/cluster/ClusterMonitor.java b/submarine-commons/commons-cluster/src/main/java/org/apache/submarine/commons/cluster/ClusterMonitor.java
index ddc5527..afbd3e6 100644
--- a/submarine-commons/commons-cluster/src/main/java/org/apache/submarine/commons/cluster/ClusterMonitor.java
+++ b/submarine-commons/commons-cluster/src/main/java/org/apache/submarine/commons/cluster/ClusterMonitor.java
@@ -75,7 +75,7 @@ public class ClusterMonitor {
public ClusterMonitor(ClusterManager clusterManager) {
this.clusterManager = clusterManager;
- SubmarineConfiguration sconf = SubmarineConfiguration.create();
+ SubmarineConfiguration sconf = SubmarineConfiguration.getInstance();
heartbeatInterval = sconf.getClusterHeartbeatInterval();
heartbeatTimeout = sconf.getClusterHeartbeatTimeout();
diff --git a/submarine-commons/commons-cluster/src/test/java/org/apache/submarine/commons/cluster/ClusterMultiNodeTest.java b/submarine-commons/commons-cluster/src/test/java/org/apache/submarine/commons/cluster/ClusterMultiNodeTest.java
index 35af147..15aa24f 100644
--- a/submarine-commons/commons-cluster/src/test/java/org/apache/submarine/commons/cluster/ClusterMultiNodeTest.java
+++ b/submarine-commons/commons-cluster/src/test/java/org/apache/submarine/commons/cluster/ClusterMultiNodeTest.java
@@ -58,7 +58,7 @@ public class ClusterMultiNodeTest {
}
}
LOG.info("clusterAddrList = {}", clusterAddrList);
- SubmarineConfiguration sconf = SubmarineConfiguration.create();
+ SubmarineConfiguration sconf = SubmarineConfiguration.getInstance();
sconf.setClusterAddress(clusterAddrList);
// mock cluster manager server
diff --git a/submarine-commons/commons-runtime/pom.xml b/submarine-commons/commons-runtime/pom.xml
index 9e55f1f..86b5923 100644
--- a/submarine-commons/commons-runtime/pom.xml
+++ b/submarine-commons/commons-runtime/pom.xml
@@ -35,6 +35,17 @@
<dependencies>
<dependency>
+ <groupId>org.apache.submarine</groupId>
+ <artifactId>commons-utils</artifactId>
+ <version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>${slf4j.version}</version>
diff --git a/submarine-commons/commons-runtime/src/main/java/org/apache/submarine/commons/runtime/ClientContext.java b/submarine-commons/commons-runtime/src/main/java/org/apache/submarine/commons/runtime/ClientContext.java
index 50f87fa..a2a07c1 100644
--- a/submarine-commons/commons-runtime/src/main/java/org/apache/submarine/commons/runtime/ClientContext.java
+++ b/submarine-commons/commons-runtime/src/main/java/org/apache/submarine/commons/runtime/ClientContext.java
@@ -19,7 +19,7 @@
package org.apache.submarine.commons.runtime;
-import org.apache.submarine.commons.runtime.conf.SubmarineConfiguration;
+import org.apache.submarine.commons.utils.SubmarineConfiguration;
import org.apache.submarine.commons.runtime.fs.DefaultRemoteDirectoryManager;
import org.apache.submarine.commons.runtime.fs.RemoteDirectoryManager;
import org.apache.hadoop.conf.Configuration;
@@ -31,11 +31,11 @@ public class ClientContext {
private volatile RemoteDirectoryManager remoteDirectoryManager;
private YarnClient yarnClient;
- private Configuration submarineConfig;
+ private SubmarineConfiguration submarineConfig;
private RuntimeFactory runtimeFactory;
public ClientContext() {
- submarineConfig = new SubmarineConfiguration();
+ submarineConfig = SubmarineConfiguration.getInstance();
}
public synchronized YarnClient getOrCreateYarnClient() {
@@ -51,7 +51,7 @@ public class ClientContext {
return yarnConf;
}
- public void setConfiguration(Configuration conf) {
+ public void setYarnConfig(Configuration conf) {
this.yarnConf = conf;
}
@@ -66,11 +66,11 @@ public class ClientContext {
return remoteDirectoryManager;
}
- public Configuration getSubmarineConfig() {
+ public SubmarineConfiguration getSubmarineConfig() {
return submarineConfig;
}
- public void setSubmarineConfig(Configuration submarineConfig) {
+ public void setSubmarineConfig(SubmarineConfiguration submarineConfig) {
this.submarineConfig = submarineConfig;
}
diff --git a/submarine-commons/commons-runtime/src/main/java/org/apache/submarine/commons/runtime/RuntimeFactory.java b/submarine-commons/commons-runtime/src/main/java/org/apache/submarine/commons/runtime/RuntimeFactory.java
index f7f1b81..b838a87 100644
--- a/submarine-commons/commons-runtime/src/main/java/org/apache/submarine/commons/runtime/RuntimeFactory.java
+++ b/submarine-commons/commons-runtime/src/main/java/org/apache/submarine/commons/runtime/RuntimeFactory.java
@@ -20,8 +20,7 @@
package org.apache.submarine.commons.runtime;
import com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.submarine.commons.runtime.conf.SubmarineConfiguration;
+import org.apache.submarine.commons.utils.SubmarineConfiguration;
import org.apache.submarine.commons.runtime.exception.SubmarineRuntimeException;
import org.apache.submarine.commons.runtime.fs.SubmarineStorage;
@@ -39,11 +38,10 @@ public abstract class RuntimeFactory {
public static RuntimeFactory getRuntimeFactory(
ClientContext clientContext) {
- Configuration submarineConfiguration =
+ SubmarineConfiguration submarineConfiguration =
clientContext.getSubmarineConfig();
- String runtimeClass = submarineConfiguration.get(
- SubmarineConfiguration.RUNTIME_CLASS,
- SubmarineConfiguration.DEFAULT_RUNTIME_CLASS);
+ String runtimeClass = submarineConfiguration.getString(
+ SubmarineConfiguration.ConfVars.SUBMARINE_RUNTIME_CLASS);
try {
Class<?> runtimeClazz = Class.forName(runtimeClass);
diff --git a/submarine-commons/commons-runtime/src/main/java/org/apache/submarine/commons/runtime/conf/SubmarineConfiguration.java b/submarine-commons/commons-runtime/src/main/java/org/apache/submarine/commons/runtime/conf/SubmarineConfiguration.java
deleted file mode 100644
index b51446b..0000000
--- a/submarine-commons/commons-runtime/src/main/java/org/apache/submarine/commons/runtime/conf/SubmarineConfiguration.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.submarine.commons.runtime.conf;
-
-import org.apache.hadoop.conf.Configuration;
-
-public class SubmarineConfiguration extends Configuration {
- private static final String SUBMARINE_CONFIGURATION_FILE = "submarine.xml";
-
- public static final String SUBMARINE_CONFIGURATION_PREFIX = "submarine.";
-
- public static final String SUBMARINE_LOCALIZATION_PREFIX =
- SUBMARINE_CONFIGURATION_PREFIX + "localization.";
- /**
- * Limit the size of directory/file to be localized.
- * To avoid exhausting local disk space,
- * this limit both remote and local file to be localized
- */
- public static final String LOCALIZATION_MAX_ALLOWED_FILE_SIZE_MB =
- SUBMARINE_LOCALIZATION_PREFIX + "max-allowed-file-size-mb";
-
- // Default 2GB
- public static final long DEFAULT_MAX_ALLOWED_REMOTE_URI_SIZE_MB = 2048;
-
- public static final String SUBMARINE_RUNTIME_APP_TYPE = "SUBMARINE";
-
- public SubmarineConfiguration() {
- this(new Configuration(false), true);
- }
-
- public SubmarineConfiguration(Configuration configuration) {
- this(configuration, false);
- }
-
- public SubmarineConfiguration(Configuration configuration,
- boolean loadLocalConfig) {
- super(configuration);
- if (loadLocalConfig) {
- addResource(SUBMARINE_CONFIGURATION_FILE);
- }
- }
-
- private static final String PREFIX = "submarine.";
-
- public static final String RUNTIME_CLASS = PREFIX + "runtime.class";
- public static final String DEFAULT_RUNTIME_CLASS =
- "org.apache.submarine.server.submitter.yarn.YarnRuntimeFactory";
-
- public void setSubmarineRuntimeClass(String runtimeClass) {
- set(RUNTIME_CLASS, runtimeClass);
- }
-}
diff --git a/submarine-commons/commons-utils/pom.xml b/submarine-commons/commons-utils/pom.xml
index 9aecea4..73396e7 100644
--- a/submarine-commons/commons-utils/pom.xml
+++ b/submarine-commons/commons-utils/pom.xml
@@ -39,18 +39,11 @@
<groupId>commons-configuration</groupId>
<artifactId>commons-configuration</artifactId>
<version>${commons-configuration.version}</version>
- <exclusions>
- <exclusion>
- <groupId>commons-lang</groupId>
- <artifactId>commons-lang</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
<groupId>commons-lang</groupId>
<artifactId>commons-lang</artifactId>
- <version>${commons-lang.version}</version>
</dependency>
<dependency>
diff --git a/submarine-commons/commons-utils/src/main/java/org/apache/submarine/commons/utils/SubmarineConfiguration.java b/submarine-commons/commons-utils/src/main/java/org/apache/submarine/commons/utils/SubmarineConfiguration.java
index 44cea26..f9dea58 100644
--- a/submarine-commons/commons-utils/src/main/java/org/apache/submarine/commons/utils/SubmarineConfiguration.java
+++ b/submarine-commons/commons-utils/src/main/java/org/apache/submarine/commons/utils/SubmarineConfiguration.java
@@ -38,6 +38,8 @@ public class SubmarineConfiguration extends XMLConfiguration {
private static final String SUBMARINE_SITE_XML = "submarine-site.xml";
+ public static final String SUBMARINE_RUNTIME_APP_TYPE = "SUBMARINE";
+
private static SubmarineConfiguration conf;
private Map<String, String> properties = new HashMap<>();
@@ -81,11 +83,19 @@ public class SubmarineConfiguration extends XMLConfiguration {
}
}
- public static synchronized SubmarineConfiguration create() {
- if (conf != null) {
- return conf;
+ public static SubmarineConfiguration getInstance() {
+ if (conf == null) {
+ synchronized (SubmarineConfiguration.class) {
+ if (conf == null) {
+ conf = newInstance();
+ }
+ }
}
+ return conf;
+ }
+ public static SubmarineConfiguration newInstance() {
+ SubmarineConfiguration submarineConfig;
ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
URL url;
@@ -102,25 +112,18 @@ public class SubmarineConfiguration extends XMLConfiguration {
if (url == null) {
LOG.warn("Failed to load configuration, proceeding with a default");
- conf = new SubmarineConfiguration();
+ submarineConfig = new SubmarineConfiguration();
} else {
try {
LOG.info("Load configuration from " + url);
- conf = new SubmarineConfiguration(url);
+ submarineConfig = new SubmarineConfiguration(url);
} catch (ConfigurationException e) {
LOG.warn("Failed to load configuration from " + url + " proceeding with a default", e);
- conf = new SubmarineConfiguration();
+ submarineConfig = new SubmarineConfiguration();
}
}
- LOG.info("Workbench server Host: " + conf.getServerAddress());
- if (conf.useSsl() == false) {
- LOG.info("Workbench server Port: " + conf.getServerPort());
- } else {
- LOG.info("Workbench server SSL Port: " + conf.getServerSslPort());
- }
-
- return conf;
+ return submarineConfig;
}
public String getServerAddress() {
@@ -351,10 +354,18 @@ public class SubmarineConfiguration extends XMLConfiguration {
return getStringValue(propertyName, defaultValue);
}
+ public void setString(ConfVars c, String value) {
+ properties.put(c.getVarName(), value);
+ }
+
public int getInt(ConfVars c) {
return getInt(c.name(), c.getVarName(), c.getIntValue());
}
+ public void setInt(ConfVars c, int value) {
+ properties.put(c.getVarName(), String.valueOf(value));
+ }
+
public int getInt(String envName, String propertyName, int defaultValue) {
if (System.getenv(envName) != null) {
return Integer.parseInt(System.getenv(envName));
@@ -370,6 +381,10 @@ public class SubmarineConfiguration extends XMLConfiguration {
return getLong(c.name(), c.getVarName(), c.getLongValue());
}
+ public void setLong(ConfVars c, long value) {
+ properties.put(c.getVarName(), String.valueOf(value));
+ }
+
public long getLong(String envName, String propertyName, long defaultValue) {
if (System.getenv(envName) != null) {
return Long.parseLong(System.getenv(envName));
@@ -412,6 +427,11 @@ public class SubmarineConfiguration extends XMLConfiguration {
public enum ConfVars {
SUBMARINE_CONF_DIR("submarine.conf.dir", "conf"),
+ SUBMARINE_LOCALIZATION_MAX_ALLOWED_FILE_SIZE_MB(
+ "submarine.localization.max-allowed-file-size-mb", 2048L),
+ SUBMARINE_RUNTIME_CLASS(
+ "submarine.runtime.class",
+ "org.apache.submarine.server.submitter.yarn.YarnRuntimeFactory"),
SERVER_ADDR("workbench.server.addr", "0.0.0.0"),
SERVER_PORT("workbench.server.port", 8080),
SERVER_SSL("workbench.server.ssl", false),
diff --git a/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/JobServer.java b/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/JobServer.java
index 298081e..607d4f9 100644
--- a/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/JobServer.java
+++ b/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/JobServer.java
@@ -45,7 +45,7 @@ public class JobServer {
private static final Logger LOG = LoggerFactory.getLogger(JobServer.class);
- private SubmarineConfiguration conf = SubmarineConfiguration.create();
+ private SubmarineConfiguration conf = SubmarineConfiguration.getInstance();
private Server jobServer;
diff --git a/submarine-server/server-submitter/submitter-yarn/src/main/java/org/apache/submarine/server/submitter/yarn/YarnUtils.java b/submarine-server/server-submitter/submitter-yarn/src/main/java/org/apache/submarine/server/submitter/yarn/YarnUtils.java
index 277c585..79b3da6 100644
--- a/submarine-server/server-submitter/submitter-yarn/src/main/java/org/apache/submarine/server/submitter/yarn/YarnUtils.java
+++ b/submarine-server/server-submitter/submitter-yarn/src/main/java/org/apache/submarine/server/submitter/yarn/YarnUtils.java
@@ -33,7 +33,7 @@ import java.util.Arrays;
import java.util.List;
import java.util.Map;
-import static org.apache.submarine.commons.runtime.conf.SubmarineConfiguration.SUBMARINE_RUNTIME_APP_TYPE;
+import static org.apache.submarine.commons.utils.SubmarineConfiguration.SUBMARINE_RUNTIME_APP_TYPE;
/**
* Utilities for YARN Runtime.
diff --git a/submarine-server/server-submitter/submitter-yarnservice/src/main/java/org/apache/submarine/server/submitter/yarnservice/FileSystemOperations.java b/submarine-server/server-submitter/submitter-yarnservice/src/main/java/org/apache/submarine/server/submitter/yarnservice/FileSystemOperations.java
index 261286e..219d6a7 100644
--- a/submarine-server/server-submitter/submitter-yarnservice/src/main/java/org/apache/submarine/server/submitter/yarnservice/FileSystemOperations.java
+++ b/submarine-server/server-submitter/submitter-yarnservice/src/main/java/org/apache/submarine/server/submitter/yarnservice/FileSystemOperations.java
@@ -29,7 +29,7 @@ import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.yarn.service.api.records.Component;
import org.apache.hadoop.yarn.service.api.records.ConfigFile;
import org.apache.submarine.commons.runtime.ClientContext;
-import org.apache.submarine.commons.runtime.conf.SubmarineConfiguration;
+import org.apache.submarine.commons.utils.SubmarineConfiguration;
import org.apache.submarine.commons.runtime.conf.SubmarineLogs;
import org.apache.submarine.commons.runtime.fs.RemoteDirectoryManager;
import org.apache.submarine.server.submitter.yarnservice.utils.ZipUtilities;
@@ -54,7 +54,7 @@ import java.util.Set;
public class FileSystemOperations {
private static final Logger LOG =
LoggerFactory.getLogger(FileSystemOperations.class);
- private final Configuration submarineConfig;
+ private final SubmarineConfiguration submarineConfig;
private final Configuration yarnConfig;
private Set<Path> uploadedFiles = new HashSet<>();
@@ -179,9 +179,9 @@ public class FileSystemOperations {
} else {
actualSizeByte = FileUtil.getDU(new File(uri));
}
- long maxFileSizeMB = submarineConfig
- .getLong(SubmarineConfiguration.LOCALIZATION_MAX_ALLOWED_FILE_SIZE_MB,
- SubmarineConfiguration.DEFAULT_MAX_ALLOWED_REMOTE_URI_SIZE_MB);
+ long maxFileSizeMB = submarineConfig.getLong(
+ SubmarineConfiguration.ConfVars.
+ SUBMARINE_LOCALIZATION_MAX_ALLOWED_FILE_SIZE_MB);
LOG.info("{} fie/dir: {}, size(Byte):{},"
+ " Allowed max file/dir size: {}",
locationType, uri, actualSizeByte, maxFileSizeMB * 1024 * 1024);
diff --git a/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/client/cli/yarnservice/TestYarnServiceRunJobCliLocalization.java b/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/client/cli/yarnservice/TestYarnServiceRunJobCliLocalization.java
index a6a7731..ac38e15 100644
--- a/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/client/cli/yarnservice/TestYarnServiceRunJobCliLocalization.java
+++ b/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/client/cli/yarnservice/TestYarnServiceRunJobCliLocalization.java
@@ -25,7 +25,7 @@ import org.apache.hadoop.yarn.service.api.records.ConfigFile;
import org.apache.hadoop.yarn.service.api.records.Service;
import org.apache.submarine.client.cli.runjob.RunJobCli;
import org.apache.submarine.commons.runtime.MockClientContext;
-import org.apache.submarine.commons.runtime.conf.SubmarineConfiguration;
+import org.apache.submarine.commons.utils.SubmarineConfiguration;
import org.apache.submarine.commons.runtime.conf.SubmarineLogs;
import org.apache.submarine.commons.runtime.fs.RemoteDirectoryManager;
@@ -415,12 +415,13 @@ public class TestYarnServiceRunJobCliLocalization {
String localUri1 = "/temp/script2";
String containerLocal3 = "./";
- SubmarineConfiguration submarineConf = new SubmarineConfiguration();
+ SubmarineConfiguration submarineConf =
+ SubmarineConfiguration.newInstance();
// Max 10MB, mock remote will always return file size 100MB.
- submarineConf.set(
- SubmarineConfiguration.LOCALIZATION_MAX_ALLOWED_FILE_SIZE_MB,
- "10");
+ submarineConf.setLong(
+ SubmarineConfiguration.ConfVars.
+ SUBMARINE_LOCALIZATION_MAX_ALLOWED_FILE_SIZE_MB, 10L);
mockClientContext.setSubmarineConfig(submarineConf);
assertFalse(SubmarineLogs.isVerbose());
diff --git a/submarine-workbench/interpreter/interpreter-engine/src/main/java/org/apache/submarine/interpreter/InterpreterProcess.java b/submarine-workbench/interpreter/interpreter-engine/src/main/java/org/apache/submarine/interpreter/InterpreterProcess.java
index 202f6a2..5789762 100644
--- a/submarine-workbench/interpreter/interpreter-engine/src/main/java/org/apache/submarine/interpreter/InterpreterProcess.java
+++ b/submarine-workbench/interpreter/interpreter-engine/src/main/java/org/apache/submarine/interpreter/InterpreterProcess.java
@@ -58,7 +58,7 @@ public class InterpreterProcess extends Thread implements Interpreter {
// cluster manager client
private ClusterClient clusterClient = ClusterClient.getInstance();
- private SubmarineConfiguration sconf = SubmarineConfiguration.create();
+ private SubmarineConfiguration sconf = SubmarineConfiguration.getInstance();
protected String interpreterId;
diff --git a/submarine-workbench/interpreter/python-interpreter/src/test/java/org/apache/submarine/interpreter/InterpreterClusterTest.java b/submarine-workbench/interpreter/python-interpreter/src/test/java/org/apache/submarine/interpreter/InterpreterClusterTest.java
index f7673c3..2a2da4a 100644
--- a/submarine-workbench/interpreter/python-interpreter/src/test/java/org/apache/submarine/interpreter/InterpreterClusterTest.java
+++ b/submarine-workbench/interpreter/python-interpreter/src/test/java/org/apache/submarine/interpreter/InterpreterClusterTest.java
@@ -52,7 +52,7 @@ public class InterpreterClusterTest {
public static void startCluster() throws IOException, InterruptedException {
LOG.info("startCluster >>>");
- sconf = SubmarineConfiguration.create();
+ sconf = SubmarineConfiguration.getInstance();
// Set the cluster IP and port
serverHost = NetworkUtils.findAvailableHostAddress();
diff --git a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/MyBatisUtil.java b/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/MyBatisUtil.java
index 40ac9a8..94cb76a 100755
--- a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/MyBatisUtil.java
+++ b/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/MyBatisUtil.java
@@ -47,7 +47,7 @@ public class MyBatisUtil {
checkCalledByTestMethod();
- SubmarineConfiguration conf = SubmarineConfiguration.create();
+ SubmarineConfiguration conf = SubmarineConfiguration.getInstance();
String jdbcClassName = conf.getJdbcDriverClassName();
String jdbcUrl = conf.getJdbcUrl();
String jdbcUserName = conf.getJdbcUserName();
@@ -95,7 +95,7 @@ public class MyBatisUtil {
private static void usingTestDatabase() {
LOG.info("Run the test unit using the test database");
// Run the test unit using the test database
- SubmarineConfiguration conf = SubmarineConfiguration.create();
+ SubmarineConfiguration conf = SubmarineConfiguration.getInstance();
conf.setJdbcUrl("jdbc:mysql://127.0.0.1:3306/submarineDB_test?" +
"useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&" +
"failOverReadOnly=false&zeroDateTimeBehavior=convertToNull&useSSL=false");
diff --git a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/server/WorkbenchServer.java b/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/server/WorkbenchServer.java
index 998d835..ad1dc74 100644
--- a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/server/WorkbenchServer.java
+++ b/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/server/WorkbenchServer.java
@@ -59,12 +59,18 @@ public class WorkbenchServer extends ResourceConfig {
public static Server jettyWebServer;
public static ServiceLocator sharedServiceLocator;
- private static SubmarineConfiguration conf = SubmarineConfiguration.create();
+ private static SubmarineConfiguration conf = SubmarineConfiguration.getInstance();
public static void main(String[] args) throws InterruptedException {
PropertyConfigurator.configure(ClassLoader.getSystemResource("log4j.properties"));
- final SubmarineConfiguration conf = SubmarineConfiguration.create();
+ final SubmarineConfiguration conf = SubmarineConfiguration.getInstance();
+ LOG.info("Workbench server Host: " + conf.getServerAddress());
+ if (conf.useSsl() == false) {
+ LOG.info("Workbench server Port: " + conf.getServerPort());
+ } else {
+ LOG.info("Workbench server SSL Port: " + conf.getServerSslPort());
+ }
jettyWebServer = setupJettyServer(conf);
diff --git a/submarine-workbench/workbench-server/src/test/java/org/apache/submarine/server/WorkbenchClusterServerTest.java b/submarine-workbench/workbench-server/src/test/java/org/apache/submarine/server/WorkbenchClusterServerTest.java
index 8ae8d91..9c1f863 100644
--- a/submarine-workbench/workbench-server/src/test/java/org/apache/submarine/server/WorkbenchClusterServerTest.java
+++ b/submarine-workbench/workbench-server/src/test/java/org/apache/submarine/server/WorkbenchClusterServerTest.java
@@ -45,7 +45,7 @@ public class WorkbenchClusterServerTest {
public static void start() throws Exception {
LOG.info("WorkbenchClusterServerTest:start()");
- SubmarineConfiguration conf = SubmarineConfiguration.create();
+ SubmarineConfiguration conf = SubmarineConfiguration.getInstance();
String serverHost = NetworkUtils.findAvailableHostAddress();
int serverPort = NetworkUtils.findRandomAvailablePortOnAllLocalInterfaces();
String clusterAdd = serverHost + ":" + serverPort;
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@submarine.apache.org
For additional commands, e-mail: dev-help@submarine.apache.org