You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@submarine.apache.org by pi...@apache.org on 2021/10/31 14:43:08 UTC
[submarine] branch master updated: SUBMARINE-1061. Rewrite some
submarine server core syntax
This is an automated email from the ASF dual-hosted git repository.
pingsutw pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/submarine.git
The following commit(s) were added to refs/heads/master by this push:
new d20fa00 SUBMARINE-1061. Rewrite some submarine server core syntax
d20fa00 is described below
commit d20fa007bea5968479b62d7fe362c5fbab539d3d
Author: woodcutter-eric <yi...@gmail.com>
AuthorDate: Mon Oct 25 17:02:38 2021 +0800
SUBMARINE-1061. Rewrite some submarine server core syntax
### What is this PR for?
<!-- A few sentences describing the overall goals of the pull request's commits.
First time? Check out the contributing guide - https://submarine.apache.org/contribution/contributions.html
-->
There are some old features used in the server core now, which can be replaced to a new and safer one.
### What type of PR is it?
[ Improvement | Refactoring]
### Todos
* [x] - Examine server core code to rewrite code to use new language feature
* [x] - Rewrite some code to make the code cleaner
* [x] - Modify some code documentation.
### What is the Jira issue?
<!-- * Open an issue on Jira https://issues.apache.org/jira/browse/SUBMARINE/
* Put link here, and add [SUBMARINE-*Jira number*] in PR title, eg. `SUBMARINE-23. PR title`
-->
https://issues.apache.org/jira/projects/SUBMARINE/issues/SUBMARINE-1061?filter=allissues
### How should this be tested?
<!--
* First time? Setup Travis CI as described on https://submarine.apache.org/contribution/contributions.html#continuous-integration
* Strongly recommended: add automated unit tests for any new or changed behavior
* Outline any manual steps to test the PR here.
-->
### Screenshots (if appropriate)
### Questions:
* Do the license files need updating? No
* Are there breaking changes for older versions? No
* Does this need new documentation? No
Author: woodcutter-eric <yi...@gmail.com>
Signed-off-by: Kevin <pi...@apache.org>
Closes #786 from woodcutter-eric/SUBMARINE-1061 and squashes the following commits:
89045123 [woodcutter-eric] SUBMARINE-1061. Rewrite some syntax.
---
.../apache/submarine/server/SubmarineServer.java | 48 +-
.../server/database/entity/BaseEntity.java | 9 +-
.../server/database/utils/ModelBatisUtil.java | 28 +-
.../server/database/utils/MyBatisUtil.java | 29 +-
.../server/environment/EnvironmentManager.java | 24 +-
.../server/experiment/ExperimentManager.java | 42 +-
.../ExperimentTemplateManager.java | 1036 ++++++++++----------
.../database/mappers/ExperimentTemplateMapper.java | 78 +-
.../submarine/server/notebook/NotebookManager.java | 17 +-
.../notebook/database/service/NotebookService.java | 8 +-
.../submarine/server/response/DictAnnotation.java | 14 +-
.../submarine/server/rest/RestConstants.java | 10 +-
.../submarine/server/utils/GitHttpRequest.java | 37 +-
.../apache/submarine/server/utils/GitUtils.java | 194 ++--
.../workbench/database/utils/DepartmentUtil.java | 12 +-
.../workbench/database/utils/MybatisGenerator.java | 11 +-
.../workbench/websocket/ConnectionManager.java | 4 +-
17 files changed, 768 insertions(+), 833 deletions(-)
diff --git a/submarine-server/server-core/src/main/java/org/apache/submarine/server/SubmarineServer.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/SubmarineServer.java
index b6be245..5fb12c4 100644
--- a/submarine-server/server-core/src/main/java/org/apache/submarine/server/SubmarineServer.java
+++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/SubmarineServer.java
@@ -67,12 +67,12 @@ import java.io.InputStreamReader;
public class SubmarineServer extends ResourceConfig {
private static final Logger LOG = LoggerFactory.getLogger(SubmarineServer.class);
- private static long serverTimeStamp = System.currentTimeMillis();
+ private static final long serverTimeStamp = System.currentTimeMillis();
public static Server jettyWebServer;
public static ServiceLocator sharedServiceLocator;
private static WebAppContext webApp;
- private static SubmarineConfiguration conf = SubmarineConfiguration.getInstance();
+ private static final SubmarineConfiguration conf = SubmarineConfiguration.getInstance();
public static long getServerTimeStamp() {
return serverTimeStamp;
@@ -82,9 +82,8 @@ public class SubmarineServer extends ResourceConfig {
IOException {
PropertyConfigurator.configure(ClassLoader.getSystemResource("log4j.properties"));
- final SubmarineConfiguration conf = SubmarineConfiguration.getInstance();
LOG.info("Submarine server Host: " + conf.getServerAddress());
- if (conf.useSsl() == false) {
+ if (!conf.useSsl()) {
LOG.info("Submarine server Port: " + conf.getServerPort());
} else {
LOG.info("Submarine server SSL Port: " + conf.getServerSslPort());
@@ -134,7 +133,8 @@ public class SubmarineServer extends ResourceConfig {
private static void startServer() throws InterruptedException {
LOG.info("Starting submarine server");
try {
- jettyWebServer.start(); // Instantiates SubmarineServer
+ // Instantiates SubmarineServer
+ jettyWebServer.start();
} catch (Exception e) {
LOG.error("Error while running jettyServer", e);
System.exit(-1);
@@ -235,7 +235,7 @@ public class SubmarineServer extends ResourceConfig {
// Set some timeout options to make debugging easier.
int timeout = 1000 * 30;
connector.setIdleTimeout(timeout);
- connector.setSoLingerTime(-1);
+ // connector.setSoLingerTime(-1);
connector.setHost(conf.getServerAddress());
if (conf.useSsl()) {
connector.setPort(conf.getServerSslPort());
@@ -320,32 +320,30 @@ public class SubmarineServer extends ResourceConfig {
} else {
// Product mode, read from war file
File warFile = webApp.getTempDirectory();
- if (false == warFile.exists()) {
+ if (!warFile.exists()) {
throw new ServletException("Can't found war directory!");
}
indexFile = new File(warFile.getAbsolutePath() + "/webapp/index.html");
}
- InputStreamReader reader = null;
- StringBuffer sbIndexBuf = new StringBuffer();
- try {
- if (indexFile.isFile() && indexFile.exists()) {
- reader = new InputStreamReader(new FileInputStream(indexFile), "GBK");
- BufferedReader bufferedReader = new BufferedReader(reader);
- String lineTxt = null;
-
- while ((lineTxt = bufferedReader.readLine()) != null) {
- sbIndexBuf.append(lineTxt);
- }
- bufferedReader.close();
- } else {
+ // If index.html does not exist, throw ServletException
+ if (!(indexFile.isFile() && indexFile.exists())) {
+ try {
throw new Exception("Can't found index html!");
+ } catch (Exception e) {
+ LOG.error(e.getMessage(), e);
+ throw new ServletException("Can't found index html!");
+ }
+ }
+
+ StringBuilder sbIndexBuf = new StringBuilder();
+ try (InputStreamReader reader =
+ new InputStreamReader(new FileInputStream(indexFile), "GBK");
+ BufferedReader bufferedReader = new BufferedReader(reader);) {
+ String lineTxt = null;
+ while ((lineTxt = bufferedReader.readLine()) != null) {
+ sbIndexBuf.append(lineTxt);
}
- } catch (Exception e) {
- LOG.error(e.getMessage(), e);
- throw new ServletException("Can't found index html!");
- } finally {
- reader.close();
}
response.getWriter().print(sbIndexBuf.toString());
diff --git a/submarine-server/server-core/src/main/java/org/apache/submarine/server/database/entity/BaseEntity.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/database/entity/BaseEntity.java
index 0d3b0c6..507ee9c 100644
--- a/submarine-server/server-core/src/main/java/org/apache/submarine/server/database/entity/BaseEntity.java
+++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/database/entity/BaseEntity.java
@@ -80,7 +80,7 @@ public abstract class BaseEntity {
}
public String toString() {
- StringBuffer buffer = new StringBuffer();
+ StringBuilder buffer = new StringBuilder();
Class clazz = getClass();
String fullName = clazz.getName();
@@ -92,16 +92,13 @@ public abstract class BaseEntity {
Field[] fields = clazz.getDeclaredFields();
Field.setAccessible(fields, true);
- for (int i = 0; i < fields.length; i++) {
- Field field = fields[i];
+ for (Field field: fields) {
try {
buffer.append(field.getName());
buffer.append("=");
buffer.append(field.get(this));
buffer.append(", ");
- } catch (IllegalArgumentException e) {
- e.printStackTrace();
- } catch (IllegalAccessException e) {
+ } catch (IllegalArgumentException | IllegalAccessException e) {
e.printStackTrace();
}
}
diff --git a/submarine-server/server-core/src/main/java/org/apache/submarine/server/database/utils/ModelBatisUtil.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/database/utils/ModelBatisUtil.java
index 9483854..58e2d9f 100644
--- a/submarine-server/server-core/src/main/java/org/apache/submarine/server/database/utils/ModelBatisUtil.java
+++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/database/utils/ModelBatisUtil.java
@@ -32,18 +32,13 @@ import java.util.Properties;
public class ModelBatisUtil {
private static final Logger LOG = LoggerFactory.getLogger(ModelBatisUtil.class);
- private static SqlSessionFactory sqlSessionFactory;
+ private static final SqlSessionFactory sqlSessionFactory;
static {
- Reader reader = null;
- try {
- try {
- reader = Resources.getResourceAsReader("modelbatis-config.xml");
- } catch (IOException e) {
- LOG.error(e.getMessage(), e);
- throw new RuntimeException(e.getMessage());
- }
+ try (Reader reader =
+ Resources.getResourceAsReader("modelbatis-config.xml");
+ ) {
String jdbcClassName = "com.mysql.jdbc.Driver";
String jdbcUrl = "jdbc:mysql://127.0.0.1:3306/mlflowdb";
String jdbcUserName = "mlflow";
@@ -58,21 +53,16 @@ public class ModelBatisUtil {
props.setProperty("jdbc.password", jdbcPassword);
sqlSessionFactory = new SqlSessionFactoryBuilder().build(reader, props);
- } finally {
- try {
- if (null != reader) {
- reader.close();
- }
- } catch (IOException e) {
- LOG.error(e.getMessage(), e);
- }
+ } catch (IOException e) {
+ LOG.error(e.getMessage(), e);
+ throw new RuntimeException(e.getMessage());
}
}
/**
- * Get Session
+ * Get Session.
*
- * @return
+ * @return SqlSession
*/
public static SqlSession getSqlSession() {
return sqlSessionFactory.openSession();
diff --git a/submarine-server/server-core/src/main/java/org/apache/submarine/server/database/utils/MyBatisUtil.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/database/utils/MyBatisUtil.java
index cfd0192..28c4bd4 100755
--- a/submarine-server/server-core/src/main/java/org/apache/submarine/server/database/utils/MyBatisUtil.java
+++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/database/utils/MyBatisUtil.java
@@ -33,18 +33,12 @@ import java.util.Properties;
public class MyBatisUtil {
private static final Logger LOG = LoggerFactory.getLogger(MyBatisUtil.class);
- private static SqlSessionFactory sqlSessionFactory;
+ private static final SqlSessionFactory sqlSessionFactory;
static {
- Reader reader = null;
- try {
- try {
- reader = Resources.getResourceAsReader("mybatis-config.xml");
- } catch (IOException e) {
- LOG.error(e.getMessage(), e);
- throw new RuntimeException(e.getMessage());
- }
-
+ try (Reader reader =
+ Resources.getResourceAsReader("mybatis-config.xml");
+ ) {
checkCalledByTestMethod();
SubmarineConfiguration conf = SubmarineConfiguration.getInstance();
@@ -62,21 +56,16 @@ public class MyBatisUtil {
props.setProperty("jdbc.password", jdbcPassword);
sqlSessionFactory = new SqlSessionFactoryBuilder().build(reader, props);
- } finally {
- try {
- if (null != reader) {
- reader.close();
- }
- } catch (IOException e) {
- LOG.error(e.getMessage(), e);
- }
+ } catch (IOException e) {
+ LOG.error(e.getMessage(), e);
+ throw new RuntimeException(e.getMessage());
}
}
/**
- * Get Session
+ * Get Session.
*
- * @return
+ * @return SqlSession
*/
public static SqlSession getSqlSession() {
return sqlSessionFactory.openSession();
diff --git a/submarine-server/server-core/src/main/java/org/apache/submarine/server/environment/EnvironmentManager.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/environment/EnvironmentManager.java
index 0721746..f129251 100644
--- a/submarine-server/server-core/src/main/java/org/apache/submarine/server/environment/EnvironmentManager.java
+++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/environment/EnvironmentManager.java
@@ -43,7 +43,7 @@ import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
/**
- * Environment Management
+ * Environment Management.
*/
public class EnvironmentManager {
@@ -57,13 +57,13 @@ public class EnvironmentManager {
private static Boolean readedDB = true;
/**
- * Environment Cache
+ * Environment Cache.
*/
private final ConcurrentMap<String, Environment> cachedEnvironments =
new ConcurrentHashMap<>();
/**
- * Get the singleton instance
+ * Get the singleton instance.
* @return object
*/
public static EnvironmentManager getInstance() {
@@ -82,7 +82,7 @@ public class EnvironmentManager {
}
/**
- * Create Environment
+ * Create Environment.
* @param spec environment spec
* @return Environment environment
* @throws SubmarineRuntimeException the service error
@@ -95,7 +95,7 @@ public class EnvironmentManager {
}
/**
- * Update environment
+ * Update environment.
* @param name Name of the environment
* @param spec environment spec
* @return Environment environment
@@ -152,7 +152,7 @@ public class EnvironmentManager {
}
/**
- * Delete environment
+ * Delete environment.
* @param name Name of the environment
* @return Environment environment
* @throws SubmarineRuntimeException the service error
@@ -183,7 +183,7 @@ public class EnvironmentManager {
}
/**
- * Get Environment
+ * Get Environment.
* @param name Name of the environment
* @return Environment environment
* @throws SubmarineRuntimeException the service error
@@ -199,7 +199,7 @@ public class EnvironmentManager {
}
/**
- * List environments
+ * List environments.
* @param status environment status, if null will return all status
* @return environment list
* @throws SubmarineRuntimeException the service error
@@ -210,12 +210,12 @@ public class EnvironmentManager {
new ArrayList<Environment>(cachedEnvironments.values());
// Is it available in cache?
- if (this.readedDB == true) {
+ if (readedDB) {
try (SqlSession sqlSession = MyBatisUtil.getSqlSession()) {
EnvironmentMapper environmentMapper =
sqlSession.getMapper(EnvironmentMapper.class);
- List<EnvironmentEntity> environmentEntitys = environmentMapper.selectAll();
- for (EnvironmentEntity environmentEntity : environmentEntitys) {
+ List<EnvironmentEntity> environmentEntities = environmentMapper.selectAll();
+ for (EnvironmentEntity environmentEntity : environmentEntities) {
if (environmentEntity != null) {
Environment env = new Environment();
env.setEnvironmentSpec(new Gson().fromJson(
@@ -232,7 +232,7 @@ public class EnvironmentManager {
"Unable to get the environment list.");
}
}
- this.readedDB = false;
+ readedDB = false;
return environmentList;
}
diff --git a/submarine-server/server-core/src/main/java/org/apache/submarine/server/experiment/ExperimentManager.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/experiment/ExperimentManager.java
index cb55a25..f0776d1 100644
--- a/submarine-server/server-core/src/main/java/org/apache/submarine/server/experiment/ExperimentManager.java
+++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/experiment/ExperimentManager.java
@@ -52,7 +52,7 @@ import org.slf4j.LoggerFactory;
import org.mlflow.tracking.MlflowClient;
/**
- * It's responsible for managing the experiment CRUD and cache them
+ * It's responsible for managing the experiment CRUD and cache them.
*/
public class ExperimentManager {
private static final Logger LOG = LoggerFactory.getLogger(ExperimentManager.class);
@@ -74,7 +74,7 @@ public class ExperimentManager {
private final ExperimentService experimentService;
/**
- * Get the singleton instance
+ * Get the singleton instance.
*
* @return object
*/
@@ -96,7 +96,7 @@ public class ExperimentManager {
}
/**
- * Create experiment
+ * Create experiment.
*
* @param spec spec
* @return object
@@ -134,7 +134,7 @@ public class ExperimentManager {
}
/**
- * Get experiment
+ * Get experiment.
*
* @param id experiment id
* @return object
@@ -152,7 +152,7 @@ public class ExperimentManager {
}
/**
- * List experiments
+ * List experiments.
*
* @param status status, if null will return all experiments
* @return list
@@ -183,9 +183,9 @@ public class ExperimentManager {
}
/**
- * List experiments
+ * List experiments.
*
- * @param searchTag, if null will return all experiments
+ * @param searchTag String, if null will return all experiments
* @return list
* @throws SubmarineRuntimeException the service error
*/
@@ -222,7 +222,7 @@ public class ExperimentManager {
}
/**
- * Patch the experiment
+ * Patch the experiment.
*
* @param id experiment id
* @param newSpec spec
@@ -253,7 +253,7 @@ public class ExperimentManager {
}
/**
- * Delete experiment
+ * Delete experiment.
*
* @param id experiment id
* @return object
@@ -281,7 +281,7 @@ public class ExperimentManager {
}
/**
- * List experiment logs
+ * List experiment logs.
*
* @param status status, if null will return all experiment logs
* @return log list
@@ -311,7 +311,7 @@ public class ExperimentManager {
}
/**
- * Get experiment log
+ * Get experiment log.
*
* @param id experiment id
* @return object
@@ -327,9 +327,9 @@ public class ExperimentManager {
}
/**
- * Get tensorboard meta data
+ * Get tensorboard meta data.
*
- * @return tensorboardinfo
+ * @return TensorboardInfo
* @throws SubmarineRuntimeException the service error
*/
public TensorboardInfo getTensorboardInfo() throws SubmarineRuntimeException {
@@ -337,9 +337,9 @@ public class ExperimentManager {
}
/**
- * Get mlflow meta data
+ * Get mlflow meta data.
*
- * @return mlflowinfo
+ * @return MlflowInfo
* @throws SubmarineRuntimeException the service error
*/
public MlflowInfo getMLflowInfo() throws SubmarineRuntimeException {
@@ -347,7 +347,7 @@ public class ExperimentManager {
}
/**
- * Create serve
+ * Create serve.
*
* @param spec spec
* @return object
@@ -360,7 +360,7 @@ public class ExperimentManager {
}
/**
- * Delete serve
+ * Delete serve.
*
* @param spec spec
* @return object
@@ -400,11 +400,11 @@ public class ExperimentManager {
}
/**
- * Create a new experiment instance from entity, and filled
+ * Create a new experiment instance from entity, and filled.
* 1. experimentId
* 2. spec
*
- * @param entity
+ * @param entity ExperimentEntity
* @return Experiment
*/
private Experiment buildExperimentFromEntity(ExperimentEntity entity) {
@@ -415,9 +415,9 @@ public class ExperimentManager {
}
/**
- * Create a ExperimentEntity instance from experiment
+ * Create a ExperimentEntity instance from experiment.
*
- * @param experiment
+ * @param experiment Experiment
* @return ExperimentEntity
*/
private ExperimentEntity buildEntityFromExperiment(Experiment experiment) {
diff --git a/submarine-server/server-core/src/main/java/org/apache/submarine/server/experimenttemplate/ExperimentTemplateManager.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/experimenttemplate/ExperimentTemplateManager.java
index 8c3086b..aea2a4f 100644
--- a/submarine-server/server-core/src/main/java/org/apache/submarine/server/experimenttemplate/ExperimentTemplateManager.java
+++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/experimenttemplate/ExperimentTemplateManager.java
@@ -1,518 +1,518 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.submarine.server.experimenttemplate;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import javax.ws.rs.core.Response.Status;
-
-import org.apache.ibatis.session.SqlSession;
-import org.apache.submarine.commons.utils.exception.SubmarineRuntimeException;
-import org.apache.submarine.server.SubmarineServer;
-import org.apache.submarine.server.api.experiment.Experiment;
-import org.apache.submarine.server.api.experiment.ExperimentId;
-import org.apache.submarine.server.api.experimenttemplate.ExperimentTemplate;
-import org.apache.submarine.server.api.experimenttemplate.ExperimentTemplateId;
-import org.apache.submarine.server.api.experimenttemplate.ExperimentTemplateSubmit;
-import org.apache.submarine.server.api.spec.ExperimentSpec;
-import org.apache.submarine.server.api.spec.ExperimentTaskSpec;
-import org.apache.submarine.server.api.spec.ExperimentTemplateParamSpec;
-import org.apache.submarine.server.api.spec.ExperimentTemplateSpec;
-import org.apache.submarine.server.database.utils.MyBatisUtil;
-import org.apache.submarine.server.experiment.ExperimentManager;
-import org.apache.submarine.server.experimenttemplate.database.entity.ExperimentTemplateEntity;
-import org.apache.submarine.server.experimenttemplate.database.mappers.ExperimentTemplateMapper;
-import org.apache.submarine.server.gson.ExperimentIdDeserializer;
-import org.apache.submarine.server.gson.ExperimentIdSerializer;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.esotericsoftware.minlog.Log;
-import com.github.wnameless.json.flattener.JsonFlattener;
-import com.github.wnameless.json.unflattener.JsonUnflattener;
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
-
-/**
- * ExperimentTemplate Management
- */
-public class ExperimentTemplateManager {
-
- private static final Logger LOG = LoggerFactory.getLogger(ExperimentTemplateManager.class);
- private static volatile ExperimentTemplateManager manager;
- private final AtomicInteger experimentTemplateIdCounter = new AtomicInteger(0);
-
- private static final GsonBuilder gsonBuilder = new GsonBuilder()
- .registerTypeAdapter(ExperimentId.class, new ExperimentIdSerializer())
- .registerTypeAdapter(ExperimentId.class, new ExperimentIdDeserializer());
- private static Gson gson = gsonBuilder.setDateFormat("yyyy-MM-dd HH:mm:ss").create();
-
-
-
- /**
- * ExperimentTemplate Cache
- */
- private final ConcurrentMap<String, ExperimentTemplate> cachedExperimentTemplates =
- new ConcurrentHashMap<>();
-
- /**
- * Get the singleton instance
- *
- * @return object
- */
- public static ExperimentTemplateManager getInstance() {
- if (manager == null) {
- synchronized (ExperimentTemplateManager.class) {
- if (manager == null) {
- manager = new ExperimentTemplateManager();
- }
- }
- }
- return manager;
- }
-
- private ExperimentTemplateManager() {
-
- }
-
- /**
- * Create ExperimentTemplate
- *
- * @param spec experimentTemplate spec
- * @return ExperimentTemplate experimentTemplate
- * @throws SubmarineRuntimeException the service error
- */
- public ExperimentTemplate createExperimentTemplate(ExperimentTemplateSpec spec)
- throws SubmarineRuntimeException {
- checkSpec(spec);
- LOG.info("Create ExperimentTemplate using spec: " + spec.toString());
- return createOrUpdateExperimentTemplate(spec, "c");
- }
-
- /**
- * Update experimentTemplate
- *
- * @param name Name of the experimentTemplate
- * @param spec experimentTemplate spec
- * @return ExperimentTemplate experimentTemplate
- * @throws SubmarineRuntimeException the service error
- */
- public ExperimentTemplate updateExperimentTemplate(String name, ExperimentTemplateSpec spec)
- throws SubmarineRuntimeException {
- ExperimentTemplate tpl = getExperimentTemplateDetails(name);
- if (tpl == null) {
- throw new SubmarineRuntimeException(Status.NOT_FOUND.getStatusCode(), "ExperimentTemplate not found.");
- }
- checkSpec(spec);
- LOG.info("Update ExperimentTemplate using spec: " + spec.toString());
- return createOrUpdateExperimentTemplate(spec, "u");
- }
-
- private ExperimentTemplate createOrUpdateExperimentTemplate(ExperimentTemplateSpec spec, String operation) {
-
- spec = addResourcesParameter(spec);
-
- ExperimentTemplateEntity entity = new ExperimentTemplateEntity();
- String experimentTemplateId = generateExperimentTemplateId().toString();
- entity.setId(experimentTemplateId);
- entity.setExperimentTemplateName(spec.getName());
- entity.setExperimentTemplateSpec(gsonBuilder.disableHtmlEscaping().create().toJson(spec));
-
-
- parameterMapping(spec);
-
- try (SqlSession sqlSession = MyBatisUtil.getSqlSession()) {
- ExperimentTemplateMapper experimentTemplateMapper =
- sqlSession.getMapper(ExperimentTemplateMapper.class);
-
- if (operation.equals("c")) {
- experimentTemplateMapper.insert(entity);
- } else {
- experimentTemplateMapper.update(entity);
- }
- sqlSession.commit();
-
- } catch (Exception e) {
- LOG.error(e.getMessage(), e);
- throw new SubmarineRuntimeException(Status.BAD_REQUEST.getStatusCode(),
- "Unable to insert or update the experimentTemplate spec: " + e.getMessage());
- }
-
- ExperimentTemplate experimentTemplate;
- try {
- experimentTemplate = new ExperimentTemplate();
- experimentTemplate.setExperimentTemplateId(ExperimentTemplateId.fromString(experimentTemplateId));
- experimentTemplate.setExperimentTemplateSpec(spec);
-
- } catch (Exception e) {
- LOG.error(e.getMessage(), e);
- throw new SubmarineRuntimeException(Status.BAD_REQUEST.getStatusCode(),
- "Unable to parse the experimentTemplate spec: " + e.getMessage());
- }
- // Update cache
- cachedExperimentTemplates.put(spec.getName(), experimentTemplate);
-
- return experimentTemplate;
- }
-
- private ExperimentTemplateId generateExperimentTemplateId() {
- return ExperimentTemplateId.newInstance(SubmarineServer.getServerTimeStamp(),
- experimentTemplateIdCounter.incrementAndGet());
- }
-
- /**
- * Delete experimentTemplate
- *
- * @param name Name of the experimentTemplate
- * @return ExperimentTemplate experimentTemplate
- * @throws SubmarineRuntimeException the service error
- */
- public ExperimentTemplate deleteExperimentTemplate(String name) throws SubmarineRuntimeException {
- ExperimentTemplate tpl = getExperimentTemplateDetails(name);
- if (tpl == null) {
- throw new SubmarineRuntimeException(Status.NOT_FOUND.getStatusCode(), "ExperimentTemplate not found.");
- }
-
- LOG.info("Delete ExperimentTemplate for " + name);
- try (SqlSession sqlSession = MyBatisUtil.getSqlSession()) {
- ExperimentTemplateMapper experimentTemplateMapper =
- sqlSession.getMapper(ExperimentTemplateMapper.class);
-
- experimentTemplateMapper.delete(name);
- sqlSession.commit();
-
- // Invalidate cache
- cachedExperimentTemplates.remove(name);
- return tpl;
- } catch (Exception e) {
- LOG.error(e.getMessage(), e);
- throw new SubmarineRuntimeException(Status.BAD_REQUEST.getStatusCode(),
- "Unable to delete the experimentTemplate.");
- }
- }
-
- /**
- * Get ExperimentTemplate
- *
- * @param name Name of the experimentTemplate
- * @return ExperimentTemplate experimentTemplate
- * @throws SubmarineRuntimeException the service error
- */
- public ExperimentTemplate getExperimentTemplate(String name) throws SubmarineRuntimeException {
- ExperimentTemplate experimentTemplate = getExperimentTemplateDetails(name);
- if (experimentTemplate == null) {
- throw new SubmarineRuntimeException(Status.NOT_FOUND.getStatusCode(), "ExperimentTemplate not found.");
- }
- return experimentTemplate;
- }
-
- /**
- * List experimentTemplates
- *
- * @param status experimentTemplate status, if null will return all status
- * @return experimentTemplate list
- * @throws SubmarineRuntimeException the service error
- */
- public List<ExperimentTemplate> listExperimentTemplates(String status) throws SubmarineRuntimeException {
- List<ExperimentTemplate> tpls = new ArrayList<>(cachedExperimentTemplates.values());
-
- // Is it available in cache?
- if (tpls != null && tpls.size() != 0) {
- return tpls;
- }
- try (SqlSession sqlSession = MyBatisUtil.getSqlSession()) {
- ExperimentTemplateMapper experimentTemplateMapper =
- sqlSession.getMapper(ExperimentTemplateMapper.class);
-
- List<ExperimentTemplateEntity> experimentTemplateEntitys = experimentTemplateMapper.selectByKey(null);
- for (ExperimentTemplateEntity experimentTemplateEntity : experimentTemplateEntitys) {
- if (experimentTemplateEntity != null) {
- ExperimentTemplate tpl = new ExperimentTemplate();
-
- tpl.setExperimentTemplateSpec(
- gson.fromJson(experimentTemplateEntity.getExperimentTemplateSpec(),
- ExperimentTemplateSpec.class));
- tpls.add(tpl);
- cachedExperimentTemplates.put(tpl.getExperimentTemplateSpec().getName(), tpl);
- }
- }
- } catch (Exception e) {
- LOG.error(e.getMessage(), e);
- throw new SubmarineRuntimeException(Status.BAD_REQUEST.getStatusCode(),
- "Unable to get the experimentTemplate details.");
- }
- return tpls;
- }
-
- private void checkSpec(ExperimentTemplateSpec spec) throws SubmarineRuntimeException {
- if (spec == null) {
- throw new SubmarineRuntimeException(Status.BAD_REQUEST.getStatusCode(),
- "Invalid experimentTemplate spec.");
- }
- }
-
- private ExperimentTemplate getExperimentTemplateDetails(String name) throws SubmarineRuntimeException {
-
- // Is it available in cache?
- ExperimentTemplate tpl = cachedExperimentTemplates.get(name);
- if (tpl != null) {
- return tpl;
- }
- ExperimentTemplateEntity experimentTemplateEntity;
- try (SqlSession sqlSession = MyBatisUtil.getSqlSession()) {
- ExperimentTemplateMapper experimentTemplateMapper =
- sqlSession.getMapper(ExperimentTemplateMapper.class);
-
- experimentTemplateEntity = experimentTemplateMapper.select(name);
-
- if (experimentTemplateEntity != null) {
- tpl = new ExperimentTemplate();
- tpl.setExperimentTemplateSpec(
- gson.fromJson(experimentTemplateEntity.getExperimentTemplateSpec(),
- ExperimentTemplateSpec.class));
- }
- } catch (Exception e) {
- LOG.error(e.getMessage(), e);
- throw new SubmarineRuntimeException(Status.BAD_REQUEST.getStatusCode(),
- "Unable to get the experimentTemplate details.");
- }
- return tpl;
- }
-
-
- /**
- * Create ExperimentTemplate
- *
- * @param SubmittedParam experimentTemplate spec
- * @return Experiment experiment
- * @throws SubmarineRuntimeException the service error
- */
- public Experiment submitExperimentTemplate(ExperimentTemplateSubmit SubmittedParam)
- throws SubmarineRuntimeException {
-
- if (SubmittedParam == null) {
- throw new SubmarineRuntimeException(Status.BAD_REQUEST.getStatusCode(),
- "Invalid ExperimentTemplateSubmit spec.");
- }
-
- ExperimentTemplate experimentTemplate = getExperimentTemplate(SubmittedParam.getName());
- Map<String, String> params = SubmittedParam.getParams();
-
-
- for (ExperimentTemplateParamSpec paramSpec:
- experimentTemplate.getExperimentTemplateSpec().getExperimentTemplateParamSpec()) {
-
- String value = params.get(paramSpec.getName());
- if (value != null) {
- paramSpec.setValue(value);
- }
- }
- ExperimentTemplateSpec spec = experimentTemplate.getExperimentTemplateSpec();
-
- ExperimentSpec experimentSpec = parameterMapping(spec, params);
-
- return ExperimentManager.getInstance().createExperiment(experimentSpec);
- }
-
-
- private ExperimentTemplateSpec addResourcesParameter(ExperimentTemplateSpec tplSpec) {
-
- for (Map.Entry<String, ExperimentTaskSpec> entrySet : tplSpec.getExperimentSpec().getSpec().entrySet()) {
-
- ExperimentTaskSpec taskSpec = entrySet.getValue();
- // parse resourceMap
- taskSpec.setResources(taskSpec.getResources());
-
- ExperimentTemplateParamSpec parm1 = new ExperimentTemplateParamSpec();
- parm1.setName(String.format("spec.%s.replicas", entrySet.getKey()));
- parm1.setValue(taskSpec.getReplicas() == null ? "1" : taskSpec.getReplicas().toString());
- parm1.setRequired("false");
- parm1.setDescription("");
- tplSpec.getExperimentTemplateParamSpec().add(parm1);
-
- ExperimentTemplateParamSpec parm2 = new ExperimentTemplateParamSpec();
- parm2.setName(String.format("spec.%s.resourceMap.cpu", entrySet.getKey()));
- parm2.setValue(taskSpec.getCpu() == null ? "1" : taskSpec.getCpu());
- parm2.setRequired("false");
- parm2.setDescription("");
- tplSpec.getExperimentTemplateParamSpec().add(parm2);
-
- ExperimentTemplateParamSpec parm3 = new ExperimentTemplateParamSpec();
- parm3.setName(String.format("spec.%s.resourceMap.memory", entrySet.getKey()));
- parm3.setValue(taskSpec.getMemory() == null ? "1" : taskSpec.getMemory().toString());
- parm3.setRequired("false");
- parm3.setDescription("");
- tplSpec.getExperimentTemplateParamSpec().add(parm3);
-
- }
- return tplSpec;
- }
-
-
- // use template default value to mapping
- private ExperimentSpec parameterMapping(ExperimentTemplateSpec tplSpec) {
-
- Map<String, String> paramMap = new HashMap<String, String>();
- for (ExperimentTemplateParamSpec parm : tplSpec.getExperimentTemplateParamSpec()) {
- if (parm.getValue() != null) {
- paramMap.put(parm.getName(), parm.getValue());
- } else {
- paramMap.put(parm.getName(), "");
- }
- }
- return parameterMapping(tplSpec, paramMap);
- }
-
- private ExperimentSpec parameterMapping(ExperimentTemplateSpec tplspec, Map<String, String> paramMap) {
-
- String spec = gson.toJson(tplspec.getExperimentSpec());
- Map<String, Object> flattenJson = JsonFlattener.flattenAsMap(spec);
-
- Log.info(flattenJson.toString());
- // illegalParamList: The parameters not in template parameters should not be used
- // Check at submission
- Map<String, ExperimentTemplateParamSpec> tplparamMap = new HashMap<String, ExperimentTemplateParamSpec>();
- for (ExperimentTemplateParamSpec tplParam : tplspec.getExperimentTemplateParamSpec()) {
- tplparamMap.put(tplParam.getName(), tplParam);
- }
- Set<String> illegalParamList = new HashSet<String>();
- for (String key : paramMap.keySet()) {
- if (tplparamMap.get(key) == null) {
- illegalParamList.add(key);
- }
- }
-
- if (illegalParamList.size() > 0) {
- throw new SubmarineRuntimeException(Status.BAD_REQUEST.getStatusCode(),
- "Parameters contains illegal key: " + illegalParamList.toString());
- }
-
- // unmapedParamList: Parameters that should be used in the template but could not be found
- // Check at registration and submission
- Set<String> unmapedParamList = new HashSet<String>();
- for (ExperimentTemplateParamSpec tplParam : tplspec.getExperimentTemplateParamSpec()) {
- if (paramMap.get(tplParam.getName()) == null) {
- // use default value
- if (!Boolean.parseBoolean(tplParam.getRequired())) {
- paramMap.put(tplParam.getName(), tplParam.getValue());
- } else {
- unmapedParamList.add(tplParam.getName());
- }
- }
- }
-
- // unusedParamList: Parameters not used by the template
- // Check at registration
- Set<String> unusedParamList = new HashSet<String>();
- for (String s : paramMap.keySet()) {
- unusedParamList.add(s);
- }
-
- // resourceMap needs special handling
- for (Map.Entry<String, ExperimentTaskSpec> entrySet : tplspec.getExperimentSpec().getSpec().entrySet()) {
- String cpu = paramMap.get(String.format("spec.%s.resourceMap.cpu", entrySet.getKey()));
- String memory = paramMap.get(String.format("spec.%s.resourceMap.memory", entrySet.getKey()));
- flattenJson.put(String.format("spec.%s.resources", entrySet.getKey()),
- String.format("cpu=%s,memory=%s", cpu, memory));
- }
-
- // Mapping the {{...}} param
- Pattern pattern = Pattern.compile("\\{\\{(.*?)\\}\\}");
- for (Map.Entry<String, Object> entry : flattenJson.entrySet()) {
- boolean isMatch = false;
- if (entry.getValue() instanceof String) {
- String value = (String) entry.getValue();
- Matcher matcher = pattern.matcher(value);
- StringBuffer sb = new StringBuffer();
-
- while (matcher.find()) {
- String name = matcher.group(1);
- String key = entry.getKey() + ":" + name;
-
- // match path+placeholder ("meta.cmd:parametername")
- if (paramMap.get(key) != null) {
- isMatch = true;
- matcher.appendReplacement(sb, paramMap.get(key));
- unusedParamList.remove(key);
- unmapedParamList.remove(key);
- }
- // match placeholder ("parametername")
- else if (paramMap.get(name) != null) {
- isMatch = true;
- matcher.appendReplacement(sb, paramMap.get(name));
- unusedParamList.remove(name);
- unmapedParamList.remove(name);
- } else {
- unmapedParamList.add(key);
- }
- }
- if (isMatch) {
- matcher.appendTail(sb);
- flattenJson.put(entry.getKey(), sb.toString());
- }
- }
- // match path ("meta.cmd")
- if (!isMatch) {
- String key = entry.getKey();
- if (paramMap.get(key) != null) {
- flattenJson.put(key, paramMap.get(key));
- unusedParamList.remove(key);
- }
- }
- }
-
- if (unusedParamList.size() > 0) {
- throw new SubmarineRuntimeException(Status.BAD_REQUEST.getStatusCode(),
- "Parameters contains unused key: " + unusedParamList.toString());
- }
-
- if (unmapedParamList.size() > 0) {
- throw new SubmarineRuntimeException(Status.BAD_REQUEST.getStatusCode(),
- "Template contains unmapped value: " + unmapedParamList.toString());
- }
-
- String json = flattenJson.toString();
- Log.info("flattenJson " + json);
-
- String nestedJson = JsonUnflattener.unflatten(json);
- Log.info("nestedJson " + nestedJson);
-
- ExperimentSpec returnExperimentSpec = null;
- try {
- returnExperimentSpec = gson.fromJson(nestedJson, ExperimentSpec.class);
- Log.info("ExperimentSpec " + returnExperimentSpec.toString());
-
- } catch (Exception e) {
- throw new SubmarineRuntimeException(Status.BAD_REQUEST.getStatusCode(),
- "Template mapping fail: " + e.getMessage() + nestedJson);
- }
- return returnExperimentSpec;
- }
-}
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.submarine.server.experimenttemplate;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import javax.ws.rs.core.Response.Status;
+
+import org.apache.ibatis.session.SqlSession;
+import org.apache.submarine.commons.utils.exception.SubmarineRuntimeException;
+import org.apache.submarine.server.SubmarineServer;
+import org.apache.submarine.server.api.experiment.Experiment;
+import org.apache.submarine.server.api.experiment.ExperimentId;
+import org.apache.submarine.server.api.experimenttemplate.ExperimentTemplate;
+import org.apache.submarine.server.api.experimenttemplate.ExperimentTemplateId;
+import org.apache.submarine.server.api.experimenttemplate.ExperimentTemplateSubmit;
+import org.apache.submarine.server.api.spec.ExperimentSpec;
+import org.apache.submarine.server.api.spec.ExperimentTaskSpec;
+import org.apache.submarine.server.api.spec.ExperimentTemplateParamSpec;
+import org.apache.submarine.server.api.spec.ExperimentTemplateSpec;
+import org.apache.submarine.server.database.utils.MyBatisUtil;
+import org.apache.submarine.server.experiment.ExperimentManager;
+import org.apache.submarine.server.experimenttemplate.database.entity.ExperimentTemplateEntity;
+import org.apache.submarine.server.experimenttemplate.database.mappers.ExperimentTemplateMapper;
+import org.apache.submarine.server.gson.ExperimentIdDeserializer;
+import org.apache.submarine.server.gson.ExperimentIdSerializer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.esotericsoftware.minlog.Log;
+import com.github.wnameless.json.flattener.JsonFlattener;
+import com.github.wnameless.json.unflattener.JsonUnflattener;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+
+/**
+ * ExperimentTemplate Management.
+ */
+public class ExperimentTemplateManager {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ExperimentTemplateManager.class);
+ private static volatile ExperimentTemplateManager manager;
+ private final AtomicInteger experimentTemplateIdCounter = new AtomicInteger(0);
+
+ private static final GsonBuilder gsonBuilder = new GsonBuilder()
+ .registerTypeAdapter(ExperimentId.class, new ExperimentIdSerializer())
+ .registerTypeAdapter(ExperimentId.class, new ExperimentIdDeserializer());
+ private static final Gson gson = gsonBuilder.setDateFormat("yyyy-MM-dd HH:mm:ss").create();
+
+
+
+ /**
+ * ExperimentTemplate Cache.
+ */
+ private final ConcurrentMap<String, ExperimentTemplate> cachedExperimentTemplates =
+ new ConcurrentHashMap<>();
+
+ /**
+ * Get the singleton instance.
+ *
+ * @return object
+ */
+ public static ExperimentTemplateManager getInstance() {
+ if (manager == null) {
+ synchronized (ExperimentTemplateManager.class) {
+ if (manager == null) {
+ manager = new ExperimentTemplateManager();
+ }
+ }
+ }
+ return manager;
+ }
+
+ private ExperimentTemplateManager() {
+
+ }
+
+ /**
+ * Create ExperimentTemplate.
+ *
+ * @param spec experimentTemplate spec
+ * @return ExperimentTemplate experimentTemplate
+ * @throws SubmarineRuntimeException the service error
+ */
+ public ExperimentTemplate createExperimentTemplate(ExperimentTemplateSpec spec)
+ throws SubmarineRuntimeException {
+ checkSpec(spec);
+ LOG.info("Create ExperimentTemplate using spec: " + spec.toString());
+ return createOrUpdateExperimentTemplate(spec, "c");
+ }
+
+ /**
+ * Update experimentTemplate.
+ *
+ * @param name Name of the experimentTemplate
+ * @param spec experimentTemplate spec
+ * @return ExperimentTemplate experimentTemplate
+ * @throws SubmarineRuntimeException the service error
+ */
+ public ExperimentTemplate updateExperimentTemplate(String name, ExperimentTemplateSpec spec)
+ throws SubmarineRuntimeException {
+ ExperimentTemplate tpl = getExperimentTemplateDetails(name);
+ if (tpl == null) {
+ throw new SubmarineRuntimeException(Status.NOT_FOUND.getStatusCode(), "ExperimentTemplate not found.");
+ }
+ checkSpec(spec);
+ LOG.info("Update ExperimentTemplate using spec: " + spec.toString());
+ return createOrUpdateExperimentTemplate(spec, "u");
+ }
+
+ private ExperimentTemplate createOrUpdateExperimentTemplate(ExperimentTemplateSpec spec, String operation) {
+
+ spec = addResourcesParameter(spec);
+
+ ExperimentTemplateEntity entity = new ExperimentTemplateEntity();
+ String experimentTemplateId = generateExperimentTemplateId().toString();
+ entity.setId(experimentTemplateId);
+ entity.setExperimentTemplateName(spec.getName());
+ entity.setExperimentTemplateSpec(gsonBuilder.disableHtmlEscaping().create().toJson(spec));
+
+
+ parameterMapping(spec);
+
+ try (SqlSession sqlSession = MyBatisUtil.getSqlSession()) {
+ ExperimentTemplateMapper experimentTemplateMapper =
+ sqlSession.getMapper(ExperimentTemplateMapper.class);
+
+ if (operation.equals("c")) {
+ experimentTemplateMapper.insert(entity);
+ } else {
+ experimentTemplateMapper.update(entity);
+ }
+ sqlSession.commit();
+
+ } catch (Exception e) {
+ LOG.error(e.getMessage(), e);
+ throw new SubmarineRuntimeException(Status.BAD_REQUEST.getStatusCode(),
+ "Unable to insert or update the experimentTemplate spec: " + e.getMessage());
+ }
+
+ ExperimentTemplate experimentTemplate;
+ try {
+ experimentTemplate = new ExperimentTemplate();
+ experimentTemplate.setExperimentTemplateId(ExperimentTemplateId.fromString(experimentTemplateId));
+ experimentTemplate.setExperimentTemplateSpec(spec);
+
+ } catch (Exception e) {
+ LOG.error(e.getMessage(), e);
+ throw new SubmarineRuntimeException(Status.BAD_REQUEST.getStatusCode(),
+ "Unable to parse the experimentTemplate spec: " + e.getMessage());
+ }
+ // Update cache
+ cachedExperimentTemplates.put(spec.getName(), experimentTemplate);
+
+ return experimentTemplate;
+ }
+
+ private ExperimentTemplateId generateExperimentTemplateId() {
+ return ExperimentTemplateId.newInstance(SubmarineServer.getServerTimeStamp(),
+ experimentTemplateIdCounter.incrementAndGet());
+ }
+
+ /**
+ * Delete experimentTemplate.
+ *
+ * @param name Name of the experimentTemplate
+ * @return ExperimentTemplate experimentTemplate
+ * @throws SubmarineRuntimeException the service error
+ */
+ public ExperimentTemplate deleteExperimentTemplate(String name) throws SubmarineRuntimeException {
+ ExperimentTemplate tpl = getExperimentTemplateDetails(name);
+ if (tpl == null) {
+ throw new SubmarineRuntimeException(Status.NOT_FOUND.getStatusCode(), "ExperimentTemplate not found.");
+ }
+
+ LOG.info("Delete ExperimentTemplate for " + name);
+ try (SqlSession sqlSession = MyBatisUtil.getSqlSession()) {
+ ExperimentTemplateMapper experimentTemplateMapper =
+ sqlSession.getMapper(ExperimentTemplateMapper.class);
+
+ experimentTemplateMapper.delete(name);
+ sqlSession.commit();
+
+ // Invalidate cache
+ cachedExperimentTemplates.remove(name);
+ return tpl;
+ } catch (Exception e) {
+ LOG.error(e.getMessage(), e);
+ throw new SubmarineRuntimeException(Status.BAD_REQUEST.getStatusCode(),
+ "Unable to delete the experimentTemplate.");
+ }
+ }
+
+ /**
+ * Get ExperimentTemplate.
+ *
+ * @param name Name of the experimentTemplate
+ * @return ExperimentTemplate experimentTemplate
+ * @throws SubmarineRuntimeException the service error
+ */
+ public ExperimentTemplate getExperimentTemplate(String name) throws SubmarineRuntimeException {
+ ExperimentTemplate experimentTemplate = getExperimentTemplateDetails(name);
+ if (experimentTemplate == null) {
+ throw new SubmarineRuntimeException(Status.NOT_FOUND.getStatusCode(), "ExperimentTemplate not found.");
+ }
+ return experimentTemplate;
+ }
+
+ /**
+ * List experimentTemplates.
+ *
+ * @param status experimentTemplate status, if null will return all status
+ * @return experimentTemplate list
+ * @throws SubmarineRuntimeException the service error
+ */
+ public List<ExperimentTemplate> listExperimentTemplates(String status) throws SubmarineRuntimeException {
+ List<ExperimentTemplate> tpls = new ArrayList<>(cachedExperimentTemplates.values());
+
+ // Is it available in cache?
+ if (tpls.size() != 0) {
+ return tpls;
+ }
+ try (SqlSession sqlSession = MyBatisUtil.getSqlSession()) {
+ ExperimentTemplateMapper experimentTemplateMapper =
+ sqlSession.getMapper(ExperimentTemplateMapper.class);
+
+ List<ExperimentTemplateEntity> experimentTemplateEntities = experimentTemplateMapper.selectByKey(null);
+ for (ExperimentTemplateEntity experimentTemplateEntity : experimentTemplateEntities) {
+ if (experimentTemplateEntity != null) {
+ ExperimentTemplate tpl = new ExperimentTemplate();
+
+ tpl.setExperimentTemplateSpec(
+ gson.fromJson(experimentTemplateEntity.getExperimentTemplateSpec(),
+ ExperimentTemplateSpec.class));
+ tpls.add(tpl);
+ cachedExperimentTemplates.put(tpl.getExperimentTemplateSpec().getName(), tpl);
+ }
+ }
+ } catch (Exception e) {
+ LOG.error(e.getMessage(), e);
+ throw new SubmarineRuntimeException(Status.BAD_REQUEST.getStatusCode(),
+ "Unable to get the experimentTemplate details.");
+ }
+ return tpls;
+ }
+
+ private void checkSpec(ExperimentTemplateSpec spec) throws SubmarineRuntimeException {
+ if (spec == null) {
+ throw new SubmarineRuntimeException(Status.BAD_REQUEST.getStatusCode(),
+ "Invalid experimentTemplate spec.");
+ }
+ }
+
+ private ExperimentTemplate getExperimentTemplateDetails(String name) throws SubmarineRuntimeException {
+
+ // Is it available in cache?
+ ExperimentTemplate tpl = cachedExperimentTemplates.get(name);
+ if (tpl != null) {
+ return tpl;
+ }
+ ExperimentTemplateEntity experimentTemplateEntity;
+ try (SqlSession sqlSession = MyBatisUtil.getSqlSession()) {
+ ExperimentTemplateMapper experimentTemplateMapper =
+ sqlSession.getMapper(ExperimentTemplateMapper.class);
+
+ experimentTemplateEntity = experimentTemplateMapper.select(name);
+
+ if (experimentTemplateEntity != null) {
+ tpl = new ExperimentTemplate();
+ tpl.setExperimentTemplateSpec(
+ gson.fromJson(experimentTemplateEntity.getExperimentTemplateSpec(),
+ ExperimentTemplateSpec.class));
+ }
+ } catch (Exception e) {
+ LOG.error(e.getMessage(), e);
+ throw new SubmarineRuntimeException(Status.BAD_REQUEST.getStatusCode(),
+ "Unable to get the experimentTemplate details.");
+ }
+ return tpl;
+ }
+
+
+ /**
+ * Create ExperimentTemplate.
+ *
+ * @param submittedParam experimentTemplate spec
+ * @return Experiment experiment
+ * @throws SubmarineRuntimeException the service error
+ */
+ public Experiment submitExperimentTemplate(ExperimentTemplateSubmit submittedParam)
+ throws SubmarineRuntimeException {
+
+ if (submittedParam == null) {
+ throw new SubmarineRuntimeException(Status.BAD_REQUEST.getStatusCode(),
+ "Invalid ExperimentTemplateSubmit spec.");
+ }
+
+ ExperimentTemplate experimentTemplate = getExperimentTemplate(submittedParam.getName());
+ Map<String, String> params = submittedParam.getParams();
+
+
+ for (ExperimentTemplateParamSpec paramSpec:
+ experimentTemplate.getExperimentTemplateSpec().getExperimentTemplateParamSpec()) {
+
+ String value = params.get(paramSpec.getName());
+ if (value != null) {
+ paramSpec.setValue(value);
+ }
+ }
+ ExperimentTemplateSpec spec = experimentTemplate.getExperimentTemplateSpec();
+
+ ExperimentSpec experimentSpec = parameterMapping(spec, params);
+
+ return ExperimentManager.getInstance().createExperiment(experimentSpec);
+ }
+
+
+ private ExperimentTemplateSpec addResourcesParameter(ExperimentTemplateSpec tplSpec) {
+
+ for (Map.Entry<String, ExperimentTaskSpec> entrySet : tplSpec.getExperimentSpec().getSpec().entrySet()) {
+
+ ExperimentTaskSpec taskSpec = entrySet.getValue();
+ // parse resourceMap
+ taskSpec.setResources(taskSpec.getResources());
+
+ ExperimentTemplateParamSpec parm1 = new ExperimentTemplateParamSpec();
+ parm1.setName(String.format("spec.%s.replicas", entrySet.getKey()));
+ parm1.setValue(taskSpec.getReplicas() == null ? "1" : taskSpec.getReplicas().toString());
+ parm1.setRequired("false");
+ parm1.setDescription("");
+ tplSpec.getExperimentTemplateParamSpec().add(parm1);
+
+ ExperimentTemplateParamSpec parm2 = new ExperimentTemplateParamSpec();
+ parm2.setName(String.format("spec.%s.resourceMap.cpu", entrySet.getKey()));
+ parm2.setValue(taskSpec.getCpu() == null ? "1" : taskSpec.getCpu());
+ parm2.setRequired("false");
+ parm2.setDescription("");
+ tplSpec.getExperimentTemplateParamSpec().add(parm2);
+
+ ExperimentTemplateParamSpec parm3 = new ExperimentTemplateParamSpec();
+ parm3.setName(String.format("spec.%s.resourceMap.memory", entrySet.getKey()));
+ parm3.setValue(taskSpec.getMemory() == null ? "1" : taskSpec.getMemory().toString());
+ parm3.setRequired("false");
+ parm3.setDescription("");
+ tplSpec.getExperimentTemplateParamSpec().add(parm3);
+
+ }
+ return tplSpec;
+ }
+
+
+ // use template default value to mapping
+ private ExperimentSpec parameterMapping(ExperimentTemplateSpec tplSpec) {
+
+ Map<String, String> paramMap = new HashMap<String, String>();
+ for (ExperimentTemplateParamSpec parm : tplSpec.getExperimentTemplateParamSpec()) {
+ if (parm.getValue() != null) {
+ paramMap.put(parm.getName(), parm.getValue());
+ } else {
+ paramMap.put(parm.getName(), "");
+ }
+ }
+ return parameterMapping(tplSpec, paramMap);
+ }
+
+ private ExperimentSpec parameterMapping(ExperimentTemplateSpec tplspec, Map<String, String> paramMap) {
+
+ String spec = gson.toJson(tplspec.getExperimentSpec());
+ Map<String, Object> flattenJson = JsonFlattener.flattenAsMap(spec);
+
+ Log.info(flattenJson.toString());
+ // illegalParamList: The parameters not in template parameters should not be used
+ // Check at submission
+ Map<String, ExperimentTemplateParamSpec> tplparamMap = new HashMap<String, ExperimentTemplateParamSpec>();
+ for (ExperimentTemplateParamSpec tplParam : tplspec.getExperimentTemplateParamSpec()) {
+ tplparamMap.put(tplParam.getName(), tplParam);
+ }
+ Set<String> illegalParamList = new HashSet<String>();
+ for (String key : paramMap.keySet()) {
+ if (tplparamMap.get(key) == null) {
+ illegalParamList.add(key);
+ }
+ }
+
+ if (illegalParamList.size() > 0) {
+ throw new SubmarineRuntimeException(Status.BAD_REQUEST.getStatusCode(),
+ "Parameters contains illegal key: " + illegalParamList.toString());
+ }
+
+ // unmapedParamList: Parameters that should be used in the template but could not be found
+ // Check at registration and submission
+ Set<String> unmapedParamList = new HashSet<String>();
+ for (ExperimentTemplateParamSpec tplParam : tplspec.getExperimentTemplateParamSpec()) {
+ if (paramMap.get(tplParam.getName()) == null) {
+ // use default value
+ if (!Boolean.parseBoolean(tplParam.getRequired())) {
+ paramMap.put(tplParam.getName(), tplParam.getValue());
+ } else {
+ unmapedParamList.add(tplParam.getName());
+ }
+ }
+ }
+
+ // unusedParamList: Parameters not used by the template
+ // Check at registration
+ Set<String> unusedParamList = new HashSet<String>();
+ for (String s : paramMap.keySet()) {
+ unusedParamList.add(s);
+ }
+
+ // resourceMap needs special handling
+ for (Map.Entry<String, ExperimentTaskSpec> entrySet : tplspec.getExperimentSpec().getSpec().entrySet()) {
+ String cpu = paramMap.get(String.format("spec.%s.resourceMap.cpu", entrySet.getKey()));
+ String memory = paramMap.get(String.format("spec.%s.resourceMap.memory", entrySet.getKey()));
+ flattenJson.put(String.format("spec.%s.resources", entrySet.getKey()),
+ String.format("cpu=%s,memory=%s", cpu, memory));
+ }
+
+ // Mapping the {{...}} param
+ Pattern pattern = Pattern.compile("\\{\\{(.*?)\\}\\}");
+ for (Map.Entry<String, Object> entry : flattenJson.entrySet()) {
+ boolean isMatch = false;
+ if (entry.getValue() instanceof String) {
+ String value = (String) entry.getValue();
+ Matcher matcher = pattern.matcher(value);
+ StringBuffer sb = new StringBuffer();
+
+ while (matcher.find()) {
+ String name = matcher.group(1);
+ String key = entry.getKey() + ":" + name;
+
+ // match path+placeholder ("meta.cmd:parametername")
+ if (paramMap.get(key) != null) {
+ isMatch = true;
+ matcher.appendReplacement(sb, paramMap.get(key));
+ unusedParamList.remove(key);
+ unmapedParamList.remove(key);
+ }
+ // match placeholder ("parametername")
+ else if (paramMap.get(name) != null) {
+ isMatch = true;
+ matcher.appendReplacement(sb, paramMap.get(name));
+ unusedParamList.remove(name);
+ unmapedParamList.remove(name);
+ } else {
+ unmapedParamList.add(key);
+ }
+ }
+ if (isMatch) {
+ matcher.appendTail(sb);
+ flattenJson.put(entry.getKey(), sb.toString());
+ }
+ }
+ // match path ("meta.cmd")
+ if (!isMatch) {
+ String key = entry.getKey();
+ if (paramMap.get(key) != null) {
+ flattenJson.put(key, paramMap.get(key));
+ unusedParamList.remove(key);
+ }
+ }
+ }
+
+ if (unusedParamList.size() > 0) {
+ throw new SubmarineRuntimeException(Status.BAD_REQUEST.getStatusCode(),
+ "Parameters contains unused key: " + unusedParamList.toString());
+ }
+
+ if (unmapedParamList.size() > 0) {
+ throw new SubmarineRuntimeException(Status.BAD_REQUEST.getStatusCode(),
+ "Template contains unmapped value: " + unmapedParamList.toString());
+ }
+
+ String json = flattenJson.toString();
+ Log.info("flattenJson " + json);
+
+ String nestedJson = JsonUnflattener.unflatten(json);
+ Log.info("nestedJson " + nestedJson);
+
+ ExperimentSpec returnExperimentSpec = null;
+ try {
+ returnExperimentSpec = gson.fromJson(nestedJson, ExperimentSpec.class);
+ Log.info("ExperimentSpec " + returnExperimentSpec.toString());
+
+ } catch (Exception e) {
+ throw new SubmarineRuntimeException(Status.BAD_REQUEST.getStatusCode(),
+ "Template mapping fail: " + e.getMessage() + nestedJson);
+ }
+ return returnExperimentSpec;
+ }
+}
diff --git a/submarine-server/server-core/src/main/java/org/apache/submarine/server/experimenttemplate/database/mappers/ExperimentTemplateMapper.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/experimenttemplate/database/mappers/ExperimentTemplateMapper.java
index c0646ce..553e05c 100644
--- a/submarine-server/server-core/src/main/java/org/apache/submarine/server/experimenttemplate/database/mappers/ExperimentTemplateMapper.java
+++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/experimenttemplate/database/mappers/ExperimentTemplateMapper.java
@@ -1,39 +1,39 @@
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.submarine.server.experimenttemplate.database.mappers;
-
-import java.util.List;
-
-import org.apache.submarine.server.experimenttemplate.database.entity.ExperimentTemplateEntity;
-
-
-public interface ExperimentTemplateMapper {
-
- ExperimentTemplateEntity select(String ExperimentTemplateName);
-
- int insert(ExperimentTemplateEntity ExperimentTemplate);
-
- int update(ExperimentTemplateEntity ExperimentTemplate);
-
- int delete(String ExperimentTemplateName);
-
- List<ExperimentTemplateEntity> selectByKey(ExperimentTemplateEntity ExperimentTemplate);
-
-}
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.submarine.server.experimenttemplate.database.mappers;
+
+import java.util.List;
+
+import org.apache.submarine.server.experimenttemplate.database.entity.ExperimentTemplateEntity;
+
+
+public interface ExperimentTemplateMapper {
+
+ ExperimentTemplateEntity select(String experimentTemplateName);
+
+ int insert(ExperimentTemplateEntity experimentTemplate);
+
+ int update(ExperimentTemplateEntity experimentTemplate);
+
+ int delete(String experimentTemplateName);
+
+ List<ExperimentTemplateEntity> selectByKey(ExperimentTemplateEntity experimentTemplate);
+
+}
diff --git a/submarine-server/server-core/src/main/java/org/apache/submarine/server/notebook/NotebookManager.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/notebook/NotebookManager.java
index a39320c..db8f844 100644
--- a/submarine-server/server-core/src/main/java/org/apache/submarine/server/notebook/NotebookManager.java
+++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/notebook/NotebookManager.java
@@ -56,7 +56,7 @@ public class NotebookManager {
private final AtomicInteger notebookCounter = new AtomicInteger(0);
/**
- * Get the singleton instance
+ * Get the singleton instance.
*
* @return object
*/
@@ -72,7 +72,7 @@ public class NotebookManager {
}
/**
- * Create a notebook instance
+ * Create a notebook instance.
*
* @param spec NotebookSpec
* @return object
@@ -109,7 +109,7 @@ public class NotebookManager {
}
/**
- * List notebook instances
+ * List notebook instances.
*
* @param namespace namespace, if null will return all notebooks
* @return list
@@ -129,7 +129,7 @@ public class NotebookManager {
}
/**
- * Get a list of notebook with user id
+ * Get a list of notebook with user id.
*
* @param id user id
* @return a list of notebook
@@ -146,14 +146,13 @@ public class NotebookManager {
} catch (SubmarineRuntimeException e) {
LOG.warn("Submitter can not find notebook: {}, will delete it", nb.getNotebookId());
notebookService.delete(nb.getNotebookId().toString());
- continue;
}
}
return notebookList;
}
/**
- * Get a notebook instance
+ * Get a notebook instance.
*
* @param id notebook id
* @return object
@@ -175,7 +174,7 @@ public class NotebookManager {
}
/**
- * Delete the notebook instance
+ * Delete the notebook instance.
*
* @param id notebook id
* @return object
@@ -190,7 +189,7 @@ public class NotebookManager {
}
/**
- * Generate a unique notebook id
+ * Generate a unique notebook id.
*
* @return notebook id
*/
@@ -200,7 +199,7 @@ public class NotebookManager {
}
/**
- * Check if notebook spec is valid
+ * Check if notebook spec is valid.
*
* @param spec notebook spec
*/
diff --git a/submarine-server/server-core/src/main/java/org/apache/submarine/server/notebook/database/service/NotebookService.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/notebook/database/service/NotebookService.java
index 704cb2c..c19275a 100644
--- a/submarine-server/server-core/src/main/java/org/apache/submarine/server/notebook/database/service/NotebookService.java
+++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/notebook/database/service/NotebookService.java
@@ -120,9 +120,9 @@ public class NotebookService {
}
/**
- * Create a NotebookEntity instance from experiment
+ * Create a NotebookEntity instance from experiment.
*
- * @param notebook
+ * @param notebook the Notebook used to create a NoteBookEntity
* @return NotebookEntity
*/
private NotebookEntity buildEntityFromNotebook(Notebook notebook) {
@@ -138,9 +138,9 @@ public class NotebookService {
}
/**
- * Create a new notebook instance from entity
+ * Create a new notebook instance from entity.
*
- * @param entity
+ * @param entity the NotebookEntity used to create a Notebook
* @return Notebook
*/
private Notebook buildNotebookFromEntity(NotebookEntity entity) {
diff --git a/submarine-server/server-core/src/main/java/org/apache/submarine/server/response/DictAnnotation.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/response/DictAnnotation.java
index 2a982e4..ef0cc0b 100644
--- a/submarine-server/server-core/src/main/java/org/apache/submarine/server/response/DictAnnotation.java
+++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/response/DictAnnotation.java
@@ -36,7 +36,6 @@ import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
-import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -62,8 +61,8 @@ public class DictAnnotation {
BeanGenerator generator = new BeanGenerator();
Set keySet = mapProperty.keySet();
- for (Iterator<String> it = keySet.iterator(); it.hasNext(); ) {
- String key = it.next();
+ for (Object elem : keySet) {
+ String key = (String) elem;
generator.addProperty(key, (Class) mapProperty.get(key));
}
return generator.create();
@@ -100,8 +99,7 @@ public class DictAnnotation {
BeanInfo beanInfo = Introspector.getBeanInfo(objectClass);
PropertyDescriptor[] propertyDescriptors = beanInfo.getPropertyDescriptors();
// Get data that already exists in the object
- for (int i = 0; i < propertyDescriptors.length; i++) {
- PropertyDescriptor descriptor = propertyDescriptors[i];
+ for (PropertyDescriptor descriptor : propertyDescriptors) {
String propertyName = descriptor.getName();
if (!propertyName.equals("class")) {
Method readMethod = descriptor.getReadMethod();
@@ -131,13 +129,11 @@ public class DictAnnotation {
// Map to entity object
DictAnnotation bean = new DictAnnotation(mapFieldAndType);
Set<String> keys = mapFieldAndType.keySet();
- for (Iterator<String> it = keys.iterator(); it.hasNext(); ) {
- String key = it.next();
+ for (String key : keys) {
bean.setValue(key, mapFieldValues.get(key));
}
- Object newObj = bean.getObject();
- return newObj;
+ return bean.getObject();
}
public static boolean parseDictAnnotation(Object result) throws Exception {
diff --git a/submarine-server/server-core/src/main/java/org/apache/submarine/server/rest/RestConstants.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/rest/RestConstants.java
index c5434c9..0d6d3fc 100644
--- a/submarine-server/server-core/src/main/java/org/apache/submarine/server/rest/RestConstants.java
+++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/rest/RestConstants.java
@@ -48,30 +48,30 @@ public class RestConstants {
public static final String LOGS = "logs";
/**
- * Environment
+ * Environment.
*/
public static final String ENVIRONMENT = "environment";
public static final String ENVIRONMENT_ID = "id";
/**
- * Experimect template
+ * Experiment template.
*/
public static final String EXPERIMENT_TEMPLATES = "template";
-
+
public static final String EXPERIMENT_TEMPLATE_ID = "id";
public static final String EXPERIMENT_TEMPLATE_SUBMIT = "submit";
/**
- * Notebook
+ * Notebook.
*/
public static final String NOTEBOOK = "notebook";
public static final String NOTEBOOK_ID = "id";
/**
- * Tensorboard
+ * Tensorboard.
*/
public static final String LOG_DIR_KEY = "SUBMARINE_TENSORBOARD_LOG_DIR";
public static final String LOG_DIR_VALUE = "/logs/mylog";
diff --git a/submarine-server/server-core/src/main/java/org/apache/submarine/server/utils/GitHttpRequest.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/utils/GitHttpRequest.java
index 50d301f..8a35106 100644
--- a/submarine-server/server-core/src/main/java/org/apache/submarine/server/utils/GitHttpRequest.java
+++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/utils/GitHttpRequest.java
@@ -29,6 +29,7 @@ import java.net.HttpURLConnection;
import java.net.ProtocolException;
import java.net.URL;
import java.net.URLConnection;
+import java.nio.charset.StandardCharsets;
import java.util.Date;
import java.util.Map;
@@ -36,15 +37,15 @@ public class GitHttpRequest {
private static final Logger LOG = LoggerFactory.getLogger(GitHttpRequest.class);
/**
- * Sends an HTTP request to the specified URL
- * @param url
- * @param parameter
- * @param method
- * @return
- * @throws Exception
+ * Sends an HTTP request to the specified URL.
+ * @param url the URL you want to send request to
+ * @param headParams a map of HTTP header parameters
+ * @param content the content of request
+ * @param method the HTTP method
+ * @return String
*/
public static String sendHttpRequest(String url, Map<String, String> headParams,
- byte[] parameter, String method) {
+ byte[] content, String method) {
// Create a URLConnection
URLConnection urlConnection = null;
try {
@@ -52,6 +53,7 @@ public class GitHttpRequest {
} catch (IOException e) {
LOG.error(e.getMessage(), e);
}
+
// Defines StringBuilder to facilitate string concatenation later
// when reading web pages and returning byte stream information
StringBuilder stringBuilder = new StringBuilder();
@@ -60,19 +62,22 @@ public class GitHttpRequest {
HttpURLConnection httpURLConnection = (HttpURLConnection) urlConnection;
httpURLConnection.setDoInput(true);
httpURLConnection.setDoOutput(true);
- // Set the request, which could be delete put post get
+
+ // Set the request, which could be deleted put post get
try {
httpURLConnection.setRequestMethod(method);
} catch (ProtocolException e) {
LOG.error(e.getMessage(), e);
}
- if (parameter != null) {
+
+ if (content != null) {
// Sets the length of the content
- httpURLConnection.setRequestProperty("Content-Length", String.valueOf(parameter.length));
+ httpURLConnection.setRequestProperty("Content-Length", String.valueOf(content.length));
}
+
// Set encoding format
httpURLConnection.setRequestProperty("Content-Type", "application/json;charset=utf-8");
- // Sets the format of the receive return parameter
+ // Sets the format of the received return parameter
httpURLConnection.setRequestProperty("accept", "application/json");
httpURLConnection.setRequestProperty("connection", "Keep-Alive");
httpURLConnection.setRequestProperty("user-agent",
@@ -85,19 +90,21 @@ public class GitHttpRequest {
httpURLConnection.setRequestProperty(key, headParams.get(key));
}
}
+
httpURLConnection.setUseCaches(false);
- if (parameter != null) {
+ if (content != null) {
try (BufferedOutputStream outputStream =
new BufferedOutputStream(httpURLConnection.getOutputStream())) {
- outputStream.write(parameter);
+ outputStream.write(content);
outputStream.flush();
} catch (IOException e) {
LOG.error(e.getMessage(), e);
}
}
+
try (InputStreamReader inputStreamReader =
- new InputStreamReader(httpURLConnection.getInputStream(), "utf-8");
+ new InputStreamReader(httpURLConnection.getInputStream(), StandardCharsets.UTF_8);
BufferedReader bufferedReader = new BufferedReader(inputStreamReader)) {
String line;
while ((line = bufferedReader.readLine()) != null) {
@@ -107,7 +114,7 @@ public class GitHttpRequest {
LOG.error(e.getMessage(), e);
}
- LOG.info("result:{}", stringBuilder.toString());
+ LOG.info("result:{}", stringBuilder);
return stringBuilder.toString();
}
}
diff --git a/submarine-server/server-core/src/main/java/org/apache/submarine/server/utils/GitUtils.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/utils/GitUtils.java
index 7c0dca0..76990a1 100644
--- a/submarine-server/server-core/src/main/java/org/apache/submarine/server/utils/GitUtils.java
+++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/utils/GitUtils.java
@@ -41,12 +41,11 @@ public class GitUtils {
private static final Logger LOG = LoggerFactory.getLogger(GitUtils.class);
/**
- * To execute clone command
- * @param remotePath
- * @param localPath
- * @param token
- * @param branch
- * @throws GitAPIException
+ * To execute clone command.
+ * @param remotePath the URI of the remote repository
+ * @param localPath the path of the local repository
+ * @param token the GitHub access token
+ * @param branch the git branch to use
*/
public void clone(String remotePath, String localPath, String token, String branch) {
// Clone the code base command
@@ -54,34 +53,30 @@ public class GitUtils {
CredentialsProvider credentialsProvider =
new UsernamePasswordCredentialsProvider("PRIVATE-TOKEN", token);
- Git git = null;
- try {
- git = Git.cloneRepository().setURI(remotePath) // Set remote URI
- .setBranch(branch) // Set the branch down from clone
- .setDirectory(new File(localPath)) // Set the download path
- .setCredentialsProvider(credentialsProvider) // Set permission validation
- .call();
+ try (Git git = Git.cloneRepository().setURI(remotePath) // Set remote URI
+ .setBranch(branch) // Set the branch down from clone
+ .setDirectory(new File(localPath)) // Set the download path
+ .setCredentialsProvider(credentialsProvider) // Set permission validation
+ .call();) {
+
+ LOG.info("git.tag(): {}", git.tag());
} catch (GitAPIException e) {
LOG.error(e.getMessage(), e);
- } finally {
- if (git != null) {
- git.close();
- }
}
- LOG.info("git.tag(): {}", git.tag());
}
/**
- * To execute add command
- * @param localPath
- * @param fileName
- * relative path
- * @throws IOException
- * @throws GitAPIException
+ * To execute add command.
+ * @param localPath the path of the local repository
+ * @param fileName the file name you want to add to the git stage area (relative path)
+ * @return DirCache
*/
public DirCache add(String localPath, String fileName) {
// Git repository address
- File myfile = new File(localPath + fileName);
+
+ // File myfile = new File(localPath + fileName);
+ File myfile = new File(localPath, fileName);
+
if (!myfile.exists()) {
myfile.getParentFile().mkdirs();
try {
@@ -90,25 +85,23 @@ public class GitUtils {
LOG.error(e.getMessage(), e);
}
}
+
DirCache dirCache = null;
try (Git git = Git.open(new File(localPath))) {
// Add files
dirCache = git.add().addFilepattern(fileName.substring(1)).call();
- } catch (IOException e) {
- LOG.error(e.getMessage(), e);
- } catch (GitAPIException e) {
+ } catch (IOException | GitAPIException e) {
LOG.error(e.getMessage(), e);
}
+
return dirCache;
}
/**
- * To execute rm command
- * @param localPath
- * @param fileName
- * relative path
- * @throws IOException
- * @throws GitAPIException
+ * To execute rm command.
+ * @param localPath the path of the local repository
+ * @param fileName the file name you want to remove from the git repo (relative path)
+ * @return DirCache
*/
public DirCache rm(String localPath, String fileName) {
DirCache dirCache = null;
@@ -116,41 +109,34 @@ public class GitUtils {
try (Git git = Git.open(new File(localPath))) {
// rm files
dirCache = git.rm().addFilepattern(fileName.substring(1)).call();
- } catch (IOException e) {
- LOG.error(e.getMessage(), e);
- } catch (GitAPIException e) {
+ } catch (IOException | GitAPIException e) {
LOG.error(e.getMessage(), e);
}
+
return dirCache;
}
/**
- * To execute reset command
- * @param localPath
- * @param fileName
- * relative path
- * @throws IOException
- * @throws GitAPIException
+ * To execute reset command.
+ * @param localPath the path of the local repository
+ * @param fileName the file name you want to reset (relative path)
*/
public void reset(String localPath, String fileName) {
// Git repository address
try (Git git = Git.open(new File(localPath))) {
// reset files
git.reset().addPath(fileName.substring(1)).call();
- } catch (IOException e) {
- LOG.error(e.getMessage(), e);
- } catch (GitAPIException e) {
+ } catch (IOException | GitAPIException e) {
LOG.error(e.getMessage(), e);
}
}
/**
- * To execute pull command
- * @param localPath
- * @param token
- * @throws IOException
- * @throws GitAPIException
+ * To execute pull command.
+ * @param localPath the path of the local repository
+ * @param token the GitHub access token
+ * @return PullResult
*/
public PullResult pull(String localPath, String token, String branch) {
CredentialsProvider credentialsProvider =
@@ -160,19 +146,17 @@ public class GitUtils {
try (Git git = Git.open(new File(localPath))) {
pullResult = git.pull().setRemoteBranchName(branch).
setCredentialsProvider(credentialsProvider).call();
- } catch (IOException e) {
- LOG.error(e.getMessage(), e);
- } catch (GitAPIException e) {
+ } catch (IOException | GitAPIException e) {
LOG.error(e.getMessage(), e);
}
+
return pullResult;
}
/**
- * To execute commit command
- * @param localPath
- * @throws IOException
- * @throws GitAPIException
+ * To execute commit command.
+ * @param localPath the path of the local repository
+ * @return RevCommit
*/
public RevCommit commit(String localPath, String message) {
RevCommit revCommit = null;
@@ -180,20 +164,19 @@ public class GitUtils {
try (Git git = Git.open(new File(localPath))) {
// Submit code
revCommit = git.commit().setMessage(message).call();
- } catch (IOException e) {
- LOG.error(e.getMessage(), e);
- } catch (GitAPIException e) {
+ } catch (IOException | GitAPIException e) {
LOG.error(e.getMessage(), e);
}
+
return revCommit;
}
/**
- * To execute push command
- * @param localPath
- * @param token
- * @throws IOException
- * @throws GitAPIException
+ * To execute push command.
+ * @param localPath the path of the local repository
+ * @param token the GitHub access token
+ * @param remote the URI of remote repository
+ * @return Iterable<PushResult>
*/
public Iterable<PushResult> push(String localPath, String token, String remote) {
CredentialsProvider credentialsProvider =
@@ -202,20 +185,18 @@ public class GitUtils {
// Git repository address
try (Git git = Git.open(new File(localPath))) {
iterable = git.push().setRemote(remote).setCredentialsProvider(credentialsProvider).call();
- } catch (IOException e) {
- LOG.error(e.getMessage(), e);
- } catch (GitAPIException e) {
+ } catch (IOException | GitAPIException e) {
LOG.error(e.getMessage(), e);
}
+
return iterable;
}
/**
- * To execute branchCreate command
- * @param localPath
- * @param branchName
- * @throws IOException
- * @throws GitAPIException
+ * To execute branchCreate command.
+ * @param localPath the path of the local repository
+ * @param branchName the branch you want to create
+ * @return Ref
*/
public Ref branchCreate(String localPath, String branchName) {
Ref result = null;
@@ -223,99 +204,86 @@ public class GitUtils {
ListBranchCommand listBranchCommand = git.branchList();
List<Ref> list = listBranchCommand.call();
boolean existsBranch = false;
+
for (Ref ref : list) {
if (ref.getName().endsWith(branchName)) {
existsBranch = true;
break;
}
}
+
if (!existsBranch) {
// Create branch
result = git.branchCreate().setName(branchName).call();
} else {
LOG.warn("{} already exists.", branchName);
}
- } catch (IOException e) {
- LOG.error(e.getMessage(), e);
- } catch (GitAPIException e) {
+ } catch (IOException | GitAPIException e) {
LOG.error(e.getMessage(), e);
}
+
return result;
}
/**
- * To execute branchDelete command
- * @param localPath
- * @param branchName
- * @throws IOException
- * @throws GitAPIException
+ * To execute branchDelete command.
+ * @param localPath the path of the local repository
+ * @param branchName the branch you want to delete
+ * @return List<String>
*/
public List<String> branchDelete(String localPath, String branchName) {
List<String> list = null;
try (Git git = Git.open(new File(localPath))) {
list = git.branchDelete().setForce(true).setBranchNames(branchName).call();
- } catch (IOException e) {
- LOG.error(e.getMessage(), e);
- } catch (GitAPIException e) {
+ } catch (IOException | GitAPIException e) {
LOG.error(e.getMessage(), e);
}
+
return list;
}
/**
- * To execute checkout command
- * @param localPath
- * @param branchName
- * @throws IOException
- * @throws GitAPIException
+ * To execute checkout command.
+ * @param localPath the path of the local repository
+ * @param branchName the branch you want to checkout to
+ * @return Ref
*/
public Ref checkout(String localPath, String branchName) {
Ref result = null;
try (Git git = Git.open(new File(localPath))) {
result = git.checkout().setName(branchName).call();
- } catch (IOException e) {
- LOG.error(e.getMessage(), e);
- } catch (GitAPIException e) {
+ } catch (IOException | GitAPIException e) {
LOG.error(e.getMessage(), e);
}
+
return result;
}
/**
- * To execute rebase command
- * @param localPath
- * @param branchName
- * @param upstreamName
- * @throws IOException
- * @throws GitAPIException
+ * To execute rebase command.
+ * @param localPath the path of the local repository
+ * @param branchName the branch you want to rebase from
+ * @param upstreamName the name of upstream repository
*/
public void rebase(String localPath, String branchName, String upstreamName) {
try (Git git = Git.open(new File(localPath))) {
git.rebase().setUpstream(branchName).setUpstreamName(upstreamName).call();
- } catch (IOException e) {
- LOG.error(e.getMessage(), e);
- } catch (GitAPIException e) {
+ } catch (IOException | GitAPIException e) {
LOG.error(e.getMessage(), e);
}
}
/**
- * To execute remoteAdd command
- * @param localPath
- * @param uri
- * @param remoteName
- * @throws IOException
- * @throws GitAPIException
+ * To execute remoteAdd command.
+ * @param localPath the path of the local repository
+ * @param uri the URI prefix of the remote repository
+ * @param remoteName the name of the remote repository
*/
public void remoteAdd(String localPath, String uri, String remoteName) {
try (Git git = Git.open(new File(localPath))) {
URIish urIish = new URIish(uri);
git.remoteAdd().setName(remoteName).setUri(urIish).call();
- } catch (IOException e) {
- LOG.error(e.getMessage(), e);
- } catch (GitAPIException e) {
- LOG.error(e.getMessage(), e);
- } catch (URISyntaxException e) {
+ } catch (IOException | GitAPIException | URISyntaxException e) {
LOG.error(e.getMessage(), e);
}
}
diff --git a/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/utils/DepartmentUtil.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/utils/DepartmentUtil.java
index 59fa46e..2fd0c0a 100644
--- a/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/utils/DepartmentUtil.java
+++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/utils/DepartmentUtil.java
@@ -44,8 +44,7 @@ public class DepartmentUtil {
List<SysDeptSelectEntity> sysDeptSelects) {
sysDeptSelects.clear();
List<SysDeptTree> records = new ArrayList<>();
- for (int i = 0; i < sysDeptList.size(); i++) {
- SysDeptEntity dept = sysDeptList.get(i);
+ for (SysDeptEntity dept : sysDeptList) {
records.add(new SysDeptTree(dept));
}
List<SysDeptTree> sysOrgTreeList = findChildren(records, sysDeptSelects);
@@ -71,8 +70,7 @@ public class DepartmentUtil {
private static List<SysDeptTree> findChildren(List<SysDeptTree> sysDeptList,
List<SysDeptSelectEntity> sysDeptSelects) {
List<SysDeptTree> treeList = new ArrayList<>();
- for (int i = 0; i < sysDeptList.size(); i++) {
- SysDeptTree branch = sysDeptList.get(i);
+ for (SysDeptTree branch : sysDeptList) {
if (isEmpty(branch.getParentCode())) {
treeList.add(branch);
SysDeptSelectEntity departIdModel = new SysDeptSelectEntity().convert(branch);
@@ -90,8 +88,7 @@ public class DepartmentUtil {
for (int i = 0; i < treeList.size(); i++) {
SysDeptTree model = treeList.get(i);
SysDeptSelectEntity idModel = sysDeptSelects.get(i);
- for (int i1 = 0; i1 < recordList.size(); i1++) {
- SysDeptTree m = recordList.get(i1);
+ for (SysDeptTree m : recordList) {
if (m.getParentCode() != null && m.getParentCode().equals(model.getDeptCode())) {
model.getChildren().add(m);
SysDeptSelectEntity dim = new SysDeptSelectEntity().convert(m);
@@ -103,8 +100,7 @@ public class DepartmentUtil {
}
private static void setEmptyChildrenAsNull(List<SysDeptTree> treeList) {
- for (int i = 0; i < treeList.size(); i++) {
- SysDeptTree model = treeList.get(i);
+ for (SysDeptTree model : treeList) {
if (model.getChildren().size() == 0) {
model.setChildren(null);
} else {
diff --git a/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/utils/MybatisGenerator.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/utils/MybatisGenerator.java
index f0f28e0..13c48bb 100644
--- a/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/utils/MybatisGenerator.java
+++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/utils/MybatisGenerator.java
@@ -47,11 +47,10 @@ public class MybatisGenerator {
Configuration config = null;
try {
config = cp.parseConfiguration(configFile);
- } catch (IOException e) {
- LOG.error(e.getMessage(), e);
- } catch (XMLParserException e) {
+ } catch (IOException | XMLParserException e) {
LOG.error(e.getMessage(), e);
}
+
DefaultShellCallback callback = new DefaultShellCallback(overwrite);
MyBatisGenerator myBatisGenerator = null;
try {
@@ -61,11 +60,7 @@ public class MybatisGenerator {
}
try {
myBatisGenerator.generate(null);
- } catch (SQLException e) {
- LOG.error(e.getMessage(), e);
- } catch (IOException e) {
- LOG.error(e.getMessage(), e);
- } catch (InterruptedException e) {
+ } catch (SQLException | IOException | InterruptedException e) {
LOG.error(e.getMessage(), e);
}
}
diff --git a/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/websocket/ConnectionManager.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/websocket/ConnectionManager.java
index fbb173d..e827ba2 100644
--- a/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/websocket/ConnectionManager.java
+++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/websocket/ConnectionManager.java
@@ -33,11 +33,11 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedQueue;
/**
- * Manager class for managing websocket connections
+ * Manager class for managing websocket connections.
*/
public class ConnectionManager {
private static final Logger LOG = LoggerFactory.getLogger(ConnectionManager.class);
- private static Gson gson = new GsonBuilder()
+ private static final Gson gson = new GsonBuilder()
.setDateFormat("yyyy-MM-dd'T'HH:mm:ssZ")
.registerTypeAdapter(Date.class, new DateJsonDeserializer())
.setPrettyPrinting()
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@submarine.apache.org
For additional commands, e-mail: dev-help@submarine.apache.org