You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@submarine.apache.org by pi...@apache.org on 2021/08/06 14:16:36 UTC
[submarine] branch master updated: SUBMARINE-962. Delete temporary
build artifacts before caching
This is an automated email from the ASF dual-hosted git repository.
pingsutw pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/submarine.git
The following commit(s) were added to refs/heads/master by this push:
new 214582d SUBMARINE-962. Delete temporary build artifacts before caching
214582d is described below
commit 214582d7d643dd277f32c1c43e94d3301f3b3110
Author: Kevin Su <pi...@apache.org>
AuthorDate: Fri Aug 6 12:19:30 2021 +0800
SUBMARINE-962. Delete temporary build artifacts before caching
### What is this PR for?
<!-- A few sentences describing the overall goals of the pull request's commits.
First time? Check out the contributing guide - https://submarine.apache.org/contribution/contributions.html
-->
When building the Submarine repo, some of the modules would use outdated packages in `~/.m2/repository/org/apache` in Github action cache
We should delete temporary build artifacts before caching
I believe it will fix the test failures
### What type of PR is it?
[Bug Fix]
### Todos
No
### What is the Jira issue?
<!-- * Open an issue on Jira https://issues.apache.org/jira/browse/SUBMARINE/
* Put link here, and add [SUBMARINE-*Jira number*] in PR title, eg. `SUBMARINE-23. PR title`
-->
https://issues.apache.org/jira/browse/SUBMARINE-962
### How should this be tested?
<!--
* First time? Setup Travis CI as described on https://submarine.apache.org/contribution/contributions.html#continuous-integration
* Strongly recommended: add automated unit tests for any new or changed behavior
* Outline any manual steps to test the PR here.
-->
Pass the CIs
### Screenshots (if appropriate)
### Questions:
* Do the license files need updating? No
* Are there breaking changes for older versions? No
* Does this need new documentation? No
Author: Kevin Su <pi...@apache.org>
Signed-off-by: Kevin <pi...@apache.org>
Closes #695 from pingsutw/SUBMARINE-962 and squashes the following commits:
af537c9d [Kevin Su] SUBMARINE-962. Delete temporary build artifacts before caching
---
.github/workflows/master.yml | 10 +++++++
.../server/experiment/ExperimentManager.java | 8 +++---
.../server/experiment/ExperimentManagerTest.java | 2 +-
.../apache/submarine/rest/ExperimentRestApiIT.java | 26 +++++++++++-------
.../rest/ExperimentTemplateManagerRestApiIT.java | 32 +++++++++++-----------
5 files changed, 47 insertions(+), 31 deletions(-)
diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
index 6771302..3f3248b 100644
--- a/.github/workflows/master.yml
+++ b/.github/workflows/master.yml
@@ -58,6 +58,11 @@ jobs:
with:
name: submarine-bin
path: submarine-dist/target/submarine-dist*.tar.gz
+ - name: Delete temporary build artifacts before caching
+ run: |
+ #Never cache local artifacts
+ rm -rf ~/.m2/repository/org/apache/submarine
+ if: always()
submarine-e2e:
runs-on: ubuntu-latest
timeout-minutes: 30
@@ -174,6 +179,11 @@ jobs:
kubectl get pods
kubectl -n default get events --sort-by='{.lastTimestamp}'
if: ${{ failure() }}
+ - name: Delete temporary build artifacts before caching
+ run: |
+ #Never cache local artifacts
+ rm -rf ~/.m2/repository/org/apache/submarine
+ if: always()
submarine-commons-cluster:
runs-on: ubuntu-latest
timeout-minutes: 30
diff --git a/submarine-server/server-core/src/main/java/org/apache/submarine/server/experiment/ExperimentManager.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/experiment/ExperimentManager.java
index 7466bf8..33d76ce 100644
--- a/submarine-server/server-core/src/main/java/org/apache/submarine/server/experiment/ExperimentManager.java
+++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/experiment/ExperimentManager.java
@@ -115,7 +115,7 @@ public class ExperimentManager {
spec.getMeta().getEnvVars().put(RestConstants.SUBMARINE_TRACKING_URI, url);
spec.getMeta().getEnvVars().put(RestConstants.LOG_DIR_KEY, RestConstants.LOG_DIR_VALUE);
- String lowerName = spec.getMeta().getName().toLowerCase();
+ String lowerName = spec.getMeta().getName().toLowerCase();
spec.getMeta().setName(lowerName);
spec.getMeta().setExperimentId(id.toString());
@@ -300,7 +300,7 @@ public class ExperimentManager {
experimentLogList.add(submitter.getExperimentLogName(
experiment.getSpec(),
- experiment.getExperimentId().toString()
+ experiment.getSpec().getMeta().getExperimentId()
));
}
@@ -326,7 +326,7 @@ public class ExperimentManager {
return submitter.getExperimentLog(
experiment.getSpec(),
- experiment.getExperimentId().toString()
+ experiment.getSpec().getMeta().getExperimentId()
);
}
@@ -426,7 +426,7 @@ public class ExperimentManager {
*/
private ExperimentEntity buildEntityFromExperiment(Experiment experiment) {
ExperimentEntity entity = new ExperimentEntity();
- entity.setId(experiment.getExperimentId().toString());
+ entity.setId(experiment.getSpec().getMeta().getExperimentId());
entity.setExperimentSpec(new GsonBuilder().disableHtmlEscaping().create().toJson(experiment.getSpec()));
return entity;
}
diff --git a/submarine-server/server-core/src/test/java/org/apache/submarine/server/experiment/ExperimentManagerTest.java b/submarine-server/server-core/src/test/java/org/apache/submarine/server/experiment/ExperimentManagerTest.java
index b8ec96a..bd8880b 100644
--- a/submarine-server/server-core/src/test/java/org/apache/submarine/server/experiment/ExperimentManagerTest.java
+++ b/submarine-server/server-core/src/test/java/org/apache/submarine/server/experiment/ExperimentManagerTest.java
@@ -125,7 +125,7 @@ public class ExperimentManagerTest {
experimentId2.setId(2);
ExperimentEntity entity1 = new ExperimentEntity();
- entity1.setId(experiment1.getExperimentId().toString());
+ entity1.setId(experiment1.getSpec().getMeta().getExperimentId());
entity1.setExperimentSpec(new GsonBuilder().disableHtmlEscaping().create().toJson(experiment1.getSpec()));
doReturn(Arrays.asList(entity1)).when(mockService).selectAll();
diff --git a/submarine-test/test-k8s/src/test/java/org/apache/submarine/rest/ExperimentRestApiIT.java b/submarine-test/test-k8s/src/test/java/org/apache/submarine/rest/ExperimentRestApiIT.java
index c9dc564..0f36cc3 100644
--- a/submarine-test/test-k8s/src/test/java/org/apache/submarine/rest/ExperimentRestApiIT.java
+++ b/submarine-test/test-k8s/src/test/java/org/apache/submarine/rest/ExperimentRestApiIT.java
@@ -105,7 +105,7 @@ public class ExperimentRestApiIT extends AbstractSubmarineServerTest {
@Before
public void setUp() throws Exception {
- Thread.sleep(5000); // timeout for each case, ensuring k8s-client has enough time to delete resoures
+ Thread.sleep(5000); // timeout for each case, ensuring k8s-client has enough time to delete resources
}
@Test
@@ -204,8 +204,10 @@ public class ExperimentRestApiIT extends AbstractSubmarineServerTest {
Assert.assertEquals(Response.Status.OK.getStatusCode(), jsonResponse.getCode());
Experiment createdExperiment = gson.fromJson(gson.toJson(jsonResponse.getResult()), Experiment.class);
verifyCreateJobApiResult(createdExperiment);
+
// find
- GetMethod getMethod = httpGet(BASE_API_PATH + "/" + createdExperiment.getExperimentId().toString());
+ GetMethod getMethod = httpGet(BASE_API_PATH + "/" +
+ createdExperiment.getSpec().getMeta().getExperimentId());
Assert.assertEquals(Response.Status.OK.getStatusCode(), getMethod.getStatusCode());
json = getMethod.getResponseBodyAsString();
@@ -223,7 +225,8 @@ public class ExperimentRestApiIT extends AbstractSubmarineServerTest {
// https://tools.ietf.org/html/rfc5789
// delete
- DeleteMethod deleteMethod = httpDelete(BASE_API_PATH + "/" + createdExperiment.getExperimentId().toString());
+ DeleteMethod deleteMethod = httpDelete(BASE_API_PATH + "/" +
+ createdExperiment.getSpec().getMeta().getExperimentId());
Assert.assertEquals(Response.Status.OK.getStatusCode(), deleteMethod.getStatusCode());
json = deleteMethod.getResponseBodyAsString();
@@ -253,7 +256,7 @@ public class ExperimentRestApiIT extends AbstractSubmarineServerTest {
// find
GetMethod getMethod =
- httpGet(BASE_API_PATH + "/" + createdExperiment.getExperimentId().toString());
+ httpGet(BASE_API_PATH + "/" + createdExperiment.getSpec().getMeta().getExperimentId());
Assert.assertEquals(Response.Status.OK.getStatusCode(),
getMethod.getStatusCode());
@@ -268,7 +271,7 @@ public class ExperimentRestApiIT extends AbstractSubmarineServerTest {
// delete
DeleteMethod deleteMethod =
- httpDelete(BASE_API_PATH + "/" + createdExperiment.getExperimentId().toString());
+ httpDelete(BASE_API_PATH + "/" + createdExperiment.getSpec().getMeta().getExperimentId());
Assert.assertEquals(Response.Status.OK.getStatusCode(),
deleteMethod.getStatusCode());
@@ -304,7 +307,8 @@ public class ExperimentRestApiIT extends AbstractSubmarineServerTest {
Experiment createdExperiment, Experiment foundExperiment) throws Exception {
Assert.assertEquals(createdExperiment.getExperimentId(), foundExperiment.getExperimentId());
Assert.assertEquals(createdExperiment.getUid(), foundExperiment.getUid());
- Assert.assertEquals(createdExperiment.getExperimentId().toString(), foundExperiment.getExperimentId().toString());
+ Assert.assertEquals(createdExperiment.getSpec().getMeta().getExperimentId(),
+ foundExperiment.getSpec().getMeta().getExperimentId());
Assert.assertEquals(createdExperiment.getAcceptedTime(), foundExperiment.getAcceptedTime());
assertK8sResultEquals(foundExperiment);
@@ -314,7 +318,7 @@ public class ExperimentRestApiIT extends AbstractSubmarineServerTest {
KfOperator operator = kfOperatorMap.get(experiment.getSpec().getMeta().getFramework().toLowerCase());
JsonObject rootObject =
getJobByK8sApi(operator.getGroup(), operator.getVersion(),
- operator.getNamespace(), operator.getPlural(), experiment.getExperimentId().toString());
+ operator.getNamespace(), operator.getPlural(), experiment.getSpec().getMeta().getExperimentId());
JsonArray actualCommand = (JsonArray) rootObject.getAsJsonObject("spec")
.getAsJsonObject("tfReplicaSpecs").getAsJsonObject("Worker")
.getAsJsonObject("template").getAsJsonObject("spec")
@@ -386,7 +390,7 @@ public class ExperimentRestApiIT extends AbstractSubmarineServerTest {
private void assertK8sResultEquals(Experiment experiment) throws Exception {
KfOperator operator = kfOperatorMap.get(experiment.getSpec().getMeta().getFramework().toLowerCase());
JsonObject rootObject = getJobByK8sApi(operator.getGroup(), operator.getVersion(),
- operator.getNamespace(), operator.getPlural(), experiment.getExperimentId().toString());
+ operator.getNamespace(), operator.getPlural(), experiment.getSpec().getMeta().getExperimentId());
JsonObject metadataObject = rootObject.getAsJsonObject("metadata");
String uid = metadataObject.getAsJsonPrimitive("uid").getAsString();
@@ -404,7 +408,8 @@ public class ExperimentRestApiIT extends AbstractSubmarineServerTest {
}
private void verifyDeleteJobApiResult(Experiment createdExperiment, Experiment deletedExperiment) {
- Assert.assertEquals(createdExperiment.getExperimentId().toString(), deletedExperiment.getExperimentId().toString());
+ Assert.assertEquals(createdExperiment.getSpec().getMeta().getExperimentId(),
+ deletedExperiment.getSpec().getMeta().getExperimentId());
Assert.assertEquals(Experiment.Status.STATUS_DELETED.getValue(), deletedExperiment.getStatus());
// verify the result by K8s api
@@ -413,7 +418,8 @@ public class ExperimentRestApiIT extends AbstractSubmarineServerTest {
JsonObject rootObject = null;
try {
rootObject = getJobByK8sApi(operator.getGroup(), operator.getVersion(),
- operator.getNamespace(), operator.getPlural(), createdExperiment.getExperimentId().toString());
+ operator.getNamespace(), operator.getPlural(),
+ createdExperiment.getSpec().getMeta().getExperimentId());
} catch (ApiException e) {
Assert.assertEquals(Response.Status.NOT_FOUND.getStatusCode(), e.getCode());
} finally {
diff --git a/submarine-test/test-k8s/src/test/java/org/apache/submarine/rest/ExperimentTemplateManagerRestApiIT.java b/submarine-test/test-k8s/src/test/java/org/apache/submarine/rest/ExperimentTemplateManagerRestApiIT.java
index f902ac7..acf3383 100644
--- a/submarine-test/test-k8s/src/test/java/org/apache/submarine/rest/ExperimentTemplateManagerRestApiIT.java
+++ b/submarine-test/test-k8s/src/test/java/org/apache/submarine/rest/ExperimentTemplateManagerRestApiIT.java
@@ -48,7 +48,7 @@ import com.google.gson.reflect.TypeToken;
@SuppressWarnings("rawtypes")
public class ExperimentTemplateManagerRestApiIT extends AbstractSubmarineServerTest {
-
+
protected static String TPL_PATH =
"/api/" + RestConstants.V1 + "/" + RestConstants.EXPERIMENT_TEMPLATES;
protected static String EXP_PATH =
@@ -57,7 +57,7 @@ public class ExperimentTemplateManagerRestApiIT extends AbstractSubmarineServerT
protected static String TPL_FILE = "experimentTemplate/test_template_1.json";
protected static String TPL_SUBMIT_FILE = "experimentTemplate/test_template_1_submit.json";
protected static String TPL_SUBMIT_NAME_PARM = "experiment_name";
-
+
private final Gson gson = new GsonBuilder()
.registerTypeAdapter(ExperimentId.class, new ExperimentIdSerializer())
.registerTypeAdapter(ExperimentId.class, new ExperimentIdDeserializer())
@@ -121,7 +121,7 @@ public class ExperimentTemplateManagerRestApiIT extends AbstractSubmarineServerT
String body = loadContent(TPL_FILE);
run(body, "application/json");
-
+
GetMethod getMethod = httpGet(TPL_PATH + "/");
Assert.assertEquals(Response.Status.OK.getStatusCode(),
getMethod.getStatusCode());
@@ -134,7 +134,7 @@ public class ExperimentTemplateManagerRestApiIT extends AbstractSubmarineServerT
List<ExperimentTemplate> getExperimentTemplates =
gson.fromJson(gson.toJson(jsonResponse.getResult()), new TypeToken<List<ExperimentTemplate>>() {
}.getType());
-
+
Assert.assertEquals(TPL_NAME, getExperimentTemplates.get(0).getExperimentTemplateSpec().getName());
deleteExperimentTemplate();
@@ -162,7 +162,7 @@ public class ExperimentTemplateManagerRestApiIT extends AbstractSubmarineServerT
LOG.info("Create ExperimentTemplate using ExperimentTemplate REST API");
PostMethod postMethod = httpPost(TPL_PATH, body, contentType);
LOG.info(postMethod.getResponseBodyAsString());
-
+
Assert.assertEquals(Response.Status.OK.getStatusCode(),
postMethod.getStatusCode());
@@ -185,24 +185,24 @@ public class ExperimentTemplateManagerRestApiIT extends AbstractSubmarineServerT
@Test
public void submitExperimentTemplate() throws Exception {
String body = loadContent(TPL_FILE);
- run(body, "application/json");
-
- ExperimentTemplateSpec tplspec =
+ run(body, "application/json");
+
+ ExperimentTemplateSpec tplspec =
gson.fromJson(body, ExperimentTemplateSpec.class);
-
+
String url = EXP_PATH + "/" + tplspec.getName();
LOG.info("Submit ExperimentTemplate using ExperimentTemplate REST API");
LOG.info(body);
String submitBody = loadContent(TPL_SUBMIT_FILE);
- ExperimentTemplateSubmit tplSubmit =
+ ExperimentTemplateSubmit tplSubmit =
gson.fromJson(submitBody, ExperimentTemplateSubmit.class);
PostMethod postMethod = httpPost(url, submitBody, "application/json");
LOG.info(postMethod.getResponseBodyAsString());
- Assert.assertEquals(Response.Status.OK.getStatusCode(),
+ Assert.assertEquals(Response.Status.OK.getStatusCode(),
postMethod.getStatusCode());
-
+
String json = postMethod.getResponseBodyAsString();
LOG.info(json);
JsonResponse jsonResponse = gson.fromJson(json, JsonResponse.class);
@@ -211,16 +211,16 @@ public class ExperimentTemplateManagerRestApiIT extends AbstractSubmarineServerT
deleteExperimentTemplate();
LOG.info(gson.toJson(jsonResponse.getResult()));
-
+
Experiment experiment = gson.fromJson(gson.toJson(jsonResponse.getResult()), Experiment.class);
- DeleteMethod deleteMethod = httpDelete("/api/" + RestConstants.V1 + "/" + RestConstants.EXPERIMENT + "/"
- + experiment.getExperimentId().toString());
+ DeleteMethod deleteMethod = httpDelete("/api/" + RestConstants.V1 + "/" + RestConstants.EXPERIMENT + "/"
+ + experiment.getSpec().getMeta().getExperimentId());
Assert.assertEquals(Response.Status.OK.getStatusCode(), deleteMethod.getStatusCode());
json = deleteMethod.getResponseBodyAsString();
jsonResponse = gson.fromJson(json, JsonResponse.class);
Assert.assertEquals(Response.Status.OK.getStatusCode(), jsonResponse.getCode());
-
+
ExperimentSpec expSpec = experiment.getSpec();
LOG.info(expSpec.getMeta().toString());
Assert.assertEquals(tplSubmit.getParams().get(TPL_SUBMIT_NAME_PARM), expSpec.getMeta().getName());
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@submarine.apache.org
For additional commands, e-mail: dev-help@submarine.apache.org