You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by ma...@apache.org on 2022/02/17 00:43:24 UTC
[airavata-django-portal] 03/16: AIRAVATA-3557 REST API for getting intermediate output info
This is an automated email from the ASF dual-hosted git repository.
machristie pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/airavata-django-portal.git
commit 1fa1f4b6f4779bdb628c4454641e46d87f46cfb3
Author: Marcus Christie <ma...@apache.org>
AuthorDate: Wed Feb 2 17:56:07 2022 -0500
AIRAVATA-3557 REST API for getting intermediate output info
---
django_airavata/apps/api/serializers.py | 78 +++++++++++++++++++++-
.../js/models/IntermediateOutput.js | 21 ++++++
.../js/models/OutputDataObjectType.js | 5 ++
.../django_airavata_api/js/models/ProcessState.js | 12 +++-
.../django_airavata_api/js/models/ProcessStatus.js | 4 ++
.../output-displays/OutputDisplayContainer.vue | 20 +++++-
.../js/store/modules/view-experiment.js | 10 ++-
7 files changed, 144 insertions(+), 6 deletions(-)
diff --git a/django_airavata/apps/api/serializers.py b/django_airavata/apps/api/serializers.py
index 4c6297e..79bef88 100644
--- a/django_airavata/apps/api/serializers.py
+++ b/django_airavata/apps/api/serializers.py
@@ -2,6 +2,7 @@ import copy
import datetime
import json
import logging
+import os
from urllib.parse import quote
from airavata.model.appcatalog.appdeployment.ttypes import (
@@ -49,7 +50,13 @@ from airavata.model.experiment.ttypes import (
)
from airavata.model.group.ttypes import GroupModel, ResourcePermissionType
from airavata.model.job.ttypes import JobModel
-from airavata.model.status.ttypes import ExperimentStatus
+from airavata.model.status.ttypes import (
+ ExperimentState,
+ ExperimentStatus,
+ ProcessStatus,
+ TaskState
+)
+from airavata.model.task.ttypes import TaskTypes
from airavata.model.user.ttypes import UserProfile
from airavata.model.workspace.ttypes import (
Notification,
@@ -407,6 +414,11 @@ class ExperimentStatusSerializer(
timeOfStateChange = UTCPosixTimestampDateTimeField()
+class ProcessStatusSerializer(
+ thrift_utils.create_serializer_class(ProcessStatus)):
+ timeOfStateChange = UTCPosixTimestampDateTimeField()
+
+
class ExperimentSerializer(
thrift_utils.create_serializer_class(ExperimentModel)):
class Meta:
@@ -450,6 +462,70 @@ class ExperimentSerializer(
request.authz_token, experiment.experimentId,
ResourcePermissionType.WRITE)
+ def to_representation(self, experiment):
+ result = super().to_representation(experiment)
+ self._add_intermediate_output_information(experiment, result)
+ return result
+
+ def _add_intermediate_output_information(self, experiment, representation):
+ request = self.context['request']
+ # sort the processes and filter to just the output fetching ones
+ processes = sorted(experiment.processes, key=lambda p: p.creationTime, reverse=True) if experiment.processes else []
+ output_fetching_processes = []
+ for process in processes:
+ if any(map(lambda t: t.taskType == TaskTypes.OUTPUT_FETCHING, process.tasks)):
+ output_fetching_processes.append(process)
+
+ # If there are output_fetching processes and experiment is EXECUTING,
+ # add intermediateOutput information to experiment outputs
+ if (len(output_fetching_processes) > 0 and
+ experiment.experimentStatus and
+ experiment.experimentStatus[-1].state == ExperimentState.EXECUTING):
+ for output in representation["experimentOutputs"]:
+ try:
+ process_status = request.airavata_client.getIntermediateOutputProcessStatus(
+ request.authz_token, experiment.experimentId, [output["name"]])
+ serializer = ProcessStatusSerializer(
+ process_status, context={'request': request})
+ output["intermediateOutput"] = {"processStatus": serializer.data}
+ most_recent_completed_process_output = None
+ for process in output_fetching_processes:
+ tasks = process.tasks
+ last_task_statuses = map(lambda t: t.taskStatuses[-1], tasks)
+ # If any tasks aren't complete, skip this process
+ if any(map(lambda ts: ts.state != TaskState.COMPLETED, last_task_statuses)):
+ continue
+ for process_output in process.processOutputs:
+ if process_output.name == output["name"]:
+ most_recent_completed_process_output = process_output
+ break
+ if most_recent_completed_process_output is not None:
+ break
+ if most_recent_completed_process_output is not None:
+
+ data_product_uris = []
+ if most_recent_completed_process_output.value.startswith('airavata-dp://'):
+ data_product_uris = most_recent_completed_process_output.value.split(',')
+ else:
+ # TODO: should really find the process and use the data
+ # product uris from the process outputs instead, this is
+ # just a temporary workaround
+ data_product_uri = user_storage.user_file_exists(
+ request, os.path.join("intermediates", most_recent_completed_process_output.value),
+ experiment_id=experiment.experimentId)
+ if data_product_uri:
+ data_product_uris.append(data_product_uri)
+ data_products = []
+ for data_product_uri in data_product_uris:
+ data_product = request.airavata_client.getDataProduct(
+ request.authz_token, data_product_uri)
+ data_products.append(data_product)
+ data_product_serializer = DataProductSerializer(
+ data_products, context={'request': request}, many=True)
+ output["intermediateOutput"]["dataProducts"] = data_product_serializer.data
+ except Exception:
+ output["intermediateOutput"] = {"processStatus": None}
+
class DataReplicaLocationSerializer(
thrift_utils.create_serializer_class(DataReplicaLocationModel)):
diff --git a/django_airavata/apps/api/static/django_airavata_api/js/models/IntermediateOutput.js b/django_airavata/apps/api/static/django_airavata_api/js/models/IntermediateOutput.js
new file mode 100644
index 0000000..45e13c2
--- /dev/null
+++ b/django_airavata/apps/api/static/django_airavata_api/js/models/IntermediateOutput.js
@@ -0,0 +1,21 @@
+import BaseModel from "./BaseModel";
+import ProcessStatus from "./ProcessStatus";
+import DataProduct from "./DataProduct";
+
+const FIELDS = [
+ {
+ name: "processStatus",
+ type: ProcessStatus,
+ },
+ {
+ name: "dataProducts",
+ type: DataProduct,
+ list: true,
+ },
+];
+
+export default class IntermediateOutput extends BaseModel {
+ constructor(data = {}) {
+ super(FIELDS, data);
+ }
+}
diff --git a/django_airavata/apps/api/static/django_airavata_api/js/models/OutputDataObjectType.js b/django_airavata/apps/api/static/django_airavata_api/js/models/OutputDataObjectType.js
index ba36e74..b3e79de 100644
--- a/django_airavata/apps/api/static/django_airavata_api/js/models/OutputDataObjectType.js
+++ b/django_airavata/apps/api/static/django_airavata_api/js/models/OutputDataObjectType.js
@@ -1,6 +1,7 @@
import BaseModel from "./BaseModel";
import DataType from "./DataType";
import uuidv4 from "uuid/v4";
+import IntermediateOutput from "./IntermediateOutput";
const FIELDS = [
"name",
@@ -35,6 +36,10 @@ const FIELDS = [
},
"storageResourceId",
"metaData",
+ {
+ name: "intermediateOutput",
+ type: IntermediateOutput,
+ }
];
export default class OutputDataObjectType extends BaseModel {
diff --git a/django_airavata/apps/api/static/django_airavata_api/js/models/ProcessState.js b/django_airavata/apps/api/static/django_airavata_api/js/models/ProcessState.js
index ba6bca0..d8e592a 100644
--- a/django_airavata/apps/api/static/django_airavata_api/js/models/ProcessState.js
+++ b/django_airavata/apps/api/static/django_airavata_api/js/models/ProcessState.js
@@ -1,6 +1,16 @@
import BaseEnum from "./BaseEnum";
-export default class ProcessState extends BaseEnum {}
+export default class ProcessState extends BaseEnum {
+
+ get isFinished() {
+ const finishedStates = [
+ ProcessState.CANCELED,
+ ProcessState.COMPLETED,
+ ProcessState.FAILED,
+ ];
+ return finishedStates.indexOf(this) >= 0;
+ }
+}
ProcessState.init([
"CREATED",
"VALIDATED",
diff --git a/django_airavata/apps/api/static/django_airavata_api/js/models/ProcessStatus.js b/django_airavata/apps/api/static/django_airavata_api/js/models/ProcessStatus.js
index 3f6148d..1a294e2 100644
--- a/django_airavata/apps/api/static/django_airavata_api/js/models/ProcessStatus.js
+++ b/django_airavata/apps/api/static/django_airavata_api/js/models/ProcessStatus.js
@@ -18,4 +18,8 @@ export default class ProcessStatus extends BaseModel {
constructor(data = {}) {
super(FIELDS, data);
}
+
+ get isFinished() {
+ return this.state && this.state.isFinished;
+ }
}
diff --git a/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/output-displays/OutputDisplayContainer.vue b/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/output-displays/OutputDisplayContainer.vue
index ed37b55..4f3231c 100644
--- a/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/output-displays/OutputDisplayContainer.vue
+++ b/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/output-displays/OutputDisplayContainer.vue
@@ -26,7 +26,15 @@
/>
<div slot="footer" v-if="dataProducts.length > 0 || isExecuting">
<template v-if="isExecuting">
- <b-btn size="sm" @click="fetchLatest">Fetch Latest</b-btn>
+ <span v-if="experimentOutput.intermediateOutput.processStatus"
+ >Status of intermediate output fetch:
+ {{
+ experimentOutput.intermediateOutput.processStatus.state.name
+ }}</span
+ >
+ <b-btn size="sm" @click="fetchLatest" :disabled="fetchLatestDisabled"
+ >Fetch Latest</b-btn
+ >
</template>
<template v-else>
<!-- TODO: support downloading URI_COLLECTIONs as well -->
@@ -157,6 +165,16 @@ export default {
hasInteractiveParameters() {
return this.viewData && this.viewData.interactive;
},
+ canFetchIntermediateOutput() {
+ return (
+ this.isExecuting &&
+ (!this.experimentOutput.intermediateOutput.processStatus ||
+ this.experimentOutput.intermediateOutput.processStatus.isFinished)
+ );
+ },
+ fetchLatestDisabled() {
+ return !this.canFetchIntermediateOutput;
+ },
},
methods: {
...mapActions("viewExperiment", ["submitFetchIntermediateOutputs"]),
diff --git a/django_airavata/apps/workspace/static/django_airavata_workspace/js/store/modules/view-experiment.js b/django_airavata/apps/workspace/static/django_airavata_workspace/js/store/modules/view-experiment.js
index 9f2fc26..534fa6d 100644
--- a/django_airavata/apps/workspace/static/django_airavata_workspace/js/store/modules/view-experiment.js
+++ b/django_airavata/apps/workspace/static/django_airavata_workspace/js/store/modules/view-experiment.js
@@ -43,10 +43,10 @@ export const actions = {
dispatch("initPollingExperiment");
}
},
- async loadExperiment({ commit }, { experimentId }) {
+ async loadExperiment({ commit }, { experimentId, showSpinner = false }) {
const fullExperiment = await services.FullExperimentService.retrieve(
{ lookup: experimentId },
- { ignoreErrors: true, showSpinner: false }
+ { ignoreErrors: true, showSpinner }
);
commit("setFullExperiment", { fullExperiment });
},
@@ -111,7 +111,11 @@ export const actions = {
outputNames,
},
});
- dispatch("loadExperiment", { experimentId: getters.experimentId });
+ // Block UI until we get the current status of intermediate output fetches
+ dispatch("loadExperiment", {
+ experimentId: getters.experimentId,
+ showSpinner: true,
+ });
},
};