You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by ma...@apache.org on 2018/12/13 19:59:44 UTC
[airavata-django-portal] 02/03: AIRAVATA-2616 Allow removing and
changing input files
This is an automated email from the ASF dual-hosted git repository.
machristie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-django-portal.git
commit cfc33747f98593fb3b237935045bff618ee11273
Author: Marcus Christie <ma...@iu.edu>
AuthorDate: Thu Dec 13 14:49:08 2018 -0500
AIRAVATA-2616 Allow removing and changing input files
---
django_airavata/apps/api/datastore.py | 14 ++++
django_airavata/apps/api/serializers.py | 24 ++++++-
.../api/static/django_airavata_api/js/index.js | 3 +
.../django_airavata_api/js/service_config.js | 10 +++
django_airavata/apps/api/urls.py | 3 +
django_airavata/apps/api/views.py | 41 +++++++++---
.../js/components/experiment/DataProductViewer.vue | 40 ++++++++++++
.../js/components/experiment/ExperimentSummary.vue | 31 ++++-----
.../experiment/input-editors/FileInputEditor.vue | 76 +++++++++++++++++++---
.../js/containers/EditExperimentContainer.vue | 2 +-
10 files changed, 205 insertions(+), 39 deletions(-)
diff --git a/django_airavata/apps/api/datastore.py b/django_airavata/apps/api/datastore.py
index aa4c17b..16576c0 100644
--- a/django_airavata/apps/api/datastore.py
+++ b/django_airavata/apps/api/datastore.py
@@ -72,6 +72,20 @@ def save(username, project_name, experiment_name, file):
return data_product
+def delete(data_product):
+ """Delete replica for data product in this data store."""
+ if exists(data_product):
+ filepath = _get_replica_filepath(data_product)
+ try:
+ experiment_data_storage.delete(filepath)
+ except Exception as e:
+ logger.error("Unable to delete file {} for data product uri {}"
+ .format(filepath, data_product.productUri))
+ raise
+ else:
+ raise ObjectDoesNotExist("Replica file does not exist")
+
+
def get_experiment_dir(username, project_name, experiment_name):
"""Return an experiment directory (full path) for the given experiment."""
experiment_dir_name = os.path.join(
diff --git a/django_airavata/apps/api/serializers.py b/django_airavata/apps/api/serializers.py
index 13bbeb5..74cdf04 100644
--- a/django_airavata/apps/api/serializers.py
+++ b/django_airavata/apps/api/serializers.py
@@ -33,7 +33,7 @@ from airavata.model.appcatalog.parser.ttypes import Parser
from airavata.model.appcatalog.storageresource.ttypes import (
StorageResourceDescription
)
-from airavata.model.application.io.ttypes import InputDataObjectType
+from airavata.model.application.io.ttypes import DataType, InputDataObjectType
from airavata.model.credential.store.ttypes import (
CredentialSummary,
SummaryType
@@ -419,6 +419,24 @@ class ExperimentSerializer(
request.authz_token, experiment.experimentId,
ResourcePermissionType.WRITE)
+ def update(self, instance, validated_data):
+ result = super().update(instance, validated_data)
+ removed_input_files = self._find_removed_input_files(
+ instance.experimentInputs, result.experimentInputs)
+ result._removed_input_files = removed_input_files
+ return result
+
+ def _find_removed_input_files(self,
+ old_experiment_inputs,
+ new_experiment_inputs):
+ old_input_data_product_uris = set(
+ inp.value for inp in old_experiment_inputs
+ if inp.type == DataType.URI)
+ new_input_data_product_uris = set(
+ inp.value for inp in new_experiment_inputs
+ if inp.type == DataType.URI)
+ return old_input_data_product_uris - new_input_data_product_uris
+
class DataReplicaLocationSerializer(
thrift_utils.create_serializer_class(DataReplicaLocationModel)):
@@ -432,6 +450,10 @@ class DataProductSerializer(
lastModifiedTime = UTCPosixTimestampDateTimeField()
replicaLocations = DataReplicaLocationSerializer(many=True)
downloadURL = serializers.SerializerMethodField()
+ url = FullyEncodedHyperlinkedIdentityField(
+ view_name='django_airavata_api:data-product-detail',
+ lookup_field='productUri',
+ lookup_url_kwarg='product_uri')
def get_downloadURL(self, data_product):
"""Getter for downloadURL field."""
diff --git a/django_airavata/apps/api/static/django_airavata_api/js/index.js b/django_airavata/apps/api/static/django_airavata_api/js/index.js
index 34842a9..406494f 100644
--- a/django_airavata/apps/api/static/django_airavata_api/js/index.js
+++ b/django_airavata/apps/api/static/django_airavata_api/js/index.js
@@ -11,6 +11,7 @@ import BatchQueue from "./models/BatchQueue";
import BatchQueueResourcePolicy from "./models/BatchQueueResourcePolicy";
import CommandObject from "./models/CommandObject";
import ComputeResourcePolicy from "./models/ComputeResourcePolicy";
+import DataProduct from "./models/DataProduct";
import DataType from "./models/DataType";
import Experiment from "./models/Experiment";
import ExperimentState from "./models/ExperimentState";
@@ -63,6 +64,7 @@ exports.models = {
BatchQueueResourcePolicy,
CommandObject,
ComputeResourcePolicy,
+ DataProduct,
DataType,
Experiment,
ExperimentState,
@@ -92,6 +94,7 @@ exports.services = {
CloudJobSubmissionService,
ComputeResourceService: ServiceFactory.service("ComputeResources"),
CredentialSummaryService: ServiceFactory.service("CredentialSummaries"),
+ DataProductService: ServiceFactory.service("DataProducts"),
ExperimentSearchService: ServiceFactory.service("ExperimentSearch"),
ExperimentService: ServiceFactory.service("Experiments"),
FullExperimentService,
diff --git a/django_airavata/apps/api/static/django_airavata_api/js/service_config.js b/django_airavata/apps/api/static/django_airavata_api/js/service_config.js
index e3adc09..9e7a7c8 100644
--- a/django_airavata/apps/api/static/django_airavata_api/js/service_config.js
+++ b/django_airavata/apps/api/static/django_airavata_api/js/service_config.js
@@ -4,6 +4,7 @@ import ApplicationModule from "./models/ApplicationModule";
import BatchQueue from "./models/BatchQueue";
import ComputeResourceDescription from "./models/ComputeResourceDescription";
import CredentialSummary from "./models/CredentialSummary";
+import DataProduct from "./models/DataProduct";
import Experiment from "./models/Experiment";
import ExperimentSummary from "./models/ExperimentSummary";
import GatewayResourceProfile from "./models/GatewayResourceProfile";
@@ -184,6 +185,15 @@ export default {
],
modelClass: CredentialSummary
},
+ DataProducts: {
+ url: "/api/data-products/",
+ viewSet: [
+ {
+ name: "retrieve"
+ }
+ ],
+ modelClass: DataProduct
+ },
Experiments: {
url: "/api/experiments/",
viewSet: [
diff --git a/django_airavata/apps/api/urls.py b/django_airavata/apps/api/urls.py
index eb81baf..064bd1c 100644
--- a/django_airavata/apps/api/urls.py
+++ b/django_airavata/apps/api/urls.py
@@ -41,6 +41,9 @@ router.register(r'storage-preferences',
views.StoragePreferenceViewSet,
base_name='storage-preference')
router.register(r'parsers', views.ParserViewSet, base_name='parser')
+router.register(r'data-products',
+ views.DataProductViewSet,
+ base_name='data-product')
app_name = 'django_airavata_api'
urlpatterns = [
diff --git a/django_airavata/apps/api/views.py b/django_airavata/apps/api/views.py
index 52a959d..0ff7e61 100644
--- a/django_airavata/apps/api/views.py
+++ b/django_airavata/apps/api/views.py
@@ -159,15 +159,7 @@ class ExperimentViewSet(APIBackedViewSet):
experiment = serializer.save(
gatewayId=self.gateway_id,
userName=self.username)
- experiment.userConfigurationData.storageId = \
- settings.GATEWAY_DATA_STORE_RESOURCE_ID
- # Set the experimentDataDir
- project = self.request.airavata_client.getProject(
- self.authz_token, experiment.projectId)
- exp_dir = datastore.get_experiment_dir(self.username,
- project.name,
- experiment.experimentName)
- experiment.userConfigurationData.experimentDataDir = exp_dir
+ self._set_storage_id_and_data_dir(experiment)
experiment_id = self.request.airavata_client.createExperiment(
self.authz_token, self.gateway_id, experiment)
experiment.experimentId = experiment_id
@@ -176,8 +168,28 @@ class ExperimentViewSet(APIBackedViewSet):
experiment = serializer.save(
gatewayId=self.gateway_id,
userName=self.username)
+ # The project or exp name may have changed, so update the exp data dir
+ self._set_storage_id_and_data_dir(experiment)
self.request.airavata_client.updateExperiment(
self.authz_token, experiment.experimentId, experiment)
+ # Process experiment._removed_input_files, removing them from storage
+ for removed_input_file in experiment._removed_input_files:
+ data_product = self.request.airavata_client.getDataProduct(
+ self.authz_token, removed_input_file)
+ datastore.delete(data_product)
+
+ def _set_storage_id_and_data_dir(self, experiment):
+ # Storage ID
+ experiment.userConfigurationData.storageId = \
+ settings.GATEWAY_DATA_STORE_RESOURCE_ID
+ # Create experiment dir and set it on model
+ project = self.request.airavata_client.getProject(
+ self.authz_token, experiment.projectId)
+ exp_dir = datastore.get_experiment_dir(self.username,
+ project.name,
+ experiment.experimentName)
+ experiment.userConfigurationData.experimentDataDir = exp_dir
+
@detail_route(methods=['post'])
def launch(self, request, experiment_id=None):
@@ -611,6 +623,17 @@ class LocalDataMovementView(APIView):
instance=data_movement).data)
+class DataProductViewSet(mixins.RetrieveModelMixin,
+ GenericAPIBackedViewSet):
+ serializer_class = serializers.DataProductSerializer
+ lookup_field = 'product_uri'
+ lookup_value_regex = '.*'
+
+ def get_instance(self, lookup_value):
+ return self.request.airavata_client.getDataProduct(
+ self.request.authz_token, lookup_value)
+
+
@login_required
def upload_input_file(request):
try:
diff --git a/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/DataProductViewer.vue b/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/DataProductViewer.vue
new file mode 100644
index 0000000..4f12b4f
--- /dev/null
+++ b/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/DataProductViewer.vue
@@ -0,0 +1,40 @@
+<template>
+
+ <span v-if="dataProduct.downloadURL">
+ <a :href="dataProduct.downloadURL">
+ <i class="fa fa-download"></i>
+ {{ filename }}
+ </a>
+ </span>
+ <span v-else>{{ filename }}</span>
+</template>
+
+<script>
+import { models } from "django-airavata-api";
+export default {
+ name: "data-product-viewer",
+ props: {
+ dataProduct: {
+ type: models.DataProduct,
+ required: true
+ },
+ inputFile: {
+ type: Boolean,
+ default: false
+ }
+ },
+ computed: {
+ filename() {
+ if (this.inputFile) {
+ // productName captures the user provided name of the file, which may
+ // not match the name of the file on the storage system (for example,
+ // because of file name collision)
+ return this.dataProduct.productName;
+ } else {
+ return this.dataProduct.filename;
+ }
+ }
+ }
+};
+</script>
+
diff --git a/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/ExperimentSummary.vue b/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/ExperimentSummary.vue
index af529f4..9b2892d 100644
--- a/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/ExperimentSummary.vue
+++ b/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/ExperimentSummary.vue
@@ -33,15 +33,7 @@
<tr>
<th scope="row">Outputs</th>
<td>
- <template v-for="output in localFullExperiment.outputDataProducts">
- <span v-if="output.downloadURL" :key="output.productUri">
- <a :href="output.downloadURL">
- <i class="fa fa-download"></i>
- {{ output.filename }}
- </a>
- </span>
- <span v-else :key="output.productUri">{{ output.filename }}</span>
- </template>
+ <data-product-viewer v-for="output in localFullExperiment.outputDataProducts" :data-product="output" class="data-product" :key="output.productUri"/>
</td>
</tr>
<!-- Going to leave this out for now -->
@@ -130,15 +122,8 @@
<tr>
<th scope="row">Inputs</th>
<td>
- <template v-for="input in localFullExperiment.inputDataProducts">
- <span v-if="input.downloadURL" :key="input.productUri">
- <a :href="input.downloadURL">
- <i class="fa fa-download"></i>
- {{ input.filename }}
- </a>
- </span>
- <span v-else :key="input.productUri">{{ input.filename }}</span>
- </template>
+ <data-product-viewer v-for="input in localFullExperiment.inputDataProducts"
+ :data-product="input" :input-file="true" class="data-product" :key="input.productUri"/>
</td>
</tr>
<tr>
@@ -157,6 +142,7 @@
<script>
import { models, services } from "django-airavata-api";
+import DataProductViewer from "./DataProductViewer.vue";
import moment from "moment";
@@ -177,7 +163,9 @@ export default {
localFullExperiment: this.fullExperiment.clone()
};
},
- components: {},
+ components: {
+ DataProductViewer,
+ },
computed: {
creationTime: function() {
return moment(this.localFullExperiment.experiment.creationTime).fromNow();
@@ -224,5 +212,8 @@ export default {
};
</script>
-<style>
+<style scoped>
+.data-product + .data-product {
+ margin-left: 0.5em;
+}
</style>
diff --git a/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/input-editors/FileInputEditor.vue b/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/input-editors/FileInputEditor.vue
index 964aa9f..9f5b496 100644
--- a/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/input-editors/FileInputEditor.vue
+++ b/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/input-editors/FileInputEditor.vue
@@ -1,14 +1,74 @@
<template>
- <b-form-file :id="id" v-model="data"
- :placeholder="experimentInput.userFriendlyDescription"
- :state="componentValidState"
- @input="valueChanged"/>
+ <div>
+ <div
+ class="row"
+ v-if="isDataProductURI && dataProduct"
+ >
+ <div class="col mr-auto">
+ <data-product-viewer :data-product="dataProduct" :input-file="true"/>
+ </div>
+ <div class="col-auto">
+ <delete-link @delete="deleteDataProduct">
+ Are you sure you want to delete input file {{ dataProduct.filename }}?
+ </delete-link>
+ </div>
+ </div>
+ <div class="row">
+ <div class="col">
+
+ <b-form-file
+ :id="id"
+ v-model="data"
+ v-if="!isDataProductURI"
+ :placeholder="experimentInput.userFriendlyDescription"
+ :state="componentValidState"
+ @input="valueChanged"
+ />
+ </div>
+ </div>
+ </div>
</template>
<script>
-import {InputEditorMixin} from 'django-airavata-workspace-plugin-api'
+import { services } from "django-airavata-api";
+import { InputEditorMixin } from "django-airavata-workspace-plugin-api";
+import DataProductViewer from "../DataProductViewer.vue";
+import { components } from "django-airavata-common-ui";
+
export default {
- name: 'file-input-editor',
- mixins: [InputEditorMixin],
-}
+ name: "file-input-editor",
+ mixins: [InputEditorMixin],
+ components: {
+ DataProductViewer,
+ "delete-link": components.DeleteLink
+ },
+ computed: {
+ isDataProductURI() {
+ // Just assume that if the value is a string then it's a data product URL
+ return this.value && typeof this.value === "string";
+ }
+ },
+ data() {
+ return {
+ dataProduct: null
+ };
+ },
+ created() {
+ if (this.isDataProductURI) {
+ this.loadDataProduct(this.value);
+ }
+ },
+ methods: {
+ loadDataProduct(dataProductURI) {
+ services.DataProductService.retrieve({ lookup: dataProductURI }).then(
+ dataProduct => (this.dataProduct = dataProduct)
+ );
+ },
+ deleteDataProduct() {
+ // Just null out the 'data' field. Backend will delete the file from storage
+ this.data = null;
+ this.valueChanged();
+ }
+ }
+};
</script>
diff --git a/django_airavata/apps/workspace/static/django_airavata_workspace/js/containers/EditExperimentContainer.vue b/django_airavata/apps/workspace/static/django_airavata_workspace/js/containers/EditExperimentContainer.vue
index a90d214..a2b9f2a 100644
--- a/django_airavata/apps/workspace/static/django_airavata_workspace/js/containers/EditExperimentContainer.vue
+++ b/django_airavata/apps/workspace/static/django_airavata_workspace/js/containers/EditExperimentContainer.vue
@@ -17,7 +17,7 @@ import ExperimentEditor from "../components/experiment/ExperimentEditor.vue";
import moment from "moment";
export default {
- name: "create-experiment-container",
+ name: "edit-experiment-container",
props: {
experimentId: {
type: String,