You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by ma...@apache.org on 2018/02/08 15:43:49 UTC
[airavata-django-portal] 02/02: AIRAVATA-2599 Upload input files
before saving/launching
This is an automated email from the ASF dual-hosted git repository.
machristie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-django-portal.git
commit 3ae702a708a0115871eac3a775b842af7b1e847d
Author: Marcus Christie <ma...@iu.edu>
AuthorDate: Thu Feb 8 10:43:40 2018 -0500
AIRAVATA-2599 Upload input files before saving/launching
---
.../api/static/django_airavata_api/js/index.js | 2 +
.../django_airavata_api/js/models/DataType.js | 13 ++++
.../js/models/InputDataObjectType.js | 6 +-
.../django_airavata_api/js/utils/FetchUtils.js | 8 ++-
.../js/components/experiment/ExperimentEditor.vue | 79 +++++++++++++++-------
django_airavata/apps/workspace/urls.py | 6 +-
django_airavata/apps/workspace/views.py | 62 +++++++++++++++++
django_airavata/settings_local.py.sample | 1 +
8 files changed, 149 insertions(+), 28 deletions(-)
diff --git a/django_airavata/apps/api/static/django_airavata_api/js/index.js b/django_airavata/apps/api/static/django_airavata_api/js/index.js
index c3015df..352dec1 100644
--- a/django_airavata/apps/api/static/django_airavata_api/js/index.js
+++ b/django_airavata/apps/api/static/django_airavata_api/js/index.js
@@ -1,6 +1,7 @@
import ApplicationInterfaceDefinition from './models/ApplicationInterfaceDefinition'
import ApplicationModule from './models/ApplicationModule'
+import DataType from './models/DataType'
import Experiment from './models/Experiment'
import InputDataObjectType from './models/InputDataObjectType'
import OutputDataObjectType from './models/OutputDataObjectType'
@@ -23,6 +24,7 @@ import PaginationIterator from './utils/PaginationIterator'
exports.models = {
ApplicationInterfaceDefinition,
ApplicationModule,
+ DataType,
Experiment,
FullExperiment,
InputDataObjectType,
diff --git a/django_airavata/apps/api/static/django_airavata_api/js/models/DataType.js b/django_airavata/apps/api/static/django_airavata_api/js/models/DataType.js
new file mode 100644
index 0000000..f8c229d
--- /dev/null
+++ b/django_airavata/apps/api/static/django_airavata_api/js/models/DataType.js
@@ -0,0 +1,13 @@
+import BaseEnum from './BaseEnum'
+
+export default class DataType extends BaseEnum {
+}
+DataType.init([
+ "STRING",
+ "INTEGER",
+ "FLOAT",
+ "URI",
+ "URI_COLLECTION",
+ "STDOUT",
+ "STDERR",
+]);
diff --git a/django_airavata/apps/api/static/django_airavata_api/js/models/InputDataObjectType.js b/django_airavata/apps/api/static/django_airavata_api/js/models/InputDataObjectType.js
index 5969555..dfe2846 100644
--- a/django_airavata/apps/api/static/django_airavata_api/js/models/InputDataObjectType.js
+++ b/django_airavata/apps/api/static/django_airavata_api/js/models/InputDataObjectType.js
@@ -1,10 +1,14 @@
import BaseModel from './BaseModel';
+import DataType from './DataType'
const FIELDS = [
'name',
'value',
- 'type',
+ {
+ name: 'type',
+ type: DataType,
+ },
'applicationArgument',
'standardInput',
'userFriendlyDescription',
diff --git a/django_airavata/apps/api/static/django_airavata_api/js/utils/FetchUtils.js b/django_airavata/apps/api/static/django_airavata_api/js/utils/FetchUtils.js
index 074e4ea..58027f0 100644
--- a/django_airavata/apps/api/static/django_airavata_api/js/utils/FetchUtils.js
+++ b/django_airavata/apps/api/static/django_airavata_api/js/utils/FetchUtils.js
@@ -21,9 +21,13 @@ export default {
},
post: function (url, body, mediaType = "application/json") {
var headers = this.createHeaders(mediaType)
+ // Browsers automatically handle content type for FormData request bodies
+ if (body instanceof FormData) {
+ headers.delete("Content-Type");
+ }
return fetch(url, {
method: 'post',
- body: typeof body !== 'string' ? JSON.stringify(body) : body,
+ body: body,
headers: headers,
credentials: "same-origin"
}).then((response) => {
@@ -42,7 +46,7 @@ export default {
var headers = this.createHeaders(mediaType)
return fetch(url, {
method: 'put',
- body: typeof body !== 'string' ? JSON.stringify(body) : body,
+ body: body,
headers: headers,
credentials: "same-origin"
}).then((response) => {
diff --git a/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/ExperimentEditor.vue b/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/ExperimentEditor.vue
index d0f21ad..954781b 100644
--- a/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/ExperimentEditor.vue
+++ b/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/ExperimentEditor.vue
@@ -54,9 +54,12 @@
:label="experimentInput.name" :label-for="experimentInput.name" :key="experimentInput.name"
:feedback="getValidationFeedback(['experimentInputs', experimentInput.name, 'value'])"
:state="getValidationState(['experimentInputs', experimentInput.name, 'value'])">
- <b-form-input :id="experimentInput.name" type="text" v-model="experimentInput.value" required
+ <b-form-input v-if="isSimpleInput(experimentInput)" :id="experimentInput.name" type="text" v-model="experimentInput.value" required
:placeholder="experimentInput.userFriendlyDescription"
:state="getValidationState(['experimentInputs', experimentInput.name, 'value'])"></b-form-input>
+ <b-form-file v-if="isFileInput(experimentInput)" :id="experimentInput.name" type="text" v-model="experimentInput.value" required
+ :placeholder="experimentInput.userFriendlyDescription"
+ :state="getValidationState(['experimentInputs', experimentInput.name, 'value'])"></b-form-file>
</b-form-group>
</div>
</div>
@@ -94,7 +97,7 @@
<script>
import ComputationalResourceSchedulingEditor from './ComputationalResourceSchedulingEditor.vue'
-import {models, services} from 'django-airavata-api'
+import {models, services, utils as apiUtils} from 'django-airavata-api'
import {utils} from 'django-airavata-common-ui'
export default {
@@ -140,34 +143,52 @@ export default {
},
methods: {
saveExperiment: function() {
- console.log(JSON.stringify(this.localExperiment));
- // TODO: validate experiment
- // save experiment
- services.ExperimentService.save(this.localExperiment)
- .then(experiment => {
- this.localExperiment = experiment;
- console.log(experiment);
- alert('Experiment saved!');
- this.$emit('saved', experiment);
+ return this.uploadInputFiles()
+ .then(uploadResults => {
+ return services.ExperimentService.save(this.localExperiment)
+ .then(experiment => {
+ this.localExperiment = experiment;
+ console.log(experiment);
+ alert('Experiment saved!');
+ this.$emit('saved', experiment);
+ });
+ })
+ .catch(result => {
+ console.log("Save failed!", result);
});
},
saveAndLaunchExperiment: function() {
- console.log(JSON.stringify(this.localExperiment));
- // TODO: validate experiment
- let savedExperiment = null;
- services.ExperimentService.save(this.localExperiment)
- .then(experiment => {
- this.localExperiment = experiment;
- return services.ExperimentService.launch(experiment.experimentId)
- .then(result => {
- alert('Experiment launched!');
- this.$emit('savedAndLaunched', experiment);
- });
- })
+ return this.uploadInputFiles()
+ .then(uploadResults => {
+ return services.ExperimentService.save(this.localExperiment)
+ .then(experiment => {
+ this.localExperiment = experiment;
+ return services.ExperimentService.launch(experiment.experimentId)
+ .then(result => {
+ alert('Experiment launched!');
+ this.$emit('savedAndLaunched', experiment);
+ });
+ })
+ })
.catch(result => {
console.log("Launch failed!", result);
});
},
+ uploadInputFiles: function() {
+ let uploads = [];
+ this.localExperiment.experimentInputs.forEach(input => {
+ if (input.type === models.DataType.URI && input.value) {
+ let data = new FormData();
+ data.append('file', input.value);
+ data.append('project-id', this.localExperiment.projectId);
+ data.append('experiment-name', this.localExperiment.experimentName);
+ let uploadRequest = apiUtils.FetchUtils.post('/workspace/upload', data)
+ .then(result => input.value = result['data-product-uri'])
+ uploads.push(uploadRequest);
+ }
+ });
+ return Promise.all(uploads);
+ },
getApplicationInputState: function(applicationInput) {
const validation = this.getApplicationInputValidation(applicationInput);
return validation !== null ? 'invalid' : null;
@@ -189,6 +210,18 @@ export default {
getValidationState: function(properties) {
return this.getValidationFeedback(properties) ? 'invalid' : null;
},
+ isSimpleInput: function(experimentInput) {
+ return [
+ models.DataType.STRING,
+ models.DataType.FLOAT,
+ models.DataType.INTEGER,
+ ].indexOf(experimentInput.type) >= 0;
+ },
+ isFileInput: function(experimentInput) {
+ return [
+ models.DataType.URI,
+ ].indexOf(experimentInput.type) >= 0;
+ },
},
watch: {
experiment: function(newValue) {
diff --git a/django_airavata/apps/workspace/urls.py b/django_airavata/apps/workspace/urls.py
index a3aabc1..b6a0f31 100644
--- a/django_airavata/apps/workspace/urls.py
+++ b/django_airavata/apps/workspace/urls.py
@@ -9,6 +9,8 @@ urlpatterns = [
url(r'^experiments/(?P<experiment_id>[^/]+)/$', views.view_experiment,
name='view_experiment'),
url(r'^experiments$', views.experiments_list, name='experiments'),
- url(r'^applications/(?P<app_module_id>[^/]+)/create_experiment$', views.create_experiment, name='create_experiment'),
+ url(r'^applications/(?P<app_module_id>[^/]+)/create_experiment$',
+ views.create_experiment, name='create_experiment'),
url(r'^dashboard$', views.dashboard, name='dashboard'),
-]
\ No newline at end of file
+ url(r'^upload$', views.upload_input_file, name='upload_input_file'),
+]
diff --git a/django_airavata/apps/workspace/views.py b/django_airavata/apps/workspace/views.py
index fd79970..c121a0f 100644
--- a/django_airavata/apps/workspace/views.py
+++ b/django_airavata/apps/workspace/views.py
@@ -1,11 +1,21 @@
import json
import logging
+import os
+from urllib.parse import urlparse
+from django.conf import settings
from django.contrib.auth.decorators import login_required
+from django.core.files.storage import FileSystemStorage
+from django.http import JsonResponse
from django.shortcuts import render
from rest_framework.renderers import JSONRenderer
+from airavata.model.data.replica.ttypes import (DataProductModel,
+ DataProductType,
+ DataReplicaLocationModel,
+ ReplicaLocationCategory,
+ ReplicaPersistentType)
from django_airavata.apps.api.views import (ExperimentSearchViewSet,
FullExperimentViewSet,
ProjectViewSet)
@@ -64,3 +74,55 @@ def view_experiment(request, experiment_id):
'full_experiment_data': full_experiment_json,
'launching': json.dumps(launching),
})
+
+
+experiment_data_storage = FileSystemStorage(
+ location=settings.GATEWAY_DATA_STORE_DIR)
+
+
+@login_required
+def upload_input_file(request):
+ try:
+ # Save input file to username/project name/exp name/filename
+ username = request.user.username
+ project_id = request.POST['project-id']
+ project = request.airavata_client.getProject(
+ request.authz_token, project_id)
+ exp_name = request.POST['experiment-name']
+ input_file = request.FILES['file']
+ exp_dir = os.path.join(
+ experiment_data_storage.get_valid_name(username),
+ experiment_data_storage.get_valid_name(project.name),
+ experiment_data_storage.get_valid_name(exp_name))
+ file_path = os.path.join(
+ exp_dir,
+ experiment_data_storage.get_valid_name(input_file.name))
+ input_file_name = experiment_data_storage.save(file_path, input_file)
+ input_file_fullpath = experiment_data_storage.path(input_file_name)
+ # Register DataProductModel with DataReplicaLocationModel
+ data_product = DataProductModel()
+ data_product.gatewayId = settings.GATEWAY_ID
+ data_product.ownerName = username
+ data_product.productName = input_file.name
+ data_product.dataProductType = DataProductType.FILE
+ data_replica_location = DataReplicaLocationModel()
+ data_replica_location.storageResourceId = \
+ settings.GATEWAY_DATA_STORE_RESOURCE_ID
+ data_replica_location.replicaName = \
+ "{} gateway data store copy".format(input_file.name)
+ data_replica_location.replicaLocationCategory = \
+ ReplicaLocationCategory.GATEWAY_DATA_STORE
+ data_replica_location.replicaPersistentType = \
+ ReplicaPersistentType.TRANSIENT
+ hostname = urlparse(request.build_absolute_uri()).hostname
+ data_replica_location.filePath = \
+ "file://{}:{}".format(hostname, input_file_fullpath)
+ data_product.replicaLocations = [data_replica_location]
+ data_product_uri = request.airavata_client.registerDataProduct(
+ request.authz_token, data_product)
+ return JsonResponse({'uploaded': True,
+ 'data-product-uri': data_product_uri})
+ except Exception as e:
+ resp = JsonResponse({'uploaded': False, 'error': str(e)})
+ resp.status_code = 500
+ return resp
diff --git a/django_airavata/settings_local.py.sample b/django_airavata/settings_local.py.sample
index e40eb04..1a1a0a2 100644
--- a/django_airavata/settings_local.py.sample
+++ b/django_airavata/settings_local.py.sample
@@ -33,6 +33,7 @@ AIRAVATA_API_HOST = 'localhost'
AIRAVATA_API_PORT = 8930
AIRAVATA_API_SECURE = False
GATEWAY_DATA_STORE_RESOURCE_ID = '...'
+GATEWAY_DATA_STORE_DIR = '...'
# Profile Service Configuration
PROFILE_SERVICE_HOST = AIRAVATA_API_HOST
--
To stop receiving notification emails like this one, please contact
machristie@apache.org.