You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by ma...@apache.org on 2021/11/19 22:40:39 UTC
[airavata-django-portal-sdk] 01/03: AIRAVATA-3542 include and exclude patterns for multi experiment download
This is an automated email from the ASF dual-hosted git repository.
machristie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-django-portal-sdk.git
commit 5a658fbac19cc623caf713664e7ad3b4e75f6455
Author: Marcus Christie <ma...@apache.org>
AuthorDate: Fri Nov 19 15:37:45 2021 -0500
AIRAVATA-3542 include and exclude patterns for multi experiment download
---
airavata_django_portal_sdk/serializers.py | 13 ++++++++---
airavata_django_portal_sdk/views.py | 37 ++++++++++++++++++++++++-------
2 files changed, 39 insertions(+), 11 deletions(-)
diff --git a/airavata_django_portal_sdk/serializers.py b/airavata_django_portal_sdk/serializers.py
index 2fc09b5..e86d010 100644
--- a/airavata_django_portal_sdk/serializers.py
+++ b/airavata_django_portal_sdk/serializers.py
@@ -1,12 +1,19 @@
from rest_framework import serializers
-# class DownloadIncludeSerializer(serializers.Serializer):
-# pattern = serializers.CharField()
+
+class FilenamePatternSerializer(serializers.Serializer):
+ pattern = serializers.CharField()
+
+
+class IncludeFilenamePatternSerializer(FilenamePatternSerializer):
+ rename = serializers.CharField(required=False)
class ExperimentDownloadSerializer(serializers.Serializer):
experiment_id = serializers.CharField()
- # includes = DownloadIncludeSerializer(many=True)
+ path = serializers.CharField(default="")
+ includes = IncludeFilenamePatternSerializer(many=True, required=False, default=None)
+ excludes = FilenamePatternSerializer(many=True, required=False, default=None)
class MultiExperimentDownloadSerializer(serializers.Serializer):
diff --git a/airavata_django_portal_sdk/views.py b/airavata_django_portal_sdk/views.py
index 0592c2d..4c697b6 100644
--- a/airavata_django_portal_sdk/views.py
+++ b/airavata_django_portal_sdk/views.py
@@ -1,3 +1,4 @@
+import fnmatch
import logging
import os
import tempfile
@@ -121,9 +122,12 @@ def download_experiments(request, download_id=None):
for experiment in experiments:
experiment_id = experiment['experiment_id']
# Load experiment to make sure user has access to experiment
- experiment = request.airavata_client.getExperiment(request.authz_token, experiment_id)
- _add_experiment_directory_to_zipfile(request, zf, experiment_id, path="",
- zipfile_prefix=get_valid_filename(experiment.experimentName))
+ experiment_model = request.airavata_client.getExperiment(request.authz_token, experiment_id)
+ path = experiment['path']
+ _add_experiment_directory_to_zipfile(
+ request, zf, experiment_id, path,
+ zipfile_prefix=os.path.join(get_valid_filename(experiment_model.experimentName), path),
+ includes=experiment['includes'], excludes=experiment['excludes'])
filename = "experiments.zip"
fp.seek(0)
@@ -146,14 +150,31 @@ def _add_directory_to_zipfile(request, zf, path, directory=""):
_add_directory_to_zipfile(request, zf, path, os.path.join(directory, d['name']))
-def _add_experiment_directory_to_zipfile(request, zf, experiment_id, path, directory="", zipfile_prefix=""):
+def _add_experiment_directory_to_zipfile(request, zf, experiment_id, path, directory="", zipfile_prefix="", includes=None, excludes=None):
directories, files = user_storage.list_experiment_dir(request, experiment_id, os.path.join(path, directory))
for file in files:
- o = user_storage.open_file(request, data_product_uri=file['data-product-uri'])
- zf.writestr(os.path.join(zipfile_prefix, directory, file['name']), o.read())
- if os.path.getsize(zf.filename) > MAX_DOWNLOAD_ZIPFILE_SIZE:
- raise Exception(f"Zip file size exceeds max of {MAX_DOWNLOAD_ZIPFILE_SIZE} bytes")
+ matches = _matches_filters(file['name'], includes=includes, excludes=excludes)
+ if matches:
+ o = user_storage.open_file(request, data_product_uri=file['data-product-uri'])
+ zf.writestr(os.path.join(zipfile_prefix, directory, file['name']), o.read())
+ if os.path.getsize(zf.filename) > MAX_DOWNLOAD_ZIPFILE_SIZE:
+ raise Exception(f"Zip file size exceeds max of {MAX_DOWNLOAD_ZIPFILE_SIZE} bytes")
for d in directories:
_add_experiment_directory_to_zipfile(request, zf, experiment_id, path,
directory=os.path.join(directory, d['name']),
zipfile_prefix=zipfile_prefix)
+
+
+def _matches_filters(filename, includes=None, excludes=None):
+ # excludes take precedence
+ if excludes is not None and len(excludes) > 0:
+ for exclude in excludes:
+ if fnmatch.fnmatch(filename, exclude['pattern']):
+ return False
+ # if there are no include patterns, default to include all
+ if includes is None or len(includes) == 0:
+ return True
+ for include in includes:
+ if fnmatch.fnmatch(filename, include['pattern']):
+ return True
+ return False