You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by ma...@apache.org on 2020/01/21 20:25:36 UTC
[airavata] branch develop updated: AIRAVATA-3284 Use script to
clean up old JS/CSS files
This is an automated email from the ASF dual-hosted git repository.
machristie pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/airavata.git
The following commit(s) were added to refs/heads/develop by this push:
new 3b69297 AIRAVATA-3284 Use script to clean up old JS/CSS files
new 7445c19 Merge branch 'AIRAVATA-3284--Ansible--clean-up-past-built-CSS/JS-files' into develop
3b69297 is described below
commit 3b69297284e2f3f3223bebf69bae3a366fa27c53
Author: Marcus Christie <ma...@apache.org>
AuthorDate: Tue Jan 21 14:17:34 2020 -0500
AIRAVATA-3284 Use script to clean up old JS/CSS files
---
.../roles/django/files/remove_old_js_css_files.py | 46 ++++++++++++++++++++++
dev-tools/ansible/roles/django/handlers/main.yml | 18 +++++++++
dev-tools/ansible/roles/django/tasks/main.yml | 24 ++---------
3 files changed, 67 insertions(+), 21 deletions(-)
diff --git a/dev-tools/ansible/roles/django/files/remove_old_js_css_files.py b/dev-tools/ansible/roles/django/files/remove_old_js_css_files.py
new file mode 100755
index 0000000..bed1444
--- /dev/null
+++ b/dev-tools/ansible/roles/django/files/remove_old_js_css_files.py
@@ -0,0 +1,46 @@
+#!/usr/bin/env python3
+
+import argparse
+import glob
+import json
+import os
+import re
+import time
+
+BASE_DIR = "/var/www/portals"
+IGNORE_DIRS = ['node_modules', 'venv']
+REGEX = re.compile(r"^([\w-]+)\.([a-f0-9]{8})\.(js|js\.map|css|css\.map)$")
+MAX_ATIME_AGE_HOURS = 24
+
+parser = argparse.ArgumentParser()
+parser.add_argument("-b", "--basedir", default=BASE_DIR, help="base directory from which to look for older built JS/CSS files")
+args = parser.parse_args()
+
+for root, dirs, files in os.walk(args.basedir):
+ for i in IGNORE_DIRS:
+ if i in dirs:
+ dirs.remove(i)
+ if "webpack-stats.json" in files:
+ # print(f"Found webpack-stats.json in {root}")
+ with open(os.path.join(root, "webpack-stats.json")) as sf:
+ stats = json.load(sf)
+ for chunk, files in stats["chunks"].items():
+ for f in files:
+ dirname, filename = os.path.split(f["name"])
+ # capture base name and hash and extension
+ m = REGEX.match(filename)
+ if m is not None:
+ basename, content_hash, file_ext = m.groups()
+ other_files = glob.glob(os.path.join(root, dirname, basename + ".*." + file_ext))
+ for other_file in other_files:
+ m = REGEX.match(os.path.basename(other_file))
+ other_hash = m.group(2)
+ if other_hash != content_hash:
+ # Check last accessed time and remove file if more than MAX_ATIME_AGE_HOURS old
+ atime = os.stat(other_file).st_atime
+ atime_hours = (time.time() - atime) / 3600
+ if atime_hours > MAX_ATIME_AGE_HOURS:
+ print(f"Deleting alternate of {filename} {file_ext}, {atime_hours} hours old: {other_file}")
+ os.remove(other_file)
+ else:
+ raise Exception(f"Regex failed on filename {filename} in {root}")
diff --git a/dev-tools/ansible/roles/django/handlers/main.yml b/dev-tools/ansible/roles/django/handlers/main.yml
index 808200f..237e3d2 100644
--- a/dev-tools/ansible/roles/django/handlers/main.yml
+++ b/dev-tools/ansible/roles/django/handlers/main.yml
@@ -31,3 +31,21 @@
state: touch
become: yes
become_user: "{{ user }}"
+
+# Cleaning up older built files should come after uwsgi restart, restart will
+# pick up newer webpack-stats.json files and only serve newer JS/CSS files
+- name: copy remove_old_js_css_files.py script
+ copy:
+ src: remove_old_js_css_files.py
+ dest: /usr/local/bin/remove_old_js_css_files.py
+ owner: "{{ user }}"
+ group: "{{ group }}"
+ mode: 0755
+ become: yes
+ listen: "delete older files"
+
+- name: execute remove_old_js_css_files.py script
+ command: /usr/local/bin/remove_old_js_css_files.py -b "{{ doc_root_dir }}/static"
+ listen: "delete older files"
+ become: yes
+ become_user: "{{ user }}"
diff --git a/dev-tools/ansible/roles/django/tasks/main.yml b/dev-tools/ansible/roles/django/tasks/main.yml
index 8565b65..6462576 100644
--- a/dev-tools/ansible/roles/django/tasks/main.yml
+++ b/dev-tools/ansible/roles/django/tasks/main.yml
@@ -124,6 +124,7 @@
# become_user: "{{user}}"
notify:
- restart uwsgi
+ - delete older files
- name: rsync built Django code to {{ airavata_django_checkout }}, deleting older built files
synchronize:
@@ -138,6 +139,7 @@
# become_user: "{{user}}"
notify:
- restart uwsgi
+ - delete older files
with_items: "{{ django_portal_js_build_dirs }}"
- name: Create virtual environment for Django portal and install dependencies
@@ -174,27 +176,6 @@
become: yes
become_user: "{{user}}"
-- name: Find built JS/CSS files in /static directory last accessed more than 30d ago
- find:
- paths: "{{ doc_root_dir }}/static"
- age: 30d
- age_stamp: atime
- # built JS files have a 8 character hash, e.g., "dashboard.567f67c4.js"
- patterns: ".*\\.[0-9a-f]{8}\\.js(\\.map)?,.*\\.[0-9a-f]{8}\\.css"
- use_regex: yes
- recurse: yes
- register: static_files_to_delete
- become: yes
- become_user: "{{user}}"
-
-- name: Remove older built files from /static directory
- file:
- path: "{{ item.path }}"
- state: absent
- with_items: "{{ static_files_to_delete.files }}"
- become: yes
- become_user: "{{user}}"
-
- name: Run Django's manage.py collectstatic
django_manage:
command: "collectstatic -i node_modules --noinput"
@@ -204,6 +185,7 @@
become_user: "{{user}}"
notify:
- restart uwsgi
+ - delete older files
# TODO: this is really slow with lots of files
- name: Create experiment data dir