You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@openserverless.apache.org by ms...@apache.org on 2024/07/26 11:26:46 UTC
(openserverless-task) 05/05: feat: moved config to module. added pydoc strings
This is an automated email from the ASF dual-hosted git repository.
msciabarra pushed a commit to branch 4.0.0-testing
in repository https://gitbox.apache.org/repos/asf/openserverless-task.git
commit 984b82da42766d387e7b29492315c03810f54bcc
Author: d4rkstar <d4...@gmail.com>
AuthorDate: Tue Jul 9 01:39:10 2024 +0200
feat: moved config to module. added pydoc strings
Created a config.py module and moved there all the common defaults params. Added pydoc fucntions documentation
---
ide/deploy/client.py | 59 +++++++++++++++++++++++------------
ide/deploy/config.py | 65 +++++++++++++++++++++++++++++++++++++++
ide/deploy/deploy.py | 87 +++++++++++++++++++++++++++++++++++++++++++---------
ide/deploy/scan.py | 51 +++++++++++++++++++-----------
ide/deploy/watch.py | 39 +++++++++++++++++------
5 files changed, 239 insertions(+), 62 deletions(-)
diff --git a/ide/deploy/client.py b/ide/deploy/client.py
index 21631fd..895df97 100644
--- a/ide/deploy/client.py
+++ b/ide/deploy/client.py
@@ -15,47 +15,66 @@
# specific language governing permissions and limitations
# under the License.
-from pathlib import Path
from subprocess import Popen, PIPE
-import os, os.path, json
+import os
+import os.path
import threading
-import asyncio
-
-def get_nuvolaris_config(key, default):
- try:
- dir = os.environ.get("NUV_PWD", "/do_not_exists")
- file = f"{dir}/package.json"
- info = json.loads(Path(file).read_text())
- return info.get("nuvolaris", {}).get(key, default)
- except:
- return default
-
-def readlines(inp):
+from typing import IO
+from .config import get_nuvolaris_config
+
+
+def readlines(inp: IO[str]):
+ """Read line from an file descriptor
+
+ Args:
+ inp (IO[str]): the file descriptor
+ """
for line in iter(inp.readline, ''):
print(line, end='')
# serve web area
-def launch(key, default):
+
+
+def launch(key: str, default: (str | list)):
+ """Launch a command in a process, reading.
+ The command is extracted from nuvolaris config in package.json
+ or a default command is used
+
+ Args:
+ key (str): the key from which read the command
+ default (str | list): the default if the key is not found
+ """
cmd = get_nuvolaris_config(key, default)
proc = Popen(
- cmd, shell=True,
- cwd=os.environ.get("NUV_PWD"), env=os.environ,
+ cmd, shell=True,
+ cwd=os.environ.get("NUV_PWD"), env=os.environ,
stdin=PIPE, stdout=PIPE, stderr=PIPE, text=True
)
threading.Thread(target=readlines, args=(proc.stdout,)).start()
threading.Thread(target=readlines, args=(proc.stderr,)).start()
-def serve():
+
+def serve():
+ """Serve the web area
+ """
launch("devel", "nuv ide serve")
-def logs():
+
+def logs():
+ """Serve the openwhisk activation's logs
+ """
launch("logs", "nuv activation poll")
# build
+
+
def build():
+ """Try to build the frontend application, if the deploy command is set.
+
+ """
deploy = get_nuvolaris_config("deploy", "true")
proc = Popen(
- deploy, shell=True,
+ deploy, shell=True,
env=os.environ,
cwd=os.environ.get("NUV_PWD"),
stdin=PIPE, stdout=PIPE, stderr=PIPE, text=True
diff --git a/ide/deploy/config.py b/ide/deploy/config.py
new file mode 100644
index 0000000..e060e16
--- /dev/null
+++ b/ide/deploy/config.py
@@ -0,0 +1,65 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import os
+import json
+from pathlib import Path
+
+MAINS = ["__main__.py",
+ "index.js",
+ "index.php",
+ "main.go"]
+
+SKIPDIR = ["virtualenv",
+ "node_modules",
+ "__pycache__"]
+
+DEFAULT_REQ_GLOBS = ["packages/*/*/requirements.txt",
+ "packages/*/*/package.json",
+ "packages/*/*/composer.json",
+ "packages/*/*/go.mod"]
+
+DEFAULT_MAIN_GLOBS = ["packages/*/*/index.js",
+ "packages/*/*/__main__.py",
+ "packages/*/*/index.php",
+ "packages/*/*/main.go"]
+
+DEFAULT_SINGLES_GLOBS = ["packages/*/*.py",
+ "packages/*/*.js",
+ "packages/*/*.php",
+ "packages/*/*.go"]
+
+
+def get_nuvolaris_config(key: str, default: list | str) -> (list | str):
+ """Read package.json if exists and retrieve the required
+ value for passed key (if defined)
+
+ Args:
+ key (str): the key of the parameter to retrieve
+ default (list | str): the default value to return
+
+ Returns:
+ _type_: the list of values or the single string value.
+ if not defined, the default value is returned
+ """
+ try:
+ dir = os.environ.get("NUV_PWD", "/do_not_exists")
+ file = f"{dir}/package.json"
+ info = json.loads(Path(file).read_text())
+ return info.get("nuvolaris", {}).get(key, default)
+ except:
+ return default
diff --git a/ide/deploy/deploy.py b/ide/deploy/deploy.py
index e698a15..9baa9fc 100644
--- a/ide/deploy/deploy.py
+++ b/ide/deploy/deploy.py
@@ -15,28 +15,49 @@
# specific language governing permissions and limitations
# under the License.
-MAINS = ["__main__.py", "index.js", "index.php", "main.go"]
-
import os
-from os.path import exists, isdir
+from os.path import exists, isdir
from subprocess import Popen
+from .config import MAINS
dry_run = False
-def set_dry_run(b):
+
+def set_dry_run(b: bool):
+ """Set global dry run
+
+ Args:
+ b (bool): true for dry run enabled
+ """
global dry_run
dry_run = b
-def exec(cmd):
+
+def exec(cmd: str):
+ """Exec a shell command and wait for it to complete.
+ If dryrun is set, the command is not executed.
+
+ Args:
+ cmd (str): command line to execue
+ """
global dry_run
print("$", cmd)
if not dry_run:
Popen(cmd, shell=True, env=os.environ).wait()
-def extract_args(files):
+
+def extract_args(files: list) -> list:
+ """Extract openwhisk args from files
+
+ Args:
+ files (list): the list of files to inspect
+
+ Returns:
+ list: a list of parameters
+ """
res = []
for file in files:
- #if dry_run:
+ # if dry_run:
# print(f": inspecting {file}")
if exists(file):
with open(file, "r") as f:
@@ -47,9 +68,16 @@ def extract_args(files):
res.append(line.strip()[2:])
return res
+
package_done = set()
-def deploy_package(package):
+
+def deploy_package(package: str):
+ """Deploy a package on nuvolaris
+
+ Args:
+ package (str): the name of package
+ """
global package_done
# package args
ppath = f"packages/{package}.args"
@@ -59,15 +87,38 @@ def deploy_package(package):
exec(cmd)
package_done.add(cmd)
-def build_zip(package, action):
+
+def build_zip(package: str, action: str) -> str:
+ """Builds a zip for the package / action
+
+ Args:
+ package (str): package
+ action (str): action
+
+ Returns:
+ str: the path of the built zip file
+ """
exec(f"nuv ide util zip A={package}/{action}")
return f"packages/{package}/{action}.zip"
-def build_action(package, action):
+
+def build_action(package: str, action: str) -> str:
+ """Invoke the nuv ide util action command on package / action
+
+ Args:
+ package (_type_): _description_
+ action (_type_): _description_
+
+ Returns:
+ str: the zip with the built action
+ """
exec(f"nuv ide util action A={package}/{action}")
return f"packages/{package}/{action}.zip"
-def deploy_action(artifact):
+
+def deploy_action(artifact: str):
+ """Deploy an artifact calling nuv action update
+ """
try:
sp = artifact.split("/")
[name, typ] = sp[-1].rsplit(".", 1)
@@ -83,18 +134,26 @@ def deploy_action(artifact):
to_inspect = [f"{base}/{x}" for x in MAINS]
else:
to_inspect = [artifact]
-
+
args = " ".join(extract_args(to_inspect))
exec(f"nuv action update {package}/{name} {artifact} {args}")
+
"""
file = "packages/deploy/hello.py"
file = "packages/deploy/multi.zip"
file = "packages/deploy/multi/__main__.py"
file = "packages/deploy/multi/requirements.txt"
"""
-def deploy(file):
- #print(f"*** {file}")
+
+
+def deploy(file: str):
+ """Deploy a package on nuvolaris
+
+ Args:
+ file (str): the file to deploy
+ """
+ # print(f"*** {file}")
if isdir(file):
for start in MAINS:
sub = f"{file}/{start}"
diff --git a/ide/deploy/scan.py b/ide/deploy/scan.py
index 4b33250..0be87e7 100644
--- a/ide/deploy/scan.py
+++ b/ide/deploy/scan.py
@@ -18,22 +18,44 @@
from glob import glob
from .deploy import *
from .client import get_nuvolaris_config
+from .config import DEFAULT_REQ_GLOBS, DEFAULT_MAIN_GLOBS, DEFAULT_SINGLES_GLOBS
def scan():
+ """This function has two stages:\n
+ Scan
+ ===
+ A) Packages requirements\n
+ Check for package requirements. It will look in a specific requirement file
+ (from web package.json) or a default set of requirements (requirements.txt,
+ composer.json etc. etc).\n
+ For each requirement found, a zip is built calling the task:
+ ``nuv ide util zip A={package}/{action}``\n
+ The zip is added to actions (deployments); the directory is added to
+ packages (packages)\n
+ B) Mains for packages\n
+ After that, the scan continue checking the "mains". For each main, an
+ action is built calling the task:\n
+ ``nuv ide util action A={package}/{action}``\n
+ where package is the base directory of main file and the action is the
+ basename of the main file.\n
+ C) Singles\n
+ Finally the scan will take care of single file functions\n
+
+ Deploy
+ ===
+ Each package is deployed with the command:\n
+ ``nuv package update {package} {pargs}``\n
+ Each deployment is deployed with the command:\n
+ ``nuv action update {package}/{name} {artifact} {args}``\n
+ """
+
# first look for requirements.txt and build the venv (add in set)
deployments = set()
packages = set()
print("> Scan:")
-
- # => REQUIREMENTS
- default_reqs_globs = ["packages/*/*/requirements.txt",
- "packages/*/*/package.json",
- "packages/*/*/composer.json",
- "packages/*/*/go.mod"]
-
- package_globs = get_nuvolaris_config("requirements", default_reqs_globs)
+ package_globs = get_nuvolaris_config("requirements", DEFAULT_REQ_GLOBS)
reqs = list()
for pkg_glob in package_globs:
@@ -51,11 +73,7 @@ def scan():
packages.add(sp[1])
# => MAINS
- default_mains_globs = ["packages/*/*/index.js",
- "packages/*/*/__main__.py",
- "packages/*/*/index.php",
- "packages/*/*/main.go"]
- mains_globs = get_nuvolaris_config("mains", default_mains_globs)
+ mains_globs = get_nuvolaris_config("mains", DEFAULT_MAIN_GLOBS)
mains = list()
for main_glob in mains_globs:
items = glob(main_glob)
@@ -71,11 +89,8 @@ def scan():
packages.add(sp[1])
# => SINGLES
- default_singles_globs = ["packages/*/*.py",
- "packages/*/*.js",
- "packages/*/*.php",
- "packages/*/*.go"]
- singles_globs = get_nuvolaris_config("singles", default_singles_globs)
+
+ singles_globs = get_nuvolaris_config("singles", DEFAULT_SINGLES_GLOBS)
singles = list()
for single_glob in singles_globs:
items = glob(single_glob)
diff --git a/ide/deploy/watch.py b/ide/deploy/watch.py
index 99d4234..7559958 100644
--- a/ide/deploy/watch.py
+++ b/ide/deploy/watch.py
@@ -15,48 +15,66 @@
# specific language governing permissions and limitations
# under the License.
-SKIPDIR = ["virtualenv", "node_modules", "__pycache__"]
-
+from typing import Tuple
import watchfiles
+
import asyncio
import os.path
import signal
-
+from .config import SKIPDIR
from .deploy import deploy
from .client import serve, logs
-def check_and_deploy(change):
+
+def check_and_deploy(change: Tuple[watchfiles.Change, str]):
+ """Deploy a file if the received change is a file modification
+
+ Args:
+ change (Tuple[watchfiles.Change, str]): the change type and filename
+ """
change_type, path = change
path = os.path.abspath(path)
cur_dir_len = len(os.getcwd())+1
src = path[cur_dir_len:]
# only modified
- if change_type != watchfiles.Change.modified: return
+ if change_type != watchfiles.Change.modified:
+ return
# no directories
- if os.path.isdir(src): return
+ if os.path.isdir(src):
+ return
# no missing files
- if not os.path.exists(src): return
+ if not os.path.exists(src):
+ return
# no generated directories
for dir in src.split("/")[:-1]:
- if dir in SKIPDIR: return
+ if dir in SKIPDIR:
+ return
# no generated files
- if src.endswith(".zip"): return
+ if src.endswith(".zip"):
+ return
# now you can deploy
deploy(src)
+
async def redeploy():
+ """Install the filesystem watcher over the packages directory
+ """
print("> Watching:")
iterator = watchfiles.awatch("packages", recursive=True)
async for changes in iterator:
for change in changes:
try:
- #print(change)
+ # print(change)
check_and_deploy(change)
except Exception as e:
print(e)
+
def watch():
+ """This function will start the webserver and show logs.
+ After that it will watch the filesystem in a loop, until SIGTERM received.
+ """
# start web server
serve()
# show logs
@@ -64,6 +82,7 @@ def watch():
loop = asyncio.get_event_loop()
task = loop.create_task(redeploy())
+
def end_loop():
print("Ending task.")
task.cancel()