You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@openserverless.apache.org by ms...@apache.org on 2024/07/26 11:45:55 UTC
(openserverless-task) branch 0.1.0-testing created (now 984b82d)
This is an automated email from the ASF dual-hosted git repository.
msciabarra pushed a change to branch 0.1.0-testing
in repository https://gitbox.apache.org/repos/asf/openserverless-task.git
at 984b82d feat: moved config to module. added pydoc strings
This branch includes the following new commits:
new 6c897c9 nuvolaris config from package.json
new 29c9e67 Merge branch 'nuvolaris:3.0.0-testing' into 3.0.0-testing
new 5cee7d8 Merge branch 'nuvolaris:3.0.0-testing' into 3.0.0-testing
new db177b4 Merge remote-tracking branch 'upstream/3.0.0-testing' into 3.0.0-testing
new 984b82d feat: moved config to module. added pydoc strings
The 5 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(openserverless-task) 04/05: Merge remote-tracking branch 'upstream/3.0.0-testing' into 3.0.0-testing
Posted by ms...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
msciabarra pushed a commit to branch 0.1.0-testing
in repository https://gitbox.apache.org/repos/asf/openserverless-task.git
commit db177b470d33b12893dd8d3910f1ce1ff4de6be2
Merge: 5cee7d8 b49f497
Author: d4rkstar <d4...@gmail.com>
AuthorDate: Mon Jul 8 23:41:49 2024 +0200
Merge remote-tracking branch 'upstream/3.0.0-testing' into 3.0.0-testing
ide/deploy/deploy.py | 2 +-
ide/deploy/scan.py | 12 +++--
ide/go/exec.env | 1 +
ide/go/launcher.go | 142 +++++++++++++++++++++++++++++++++++++++++++++++++++
ide/go/nuvfile.yml | 52 +++++++++++++++++++
ide/nuvfile.yml | 3 ++
ide/nuvopts.txt | 1 +
ide/util/nuvfile.yml | 6 +++
8 files changed, 214 insertions(+), 5 deletions(-)
(openserverless-task) 02/05: Merge branch 'nuvolaris:3.0.0-testing' into 3.0.0-testing
Posted by ms...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
msciabarra pushed a commit to branch 0.1.0-testing
in repository https://gitbox.apache.org/repos/asf/openserverless-task.git
commit 29c9e67754b3f729f9b75089b02b5f5dc8d9ab94
Merge: 6c897c9 4921deb
Author: d4rkstar <d4...@gmail.com>
AuthorDate: Mon May 27 01:37:45 2024 +0200
Merge branch 'nuvolaris:3.0.0-testing' into 3.0.0-testing
nuvroot.json | 2 +-
runtimes.json | 192 ++++++++++++++++++++++++++++++++++++++++++++--------------
2 files changed, 146 insertions(+), 48 deletions(-)
(openserverless-task) 05/05: feat: moved config to module. added pydoc strings
Posted by ms...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
msciabarra pushed a commit to branch 0.1.0-testing
in repository https://gitbox.apache.org/repos/asf/openserverless-task.git
commit 984b82da42766d387e7b29492315c03810f54bcc
Author: d4rkstar <d4...@gmail.com>
AuthorDate: Tue Jul 9 01:39:10 2024 +0200
feat: moved config to module. added pydoc strings
Created a config.py module and moved there all the common defaults params. Added pydoc fucntions documentation
---
ide/deploy/client.py | 59 +++++++++++++++++++++++------------
ide/deploy/config.py | 65 +++++++++++++++++++++++++++++++++++++++
ide/deploy/deploy.py | 87 +++++++++++++++++++++++++++++++++++++++++++---------
ide/deploy/scan.py | 51 +++++++++++++++++++-----------
ide/deploy/watch.py | 39 +++++++++++++++++------
5 files changed, 239 insertions(+), 62 deletions(-)
diff --git a/ide/deploy/client.py b/ide/deploy/client.py
index 21631fd..895df97 100644
--- a/ide/deploy/client.py
+++ b/ide/deploy/client.py
@@ -15,47 +15,66 @@
# specific language governing permissions and limitations
# under the License.
-from pathlib import Path
from subprocess import Popen, PIPE
-import os, os.path, json
+import os
+import os.path
import threading
-import asyncio
-
-def get_nuvolaris_config(key, default):
- try:
- dir = os.environ.get("NUV_PWD", "/do_not_exists")
- file = f"{dir}/package.json"
- info = json.loads(Path(file).read_text())
- return info.get("nuvolaris", {}).get(key, default)
- except:
- return default
-
-def readlines(inp):
+from typing import IO
+from .config import get_nuvolaris_config
+
+
+def readlines(inp: IO[str]):
+ """Read line from an file descriptor
+
+ Args:
+ inp (IO[str]): the file descriptor
+ """
for line in iter(inp.readline, ''):
print(line, end='')
# serve web area
-def launch(key, default):
+
+
+def launch(key: str, default: (str | list)):
+ """Launch a command in a process, reading.
+ The command is extracted from nuvolaris config in package.json
+ or a default command is used
+
+ Args:
+ key (str): the key from which read the command
+ default (str | list): the default if the key is not found
+ """
cmd = get_nuvolaris_config(key, default)
proc = Popen(
- cmd, shell=True,
- cwd=os.environ.get("NUV_PWD"), env=os.environ,
+ cmd, shell=True,
+ cwd=os.environ.get("NUV_PWD"), env=os.environ,
stdin=PIPE, stdout=PIPE, stderr=PIPE, text=True
)
threading.Thread(target=readlines, args=(proc.stdout,)).start()
threading.Thread(target=readlines, args=(proc.stderr,)).start()
-def serve():
+
+def serve():
+ """Serve the web area
+ """
launch("devel", "nuv ide serve")
-def logs():
+
+def logs():
+ """Serve the openwhisk activation's logs
+ """
launch("logs", "nuv activation poll")
# build
+
+
def build():
+ """Try to build the frontend application, if the deploy command is set.
+
+ """
deploy = get_nuvolaris_config("deploy", "true")
proc = Popen(
- deploy, shell=True,
+ deploy, shell=True,
env=os.environ,
cwd=os.environ.get("NUV_PWD"),
stdin=PIPE, stdout=PIPE, stderr=PIPE, text=True
diff --git a/ide/deploy/config.py b/ide/deploy/config.py
new file mode 100644
index 0000000..e060e16
--- /dev/null
+++ b/ide/deploy/config.py
@@ -0,0 +1,65 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import os
+import json
+from pathlib import Path
+
+MAINS = ["__main__.py",
+ "index.js",
+ "index.php",
+ "main.go"]
+
+SKIPDIR = ["virtualenv",
+ "node_modules",
+ "__pycache__"]
+
+DEFAULT_REQ_GLOBS = ["packages/*/*/requirements.txt",
+ "packages/*/*/package.json",
+ "packages/*/*/composer.json",
+ "packages/*/*/go.mod"]
+
+DEFAULT_MAIN_GLOBS = ["packages/*/*/index.js",
+ "packages/*/*/__main__.py",
+ "packages/*/*/index.php",
+ "packages/*/*/main.go"]
+
+DEFAULT_SINGLES_GLOBS = ["packages/*/*.py",
+ "packages/*/*.js",
+ "packages/*/*.php",
+ "packages/*/*.go"]
+
+
+def get_nuvolaris_config(key: str, default: list | str) -> (list | str):
+ """Read package.json if exists and retrieve the required
+ value for passed key (if defined)
+
+ Args:
+ key (str): the key of the parameter to retrieve
+ default (list | str): the default value to return
+
+ Returns:
+ _type_: the list of values or the single string value.
+ if not defined, the default value is returned
+ """
+ try:
+ dir = os.environ.get("NUV_PWD", "/do_not_exists")
+ file = f"{dir}/package.json"
+ info = json.loads(Path(file).read_text())
+ return info.get("nuvolaris", {}).get(key, default)
+ except:
+ return default
diff --git a/ide/deploy/deploy.py b/ide/deploy/deploy.py
index e698a15..9baa9fc 100644
--- a/ide/deploy/deploy.py
+++ b/ide/deploy/deploy.py
@@ -15,28 +15,49 @@
# specific language governing permissions and limitations
# under the License.
-MAINS = ["__main__.py", "index.js", "index.php", "main.go"]
-
import os
-from os.path import exists, isdir
+from os.path import exists, isdir
from subprocess import Popen
+from .config import MAINS
dry_run = False
-def set_dry_run(b):
+
+def set_dry_run(b: bool):
+ """Set global dry run
+
+ Args:
+ b (bool): true for dry run enabled
+ """
global dry_run
dry_run = b
-def exec(cmd):
+
+def exec(cmd: str):
+ """Exec a shell command and wait for it to complete.
+ If dryrun is set, the command is not executed.
+
+ Args:
+ cmd (str): command line to execue
+ """
global dry_run
print("$", cmd)
if not dry_run:
Popen(cmd, shell=True, env=os.environ).wait()
-def extract_args(files):
+
+def extract_args(files: list) -> list:
+ """Extract openwhisk args from files
+
+ Args:
+ files (list): the list of files to inspect
+
+ Returns:
+ list: a list of parameters
+ """
res = []
for file in files:
- #if dry_run:
+ # if dry_run:
# print(f": inspecting {file}")
if exists(file):
with open(file, "r") as f:
@@ -47,9 +68,16 @@ def extract_args(files):
res.append(line.strip()[2:])
return res
+
package_done = set()
-def deploy_package(package):
+
+def deploy_package(package: str):
+ """Deploy a package on nuvolaris
+
+ Args:
+ package (str): the name of package
+ """
global package_done
# package args
ppath = f"packages/{package}.args"
@@ -59,15 +87,38 @@ def deploy_package(package):
exec(cmd)
package_done.add(cmd)
-def build_zip(package, action):
+
+def build_zip(package: str, action: str) -> str:
+ """Builds a zip for the package / action
+
+ Args:
+ package (str): package
+ action (str): action
+
+ Returns:
+ str: the path of the built zip file
+ """
exec(f"nuv ide util zip A={package}/{action}")
return f"packages/{package}/{action}.zip"
-def build_action(package, action):
+
+def build_action(package: str, action: str) -> str:
+ """Invoke the nuv ide util action command on package / action
+
+ Args:
+ package (_type_): _description_
+ action (_type_): _description_
+
+ Returns:
+ str: the zip with the built action
+ """
exec(f"nuv ide util action A={package}/{action}")
return f"packages/{package}/{action}.zip"
-def deploy_action(artifact):
+
+def deploy_action(artifact: str):
+ """Deploy an artifact calling nuv action update
+ """
try:
sp = artifact.split("/")
[name, typ] = sp[-1].rsplit(".", 1)
@@ -83,18 +134,26 @@ def deploy_action(artifact):
to_inspect = [f"{base}/{x}" for x in MAINS]
else:
to_inspect = [artifact]
-
+
args = " ".join(extract_args(to_inspect))
exec(f"nuv action update {package}/{name} {artifact} {args}")
+
"""
file = "packages/deploy/hello.py"
file = "packages/deploy/multi.zip"
file = "packages/deploy/multi/__main__.py"
file = "packages/deploy/multi/requirements.txt"
"""
-def deploy(file):
- #print(f"*** {file}")
+
+
+def deploy(file: str):
+ """Deploy a package on nuvolaris
+
+ Args:
+ file (str): the file to deploy
+ """
+ # print(f"*** {file}")
if isdir(file):
for start in MAINS:
sub = f"{file}/{start}"
diff --git a/ide/deploy/scan.py b/ide/deploy/scan.py
index 4b33250..0be87e7 100644
--- a/ide/deploy/scan.py
+++ b/ide/deploy/scan.py
@@ -18,22 +18,44 @@
from glob import glob
from .deploy import *
from .client import get_nuvolaris_config
+from .config import DEFAULT_REQ_GLOBS, DEFAULT_MAIN_GLOBS, DEFAULT_SINGLES_GLOBS
def scan():
+ """This function has two stages:\n
+ Scan
+ ===
+ A) Packages requirements\n
+ Check for package requirements. It will look in a specific requirement file
+ (from web package.json) or a default set of requirements (requirements.txt,
+ composer.json etc. etc).\n
+ For each requirement found, a zip is built calling the task:
+ ``nuv ide util zip A={package}/{action}``\n
+ The zip is added to actions (deployments); the directory is added to
+ packages (packages)\n
+ B) Mains for packages\n
+ After that, the scan continue checking the "mains". For each main, an
+ action is built calling the task:\n
+ ``nuv ide util action A={package}/{action}``\n
+ where package is the base directory of main file and the action is the
+ basename of the main file.\n
+ C) Singles\n
+ Finally the scan will take care of single file functions\n
+
+ Deploy
+ ===
+ Each package is deployed with the command:\n
+ ``nuv package update {package} {pargs}``\n
+ Each deployment is deployed with the command:\n
+ ``nuv action update {package}/{name} {artifact} {args}``\n
+ """
+
# first look for requirements.txt and build the venv (add in set)
deployments = set()
packages = set()
print("> Scan:")
-
- # => REQUIREMENTS
- default_reqs_globs = ["packages/*/*/requirements.txt",
- "packages/*/*/package.json",
- "packages/*/*/composer.json",
- "packages/*/*/go.mod"]
-
- package_globs = get_nuvolaris_config("requirements", default_reqs_globs)
+ package_globs = get_nuvolaris_config("requirements", DEFAULT_REQ_GLOBS)
reqs = list()
for pkg_glob in package_globs:
@@ -51,11 +73,7 @@ def scan():
packages.add(sp[1])
# => MAINS
- default_mains_globs = ["packages/*/*/index.js",
- "packages/*/*/__main__.py",
- "packages/*/*/index.php",
- "packages/*/*/main.go"]
- mains_globs = get_nuvolaris_config("mains", default_mains_globs)
+ mains_globs = get_nuvolaris_config("mains", DEFAULT_MAIN_GLOBS)
mains = list()
for main_glob in mains_globs:
items = glob(main_glob)
@@ -71,11 +89,8 @@ def scan():
packages.add(sp[1])
# => SINGLES
- default_singles_globs = ["packages/*/*.py",
- "packages/*/*.js",
- "packages/*/*.php",
- "packages/*/*.go"]
- singles_globs = get_nuvolaris_config("singles", default_singles_globs)
+
+ singles_globs = get_nuvolaris_config("singles", DEFAULT_SINGLES_GLOBS)
singles = list()
for single_glob in singles_globs:
items = glob(single_glob)
diff --git a/ide/deploy/watch.py b/ide/deploy/watch.py
index 99d4234..7559958 100644
--- a/ide/deploy/watch.py
+++ b/ide/deploy/watch.py
@@ -15,48 +15,66 @@
# specific language governing permissions and limitations
# under the License.
-SKIPDIR = ["virtualenv", "node_modules", "__pycache__"]
-
+from typing import Tuple
import watchfiles
+
import asyncio
import os.path
import signal
-
+from .config import SKIPDIR
from .deploy import deploy
from .client import serve, logs
-def check_and_deploy(change):
+
+def check_and_deploy(change: Tuple[watchfiles.Change, str]):
+ """Deploy a file if the received change is a file modification
+
+ Args:
+ change (Tuple[watchfiles.Change, str]): the change type and filename
+ """
change_type, path = change
path = os.path.abspath(path)
cur_dir_len = len(os.getcwd())+1
src = path[cur_dir_len:]
# only modified
- if change_type != watchfiles.Change.modified: return
+ if change_type != watchfiles.Change.modified:
+ return
# no directories
- if os.path.isdir(src): return
+ if os.path.isdir(src):
+ return
# no missing files
- if not os.path.exists(src): return
+ if not os.path.exists(src):
+ return
# no generated directories
for dir in src.split("/")[:-1]:
- if dir in SKIPDIR: return
+ if dir in SKIPDIR:
+ return
# no generated files
- if src.endswith(".zip"): return
+ if src.endswith(".zip"):
+ return
# now you can deploy
deploy(src)
+
async def redeploy():
+ """Install the filesystem watcher over the packages directory
+ """
print("> Watching:")
iterator = watchfiles.awatch("packages", recursive=True)
async for changes in iterator:
for change in changes:
try:
- #print(change)
+ # print(change)
check_and_deploy(change)
except Exception as e:
print(e)
+
def watch():
+ """This function will start the webserver and show logs.
+ After that it will watch the filesystem in a loop, until SIGTERM received.
+ """
# start web server
serve()
# show logs
@@ -64,6 +82,7 @@ def watch():
loop = asyncio.get_event_loop()
task = loop.create_task(redeploy())
+
def end_loop():
print("Ending task.")
task.cancel()
(openserverless-task) 03/05: Merge branch 'nuvolaris:3.0.0-testing' into 3.0.0-testing
Posted by ms...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
msciabarra pushed a commit to branch 0.1.0-testing
in repository https://gitbox.apache.org/repos/asf/openserverless-task.git
commit 5cee7d80a91b11f32effe6343c8b6cfb04f63b93
Merge: 29c9e67 9d3cb50
Author: d4rkstar <d4...@gmail.com>
AuthorDate: Tue Jun 25 13:11:49 2024 +0200
Merge branch 'nuvolaris:3.0.0-testing' into 3.0.0-testing
admin/nuvfile.yml | 20 ++++++++++----------
admin/nuvopts.txt | 4 ++--
admin/user-crd.yaml | 8 ++++----
web/nuvfile.yml | 7 +++++++
web/upload.js | 9 ++++-----
5 files changed, 27 insertions(+), 21 deletions(-)
(openserverless-task) 01/05: nuvolaris config from package.json
Posted by ms...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
msciabarra pushed a commit to branch 0.1.0-testing
in repository https://gitbox.apache.org/repos/asf/openserverless-task.git
commit 6c897c9bca038117c1657176dd924781057e4c8d
Author: d4rkstar <d4...@gmail.com>
AuthorDate: Mon May 27 01:32:55 2024 +0200
nuvolaris config from package.json
---
ide/deploy/scan.py | 42 +++++++++++++++++++++++++++++++++++-------
1 file changed, 35 insertions(+), 7 deletions(-)
diff --git a/ide/deploy/scan.py b/ide/deploy/scan.py
index bb2205a..5382628 100644
--- a/ide/deploy/scan.py
+++ b/ide/deploy/scan.py
@@ -17,6 +17,7 @@
from glob import glob
from .deploy import *
+from .client import get_nuvolaris_config
def scan():
@@ -25,8 +26,20 @@ def scan():
packages = set()
print("> Scan:")
- reqs = glob("packages/*/*/requirements.txt") + \
- glob("packages/*/*/package.json") + glob("packages/*/*/composer.json")
+
+ # => REQUIREMENTS
+ default_reqs_globs = ["packages/*/*/requirements.txt",
+ "packages/*/*/package.json",
+ "packages/*/*/composer.json"
+ ]
+ package_globs = get_nuvolaris_config("requirements", default_reqs_globs)
+ reqs = list()
+
+ for pkg_glob in package_globs:
+ items = glob(pkg_glob)
+ # extend first list without duplicates
+ reqs.extend(x for x in items if x not in reqs)
+
# req = reqs[0]
# from util.deploy.deploy import *
for req in reqs:
@@ -36,8 +49,17 @@ def scan():
deployments.add(act)
packages.add(sp[1])
- mains = glob("packages/*/*/index.js") + \
- glob("packages/*/*/__main__.py") + glob("packages/*/*/index.php")
+ # => MAINS
+ default_mains_globs = ["packages/*/*/index.js",
+ "packages/*/*/__main__.py",
+ "packages/*/*/index.php"]
+ mains_globs = get_nuvolaris_config("mains", default_mains_globs)
+ mains = list()
+ for main_glob in mains_globs:
+ items = glob(main_glob)
+ # extend first list without duplicates
+ mains.extend(x for x in items if x not in mains)
+
# main = mains[2]
for main in mains:
print(">> Main:", main)
@@ -46,8 +68,15 @@ def scan():
deployments.add(act)
packages.add(sp[1])
- singles = glob("packages/*/*.py") + \
- glob("packages/*/*.js") + glob("packages/*/*.php")
+ # => SINGLES
+ default_singles_globs = ["packages/*/*.py",
+ "packages/*/*.js", "packages/*/*.php"]
+ singles_globs = get_nuvolaris_config("singles", default_singles_globs)
+ singles = list()
+ for single_glob in singles_globs:
+ items = glob(single_glob)
+ singles.extend(x for x in items if x not in singles)
+
# single = singles[0]
for single in singles:
print(">> Action:", single)
@@ -56,7 +85,6 @@ def scan():
packages.add(sp[1])
print("> Deploying:")
-
for package in packages:
print(">> Package:", package)
deploy_package(package)