You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@openserverless.apache.org by ms...@apache.org on 2024/07/26 11:45:55 UTC

(openserverless-task) branch 0.1.0-testing created (now 984b82d)

This is an automated email from the ASF dual-hosted git repository.

msciabarra pushed a change to branch 0.1.0-testing
in repository https://gitbox.apache.org/repos/asf/openserverless-task.git


      at 984b82d  feat: moved config to module. added pydoc strings

This branch includes the following new commits:

     new 6c897c9  nuvolaris config from package.json
     new 29c9e67  Merge branch 'nuvolaris:3.0.0-testing' into 3.0.0-testing
     new 5cee7d8  Merge branch 'nuvolaris:3.0.0-testing' into 3.0.0-testing
     new db177b4  Merge remote-tracking branch 'upstream/3.0.0-testing' into 3.0.0-testing
     new 984b82d  feat: moved config to module. added pydoc strings

The 5 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.



(openserverless-task) 04/05: Merge remote-tracking branch 'upstream/3.0.0-testing' into 3.0.0-testing

Posted by ms...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

msciabarra pushed a commit to branch 0.1.0-testing
in repository https://gitbox.apache.org/repos/asf/openserverless-task.git

commit db177b470d33b12893dd8d3910f1ce1ff4de6be2
Merge: 5cee7d8 b49f497
Author: d4rkstar <d4...@gmail.com>
AuthorDate: Mon Jul 8 23:41:49 2024 +0200

    Merge remote-tracking branch 'upstream/3.0.0-testing' into 3.0.0-testing

 ide/deploy/deploy.py |   2 +-
 ide/deploy/scan.py   |  12 +++--
 ide/go/exec.env      |   1 +
 ide/go/launcher.go   | 142 +++++++++++++++++++++++++++++++++++++++++++++++++++
 ide/go/nuvfile.yml   |  52 +++++++++++++++++++
 ide/nuvfile.yml      |   3 ++
 ide/nuvopts.txt      |   1 +
 ide/util/nuvfile.yml |   6 +++
 8 files changed, 214 insertions(+), 5 deletions(-)


(openserverless-task) 02/05: Merge branch 'nuvolaris:3.0.0-testing' into 3.0.0-testing

Posted by ms...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

msciabarra pushed a commit to branch 0.1.0-testing
in repository https://gitbox.apache.org/repos/asf/openserverless-task.git

commit 29c9e67754b3f729f9b75089b02b5f5dc8d9ab94
Merge: 6c897c9 4921deb
Author: d4rkstar <d4...@gmail.com>
AuthorDate: Mon May 27 01:37:45 2024 +0200

    Merge branch 'nuvolaris:3.0.0-testing' into 3.0.0-testing

 nuvroot.json  |   2 +-
 runtimes.json | 192 ++++++++++++++++++++++++++++++++++++++++++++--------------
 2 files changed, 146 insertions(+), 48 deletions(-)


(openserverless-task) 05/05: feat: moved config to module. added pydoc strings

Posted by ms...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

msciabarra pushed a commit to branch 0.1.0-testing
in repository https://gitbox.apache.org/repos/asf/openserverless-task.git

commit 984b82da42766d387e7b29492315c03810f54bcc
Author: d4rkstar <d4...@gmail.com>
AuthorDate: Tue Jul 9 01:39:10 2024 +0200

    feat: moved config to module. added pydoc strings
    
    Created a config.py module and moved there all the common defaults params. Added pydoc fucntions documentation
---
 ide/deploy/client.py | 59 +++++++++++++++++++++++------------
 ide/deploy/config.py | 65 +++++++++++++++++++++++++++++++++++++++
 ide/deploy/deploy.py | 87 +++++++++++++++++++++++++++++++++++++++++++---------
 ide/deploy/scan.py   | 51 +++++++++++++++++++-----------
 ide/deploy/watch.py  | 39 +++++++++++++++++------
 5 files changed, 239 insertions(+), 62 deletions(-)

diff --git a/ide/deploy/client.py b/ide/deploy/client.py
index 21631fd..895df97 100644
--- a/ide/deploy/client.py
+++ b/ide/deploy/client.py
@@ -15,47 +15,66 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from pathlib import Path
 from subprocess import Popen, PIPE
-import os, os.path, json
+import os
+import os.path
 import threading
-import asyncio
-
-def get_nuvolaris_config(key, default):
-    try:
-        dir = os.environ.get("NUV_PWD", "/do_not_exists")
-        file = f"{dir}/package.json"
-        info = json.loads(Path(file).read_text())
-        return info.get("nuvolaris", {}).get(key, default)
-    except:
-        return default
-
-def readlines(inp):
+from typing import IO
+from .config import get_nuvolaris_config
+
+
+def readlines(inp: IO[str]):
+    """Read line from an file descriptor
+
+    Args:
+        inp (IO[str]): the file descriptor
+    """
     for line in iter(inp.readline, ''):
         print(line, end='')
 
 # serve web area
-def launch(key, default):
+
+
+def launch(key: str, default: (str | list)):
+    """Launch a command in a process, reading.
+    The command is extracted from nuvolaris config in package.json 
+    or a default command is used
+
+    Args:
+        key (str): the key from which read the command
+        default (str  |  list): the default if the key is not found
+    """
     cmd = get_nuvolaris_config(key, default)
     proc = Popen(
-        cmd, shell=True, 
-        cwd=os.environ.get("NUV_PWD"), env=os.environ, 
+        cmd, shell=True,
+        cwd=os.environ.get("NUV_PWD"), env=os.environ,
         stdin=PIPE, stdout=PIPE, stderr=PIPE, text=True
     )
     threading.Thread(target=readlines, args=(proc.stdout,)).start()
     threading.Thread(target=readlines, args=(proc.stderr,)).start()
 
-def serve(): 
+
+def serve():
+    """Serve the web area
+    """
     launch("devel", "nuv ide serve")
 
-def logs(): 
+
+def logs():
+    """Serve the openwhisk activation's logs
+    """
     launch("logs", "nuv activation poll")
 
 # build
+
+
 def build():
+    """Try to build the frontend application, if the deploy command is set.
+    
+    """
     deploy = get_nuvolaris_config("deploy", "true")
     proc = Popen(
-        deploy, shell=True, 
+        deploy, shell=True,
         env=os.environ,
         cwd=os.environ.get("NUV_PWD"),
         stdin=PIPE, stdout=PIPE, stderr=PIPE, text=True
diff --git a/ide/deploy/config.py b/ide/deploy/config.py
new file mode 100644
index 0000000..e060e16
--- /dev/null
+++ b/ide/deploy/config.py
@@ -0,0 +1,65 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import os
+import json
+from pathlib import Path
+
+MAINS = ["__main__.py",
+         "index.js",
+         "index.php",
+         "main.go"]
+
+SKIPDIR = ["virtualenv",
+           "node_modules",
+           "__pycache__"]
+
+DEFAULT_REQ_GLOBS = ["packages/*/*/requirements.txt",
+                     "packages/*/*/package.json",
+                     "packages/*/*/composer.json",
+                     "packages/*/*/go.mod"]
+
+DEFAULT_MAIN_GLOBS = ["packages/*/*/index.js",
+                      "packages/*/*/__main__.py",
+                      "packages/*/*/index.php",
+                      "packages/*/*/main.go"]
+
+DEFAULT_SINGLES_GLOBS = ["packages/*/*.py",
+                         "packages/*/*.js",
+                         "packages/*/*.php",
+                         "packages/*/*.go"]
+
+
+def get_nuvolaris_config(key: str, default: list | str) -> (list | str):
+    """Read package.json if exists and retrieve the required
+    value for passed key (if defined)
+
+    Args:
+        key (str): the key of the parameter to retrieve
+        default (list | str): the default value to return
+
+    Returns:
+        _type_: the list of values or the single string value.
+        if not defined, the default value is returned
+    """
+    try:
+        dir = os.environ.get("NUV_PWD", "/do_not_exists")
+        file = f"{dir}/package.json"
+        info = json.loads(Path(file).read_text())
+        return info.get("nuvolaris", {}).get(key, default)
+    except:
+        return default
diff --git a/ide/deploy/deploy.py b/ide/deploy/deploy.py
index e698a15..9baa9fc 100644
--- a/ide/deploy/deploy.py
+++ b/ide/deploy/deploy.py
@@ -15,28 +15,49 @@
 # specific language governing permissions and limitations
 # under the License.
 
-MAINS = ["__main__.py", "index.js", "index.php", "main.go"]
-
 import os
-from  os.path import exists, isdir
+from os.path import exists, isdir
 from subprocess import Popen
+from .config import MAINS
 
 dry_run = False
 
-def set_dry_run(b):
+
+def set_dry_run(b: bool):
+    """Set global dry run
+
+    Args:
+        b (bool): true for dry run enabled
+    """
     global dry_run
     dry_run = b
 
-def exec(cmd):
+
+def exec(cmd: str):
+    """Exec a shell command and wait for it to complete.
+    If dryrun is set, the command is not executed.
+
+    Args:
+        cmd (str): command line to execue
+    """
     global dry_run
     print("$", cmd)
     if not dry_run:
         Popen(cmd, shell=True, env=os.environ).wait()
 
-def extract_args(files):
+
+def extract_args(files: list) -> list:
+    """Extract openwhisk args from files
+
+    Args:
+        files (list): the list of files to inspect
+
+    Returns:
+        list: a list of parameters
+    """
     res = []
     for file in files:
-        #if dry_run:
+        # if dry_run:
         #  print(f": inspecting {file}")
         if exists(file):
             with open(file, "r") as f:
@@ -47,9 +68,16 @@ def extract_args(files):
                         res.append(line.strip()[2:])
     return res
 
+
 package_done = set()
 
-def deploy_package(package):
+
+def deploy_package(package: str):
+    """Deploy a package on nuvolaris
+
+    Args:
+        package (str): the name of package
+    """
     global package_done
     # package args
     ppath = f"packages/{package}.args"
@@ -59,15 +87,38 @@ def deploy_package(package):
         exec(cmd)
         package_done.add(cmd)
 
-def build_zip(package, action):
+
+def build_zip(package: str, action: str) -> str:
+    """Builds a zip for the package / action
+
+    Args:
+        package (str): package
+        action (str): action
+
+    Returns:
+        str: the path of the built zip file
+    """
     exec(f"nuv ide util zip A={package}/{action}")
     return f"packages/{package}/{action}.zip"
 
-def build_action(package, action):
+
+def build_action(package: str, action: str) -> str:
+    """Invoke the nuv ide util action command on package / action
+
+    Args:
+        package (_type_): _description_
+        action (_type_): _description_
+
+    Returns:
+        str: the zip with the built action
+    """
     exec(f"nuv ide util action A={package}/{action}")
     return f"packages/{package}/{action}.zip"
 
-def deploy_action(artifact):
+
+def deploy_action(artifact: str):
+    """Deploy an artifact calling nuv action update
+    """
     try:
         sp = artifact.split("/")
         [name, typ] = sp[-1].rsplit(".", 1)
@@ -83,18 +134,26 @@ def deploy_action(artifact):
         to_inspect = [f"{base}/{x}" for x in MAINS]
     else:
         to_inspect = [artifact]
-    
+
     args = " ".join(extract_args(to_inspect))
     exec(f"nuv action update {package}/{name} {artifact} {args}")
 
+
 """
 file = "packages/deploy/hello.py"
 file = "packages/deploy/multi.zip"
 file = "packages/deploy/multi/__main__.py"
 file = "packages/deploy/multi/requirements.txt"
 """
-def deploy(file):
-    #print(f"*** {file}")
+
+
+def deploy(file: str):
+    """Deploy a package on nuvolaris
+
+    Args:
+        file (str): the file to deploy
+    """
+    # print(f"*** {file}")
     if isdir(file):
         for start in MAINS:
             sub = f"{file}/{start}"
diff --git a/ide/deploy/scan.py b/ide/deploy/scan.py
index 4b33250..0be87e7 100644
--- a/ide/deploy/scan.py
+++ b/ide/deploy/scan.py
@@ -18,22 +18,44 @@
 from glob import glob
 from .deploy import *
 from .client import get_nuvolaris_config
+from .config import DEFAULT_REQ_GLOBS, DEFAULT_MAIN_GLOBS, DEFAULT_SINGLES_GLOBS
 
 
 def scan():
+    """This function has two stages:\n
+    Scan
+    ===    
+    A) Packages requirements\n
+    Check for package requirements. It will look in a specific requirement file 
+    (from web package.json) or a default set of requirements (requirements.txt, 
+    composer.json etc. etc).\n
+    For each requirement found, a zip is built calling the task:
+    ``nuv ide util zip A={package}/{action}``\n
+    The zip is added to actions (deployments); the directory is added to 
+    packages (packages)\n
+    B) Mains for packages\n
+    After that, the scan continue checking the "mains". For each main, an
+    action is built calling the task:\n
+    ``nuv ide util action A={package}/{action}``\n
+    where package is the base directory of main file and the action is the
+    basename of the main file.\n
+    C) Singles\n
+    Finally the scan will take care of single file functions\n
+    
+    Deploy
+    ===
+    Each package is deployed with the command:\n
+    ``nuv package update {package} {pargs}``\n
+    Each deployment is deployed with the command:\n
+    ``nuv action update {package}/{name} {artifact} {args}``\n
+    """
+
     # first look for requirements.txt and build the venv (add in set)
     deployments = set()
     packages = set()
 
     print("> Scan:")
-
-    # => REQUIREMENTS
-    default_reqs_globs = ["packages/*/*/requirements.txt",
-                          "packages/*/*/package.json",
-                          "packages/*/*/composer.json",
-                          "packages/*/*/go.mod"]
-                          
-    package_globs = get_nuvolaris_config("requirements", default_reqs_globs)
+    package_globs = get_nuvolaris_config("requirements", DEFAULT_REQ_GLOBS)
     reqs = list()
 
     for pkg_glob in package_globs:
@@ -51,11 +73,7 @@ def scan():
         packages.add(sp[1])
 
     # => MAINS
-    default_mains_globs = ["packages/*/*/index.js",
-                           "packages/*/*/__main__.py",
-                           "packages/*/*/index.php",
-                           "packages/*/*/main.go"]
-    mains_globs = get_nuvolaris_config("mains", default_mains_globs)
+    mains_globs = get_nuvolaris_config("mains", DEFAULT_MAIN_GLOBS)
     mains = list()
     for main_glob in mains_globs:
         items = glob(main_glob)
@@ -71,11 +89,8 @@ def scan():
         packages.add(sp[1])
 
     # => SINGLES
-    default_singles_globs = ["packages/*/*.py",
-                             "packages/*/*.js", 
-                             "packages/*/*.php",
-                             "packages/*/*.go"]
-    singles_globs = get_nuvolaris_config("singles", default_singles_globs)
+
+    singles_globs = get_nuvolaris_config("singles", DEFAULT_SINGLES_GLOBS)
     singles = list()
     for single_glob in singles_globs:
         items = glob(single_glob)
diff --git a/ide/deploy/watch.py b/ide/deploy/watch.py
index 99d4234..7559958 100644
--- a/ide/deploy/watch.py
+++ b/ide/deploy/watch.py
@@ -15,48 +15,66 @@
 # specific language governing permissions and limitations
 # under the License.
 
-SKIPDIR = ["virtualenv", "node_modules", "__pycache__"]
-
+from typing import Tuple
 import watchfiles
+
 import asyncio
 import os.path
 import signal
 
-
+from .config import SKIPDIR
 from .deploy import deploy
 from .client import serve, logs
 
-def check_and_deploy(change):
+
+def check_and_deploy(change: Tuple[watchfiles.Change, str]):
+    """Deploy a file if the received change is a file modification
+
+    Args:
+        change (Tuple[watchfiles.Change, str]): the change type and filename
+    """
     change_type, path = change
     path = os.path.abspath(path)
     cur_dir_len = len(os.getcwd())+1
     src = path[cur_dir_len:]
     # only modified
-    if change_type != watchfiles.Change.modified: return
+    if change_type != watchfiles.Change.modified:
+        return
     # no directories
-    if os.path.isdir(src): return
+    if os.path.isdir(src):
+        return
     # no missing files
-    if not os.path.exists(src): return
+    if not os.path.exists(src):
+        return
     # no generated directories
     for dir in src.split("/")[:-1]:
-        if dir in SKIPDIR: return
+        if dir in SKIPDIR:
+            return
     # no generated files
-    if src.endswith(".zip"): return
+    if src.endswith(".zip"):
+        return
     # now you can deploy
     deploy(src)
 
+
 async def redeploy():
+    """Install the filesystem watcher over the packages directory
+    """
     print("> Watching:")
     iterator = watchfiles.awatch("packages", recursive=True)
     async for changes in iterator:
         for change in changes:
             try:
-                #print(change)
+                # print(change)
                 check_and_deploy(change)
             except Exception as e:
                 print(e)
 
+
 def watch():
+    """This function will start the webserver and show logs.
+    After that it will watch the filesystem in a loop, until SIGTERM received.
+    """
     # start web server
     serve()
     # show logs
@@ -64,6 +82,7 @@ def watch():
 
     loop = asyncio.get_event_loop()
     task = loop.create_task(redeploy())
+
     def end_loop():
         print("Ending task.")
         task.cancel()


(openserverless-task) 03/05: Merge branch 'nuvolaris:3.0.0-testing' into 3.0.0-testing

Posted by ms...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

msciabarra pushed a commit to branch 0.1.0-testing
in repository https://gitbox.apache.org/repos/asf/openserverless-task.git

commit 5cee7d80a91b11f32effe6343c8b6cfb04f63b93
Merge: 29c9e67 9d3cb50
Author: d4rkstar <d4...@gmail.com>
AuthorDate: Tue Jun 25 13:11:49 2024 +0200

    Merge branch 'nuvolaris:3.0.0-testing' into 3.0.0-testing

 admin/nuvfile.yml   | 20 ++++++++++----------
 admin/nuvopts.txt   |  4 ++--
 admin/user-crd.yaml |  8 ++++----
 web/nuvfile.yml     |  7 +++++++
 web/upload.js       |  9 ++++-----
 5 files changed, 27 insertions(+), 21 deletions(-)


(openserverless-task) 01/05: nuvolaris config from package.json

Posted by ms...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

msciabarra pushed a commit to branch 0.1.0-testing
in repository https://gitbox.apache.org/repos/asf/openserverless-task.git

commit 6c897c9bca038117c1657176dd924781057e4c8d
Author: d4rkstar <d4...@gmail.com>
AuthorDate: Mon May 27 01:32:55 2024 +0200

    nuvolaris config from package.json
---
 ide/deploy/scan.py | 42 +++++++++++++++++++++++++++++++++++-------
 1 file changed, 35 insertions(+), 7 deletions(-)

diff --git a/ide/deploy/scan.py b/ide/deploy/scan.py
index bb2205a..5382628 100644
--- a/ide/deploy/scan.py
+++ b/ide/deploy/scan.py
@@ -17,6 +17,7 @@
 
 from glob import glob
 from .deploy import *
+from .client import get_nuvolaris_config
 
 
 def scan():
@@ -25,8 +26,20 @@ def scan():
     packages = set()
 
     print("> Scan:")
-    reqs = glob("packages/*/*/requirements.txt") + \
-        glob("packages/*/*/package.json") + glob("packages/*/*/composer.json")
+
+    # => REQUIREMENTS
+    default_reqs_globs = ["packages/*/*/requirements.txt",
+                          "packages/*/*/package.json",
+                          "packages/*/*/composer.json"
+                          ]
+    package_globs = get_nuvolaris_config("requirements", default_reqs_globs)
+    reqs = list()
+
+    for pkg_glob in package_globs:
+        items = glob(pkg_glob)
+        # extend first list without duplicates
+        reqs.extend(x for x in items if x not in reqs)
+
     # req = reqs[0]
     # from util.deploy.deploy import *
     for req in reqs:
@@ -36,8 +49,17 @@ def scan():
         deployments.add(act)
         packages.add(sp[1])
 
-    mains = glob("packages/*/*/index.js") + \
-        glob("packages/*/*/__main__.py") + glob("packages/*/*/index.php")
+    # => MAINS
+    default_mains_globs = ["packages/*/*/index.js",
+                           "packages/*/*/__main__.py",
+                           "packages/*/*/index.php"]
+    mains_globs = get_nuvolaris_config("mains", default_mains_globs)
+    mains = list()
+    for main_glob in mains_globs:
+        items = glob(main_glob)
+        # extend first list without duplicates
+        mains.extend(x for x in items if x not in mains)
+
     # main = mains[2]
     for main in mains:
         print(">> Main:", main)
@@ -46,8 +68,15 @@ def scan():
         deployments.add(act)
         packages.add(sp[1])
 
-    singles = glob("packages/*/*.py") + \
-        glob("packages/*/*.js") + glob("packages/*/*.php")
+    # => SINGLES
+    default_singles_globs = ["packages/*/*.py",
+                             "packages/*/*.js", "packages/*/*.php"]
+    singles_globs = get_nuvolaris_config("singles", default_singles_globs)
+    singles = list()
+    for single_glob in singles_globs:
+        items = glob(single_glob)
+        singles.extend(x for x in items if x not in singles)
+
     # single = singles[0]
     for single in singles:
         print(">> Action:", single)
@@ -56,7 +85,6 @@ def scan():
         packages.add(sp[1])
 
     print("> Deploying:")
-
     for package in packages:
         print(">> Package:", package)
         deploy_package(package)