From e4301cb3fb913e63f7270167d712a8e4e139c487 Mon Sep 17 00:00:00 2001 From: Francesco Timperi Tiberi Date: Wed, 22 May 2024 18:54:21 +0100 Subject: [PATCH 1/8] chore: using 3.1.0-mastrogpt.2405160951 runtimes build --- nuvroot.json | 2 +- runtimes.json | 192 ++++++++++++++++++++++++++++++++++++++------------ 2 files changed, 146 insertions(+), 48 deletions(-) diff --git a/nuvroot.json b/nuvroot.json index add0b69..7a5aac8 100644 --- a/nuvroot.json +++ b/nuvroot.json @@ -1,5 +1,5 @@ { - "version": "3.0.1-beta.2401121536", + "version": "3.0.1-beta.2405062139", "config": { "nuvolaris": { "apihost": "auto", diff --git a/runtimes.json b/runtimes.json index 5fcb632..3b332c7 100644 --- a/runtimes.json +++ b/runtimes.json @@ -15,12 +15,26 @@ "runtimes": { "nodejs": [ { - "kind": "nodejs:18", + "kind": "nodejs:21", + "default": false, + "image": { + "prefix": "ghcr.io/nuvolaris", + "name": "runtime-nodejs-v21", + "tag": "3.1.0-mastrogpt.2405160951" + }, + "deprecated": false, + "attached": { + "attachmentName": "codefile", + "attachmentType": "text/plain" + } + }, + { + "kind": "nodejs:20", "default": true, "image": { "prefix": "ghcr.io/nuvolaris", - "name": "action-nodejs-v18", - "tag": "3.0.0-beta.2311231822" + "name": "runtime-nodejs-v20", + "tag": "3.1.0-mastrogpt.2405160951" }, "deprecated": false, "attached": { @@ -41,15 +55,29 @@ } ] }, + { + "kind": "nodejs:18", + "default": false, + "image": { + "prefix": "ghcr.io/nuvolaris", + "name": "runtime-nodejs-v18", + "tag": "3.1.0-mastrogpt.2405160951" + }, + "deprecated": false, + "attached": { + "attachmentName": "codefile", + "attachmentType": "text/plain" + } + }, { "kind": "nodejs:16", "default": false, "image": { "prefix": "ghcr.io/nuvolaris", "name": "action-nodejs-v16", - "tag": "3.0.0-beta.2311231822" + "tag": "3.0.0-beta.2309081459" }, - "deprecated": false, + "deprecated": true, "attached": { "attachmentName": "codefile", "attachmentType": "text/plain" @@ -62,8 +90,8 @@ "default": true, "image": { "prefix": "ghcr.io/nuvolaris", - "name": "action-python-v311", - "tag": "3.0.0-beta.2401111655" + "name": "runtime-python-v3.11", + "tag": "3.1.0-mastrogpt.2405160951" }, "deprecated": false, "attached": { @@ -84,13 +112,41 @@ } ] }, + { + "kind": "python:3.11", + "default": false, + "image": { + "prefix": "ghcr.io/nuvolaris", + "name": "runtime-python-v3.11", + "tag": "3.1.0-mastrogpt.2405160951" + }, + "deprecated": false, + "attached": { + "attachmentName": "codefile", + "attachmentType": "text/plain" + } + }, + { + "kind": "python:3.12", + "default": false, + "image": { + "prefix": "ghcr.io/nuvolaris", + "name": "runtime-python-v3.12", + "tag": "3.1.0-mastrogpt.2405160951" + }, + "deprecated": false, + "attached": { + "attachmentName": "codefile", + "attachmentType": "text/plain" + } + }, { "kind": "python:310", "default": false, "image": { "prefix": "ghcr.io/nuvolaris", - "name": "action-python-v310", - "tag": "3.0.0-beta.2401111655" + "name": "runtime-python-v3.10", + "tag": "3.1.0-mastrogpt.2405160951" }, "deprecated": false, "attached": { @@ -130,29 +186,87 @@ "requireMain": true } ], + "go": [ + { + "kind": "go:1.22", + "default": true, + "deprecated": false, + "attached": { + "attachmentName": "codefile", + "attachmentType": "text/plain" + }, + "image": { + "prefix": "ghcr.io/nuvolaris", + "name": "runtime-golang-v1.22", + "tag": "3.1.0-mastrogpt.2405160951" + } + }, + { + "kind": "go:1.21", + "default": false, + "deprecated": false, + "attached": { + "attachmentName": "codefile", + "attachmentType": "text/plain" + }, + "image": { + "prefix": "ghcr.io/nuvolaris", + "name": "runtime-golang-v1.21", + "tag": "3.1.0-mastrogpt.2405160951" + } + }, + { + "kind": "go:1.20", + "default": false, + "deprecated": false, + "attached": { + "attachmentName": "codefile", + "attachmentType": "text/plain" + }, + "image": { + "prefix": "ghcr.io/nuvolaris", + "name": "runtime-golang-v1.20", + "tag": "3.1.0-mastrogpt.2405160951" + } + }, + { + "kind": "go:1.20mf", + "default": false, + "deprecated": false, + "attached": { + "attachmentName": "codefile", + "attachmentType": "text/plain" + }, + "image": { + "prefix": "ghcr.io/nuvolaris", + "name": "go-nuvolaris-metaflow", + "tag": "bc86ab6" + } + } + ], "php": [ { - "kind": "php:8.2", + "kind": "php:8.3", "default": true, "deprecated": false, "image": { "prefix": "ghcr.io/nuvolaris", - "name": "action-php-v8.2", - "tag": "3.0.0-beta.2311031603" + "name": "runtime-php-v8.3", + "tag": "3.1.0-mastrogpt.2405160951" }, "attached": { "attachmentName": "codefile", "attachmentType": "text/plain" } - }, + }, { - "kind": "php:8.1", + "kind": "php:8.2", "default": false, "deprecated": false, "image": { "prefix": "ghcr.io/nuvolaris", - "name": "action-php-v8.1", - "tag": "3.0.0-beta.2311031603" + "name": "runtime-php-v8.2", + "tag": "3.1.0-mastrogpt.2405160951" }, "attached": { "attachmentName": "codefile", @@ -160,39 +274,37 @@ } }, { - "kind": "php:8.0", + "kind": "php:8.1", "default": false, "deprecated": false, "image": { "prefix": "ghcr.io/nuvolaris", - "name": "action-php-v8.0", - "tag": "3.0.0-beta.2311031603" + "name": "runtime-php-v8.1", + "tag": "3.1.0-mastrogpt.2405160951" }, "attached": { "attachmentName": "codefile", "attachmentType": "text/plain" } - } - ], - "ruby": [ + }, { - "kind": "ruby:2.5", - "default": true, + "kind": "php:8.0", + "default": false, "deprecated": false, + "image": { + "prefix": "ghcr.io/nuvolaris", + "name": "runtime-php-v8.0", + "tag": "3.1.0-mastrogpt.2405160951" + }, "attached": { "attachmentName": "codefile", "attachmentType": "text/plain" - }, - "image": { - "prefix": "openwhisk", - "name": "action-ruby-v2.5", - "tag": "nightly" } } ], - "go": [ + "ruby": [ { - "kind": "go:1.17", + "kind": "ruby:2.5", "default": true, "deprecated": false, "attached": { @@ -201,25 +313,11 @@ }, "image": { "prefix": "openwhisk", - "name": "action-golang-v1.17", + "name": "action-ruby-v2.5", "tag": "nightly" } - }, - { - "kind": "go:1.20mf", - "default": false, - "deprecated": false, - "attached": { - "attachmentName": "codefile", - "attachmentType": "text/plain" - }, - "image": { - "prefix": "ghcr.io/nuvolaris", - "name": "go-nuvolaris-metaflow", - "tag": "bc86ab6" - } } - ], + ], "dotnet": [ { "kind": "dotnet:2.2", From 6c897c9bca038117c1657176dd924781057e4c8d Mon Sep 17 00:00:00 2001 From: d4rkstar Date: Mon, 27 May 2024 01:32:55 +0200 Subject: [PATCH 2/8] nuvolaris config from package.json --- ide/deploy/scan.py | 42 +++++++++++++++++++++++++++++++++++------- 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/ide/deploy/scan.py b/ide/deploy/scan.py index bb2205a..5382628 100644 --- a/ide/deploy/scan.py +++ b/ide/deploy/scan.py @@ -17,6 +17,7 @@ from glob import glob from .deploy import * +from .client import get_nuvolaris_config def scan(): @@ -25,8 +26,20 @@ def scan(): packages = set() print("> Scan:") - reqs = glob("packages/*/*/requirements.txt") + \ - glob("packages/*/*/package.json") + glob("packages/*/*/composer.json") + + # => REQUIREMENTS + default_reqs_globs = ["packages/*/*/requirements.txt", + "packages/*/*/package.json", + "packages/*/*/composer.json" + ] + package_globs = get_nuvolaris_config("requirements", default_reqs_globs) + reqs = list() + + for pkg_glob in package_globs: + items = glob(pkg_glob) + # extend first list without duplicates + reqs.extend(x for x in items if x not in reqs) + # req = reqs[0] # from util.deploy.deploy import * for req in reqs: @@ -36,8 +49,17 @@ def scan(): deployments.add(act) packages.add(sp[1]) - mains = glob("packages/*/*/index.js") + \ - glob("packages/*/*/__main__.py") + glob("packages/*/*/index.php") + # => MAINS + default_mains_globs = ["packages/*/*/index.js", + "packages/*/*/__main__.py", + "packages/*/*/index.php"] + mains_globs = get_nuvolaris_config("mains", default_mains_globs) + mains = list() + for main_glob in mains_globs: + items = glob(main_glob) + # extend first list without duplicates + mains.extend(x for x in items if x not in mains) + # main = mains[2] for main in mains: print(">> Main:", main) @@ -46,8 +68,15 @@ def scan(): deployments.add(act) packages.add(sp[1]) - singles = glob("packages/*/*.py") + \ - glob("packages/*/*.js") + glob("packages/*/*.php") + # => SINGLES + default_singles_globs = ["packages/*/*.py", + "packages/*/*.js", "packages/*/*.php"] + singles_globs = get_nuvolaris_config("singles", default_singles_globs) + singles = list() + for single_glob in singles_globs: + items = glob(single_glob) + singles.extend(x for x in items if x not in singles) + # single = singles[0] for single in singles: print(">> Action:", single) @@ -56,7 +85,6 @@ def scan(): packages.add(sp[1]) print("> Deploying:") - for package in packages: print(">> Package:", package) deploy_package(package) From 5bd611a94b58b3174ccb81e93209a804a4c7e339 Mon Sep 17 00:00:00 2001 From: Francesco Timperi Tiberi Date: Fri, 31 May 2024 20:30:36 +0100 Subject: [PATCH 3/8] chore: adjusted adduser to support activation of object storage (MINIO or CEPH required) --- admin/nuvfile.yml | 18 +++++++++--------- admin/nuvopts.txt | 4 ++-- admin/user-crd.yaml | 6 +++--- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/admin/nuvfile.yml b/admin/nuvfile.yml index 28234c2..c4463c5 100644 --- a/admin/nuvfile.yml +++ b/admin/nuvfile.yml @@ -85,10 +85,10 @@ tasks: export REDIS_ENABLED=false export MONGODB_ENABLED=false - export MINIO_DATA_ENABLED=false - export MINIO_STATIC_ENABLED=false + export STORAGE_DATA_ENABLED=false + export STORAGE_STATIC_ENABLED=false export POSTGRES_ENABLED=false - export MINIO_STORAGE_QUOTA=auto + export OBJECT_STORAGE_QUOTA=auto if {{.__redis}} || {{.__all}} then @@ -110,19 +110,19 @@ tasks: fi fi - if {{.__minio}} || {{.__all}} + if {{.__storage}} || {{.__all}} then - if $NUVOLARIS_MINIO + if $NUVOLARIS_MINIO || $NUVOLARIS_COSI then - MINIO_DATA_ENABLED=true - MINIO_STATIC_ENABLED=true + STORAGE_DATA_ENABLED=true + STORAGE_STATIC_ENABLED=true else - nuv -die "Error! Minio is not enabled in Nuvolaris" + nuv -die "Error! Onject Storage is not enabled in Nuvolaris" fi fi if test -n "{{.__storagequota}}" - then MINIO_STORAGE_QUOTA={{.__storagequota}} + then OBJECT_STORAGE_QUOTA={{.__storagequota}} fi if {{.__postgres}} || {{.__all}} diff --git a/admin/nuvopts.txt b/admin/nuvopts.txt index a3d48c6..1a386d1 100644 --- a/admin/nuvopts.txt +++ b/admin/nuvopts.txt @@ -1,7 +1,7 @@ Subcommand: nuv admin Usage: - admin adduser [--all] [--redis] [--mongodb] [--minio] [--postgres] [--storagequota=|auto] + admin adduser [--all] [--redis] [--mongodb] [--storage] [--postgres] [--storagequota=|auto] admin deleteuser Commands: @@ -12,6 +12,6 @@ Options: --all enable all services --redis enable redis --mongodb enable mongodb - --minio enable minio + --storage enable object based storage --postgres enable postgres --storagequota= diff --git a/admin/user-crd.yaml b/admin/user-crd.yaml index 0342329..9a027c0 100644 --- a/admin/user-crd.yaml +++ b/admin/user-crd.yaml @@ -40,10 +40,10 @@ spec: password: ${USER_SECRET_POSTGRES} object-storage: password: ${USER_SECRET_MINIO} - quota: "${MINIO_STORAGE_QUOTA:-auto}" + quota: "${OBJECT_STORAGE_QUOTA:-auto}" data: - enabled: ${MINIO_DATA_ENABLED} + enabled: ${STORAGE_DATA_ENABLED} bucket: ${USERNAME}-data route: - enabled: ${MINIO_STATIC_ENABLED} + enabled: ${STORAGE_STATIC_ENABLED} bucket: ${USERNAME}-web From 1866b4e7317219aa5bc34528a1264de1371e2b24 Mon Sep 17 00:00:00 2001 From: Francesco Timperi Tiberi Date: Sat, 1 Jun 2024 16:31:06 +0100 Subject: [PATCH 4/8] fix: update nuv web upload to use the nuv devel minio upload action to ensure it work on both MINIO and CEPH implementation --- web/nuvfile.yml | 7 +++++++ web/upload.js | 9 ++++----- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/web/nuvfile.yml b/web/nuvfile.yml index 0c7b047..b79679f 100644 --- a/web/nuvfile.yml +++ b/web/nuvfile.yml @@ -17,6 +17,8 @@ version: 3 + + tasks: upload: silent: true @@ -34,3 +36,8 @@ tasks: sh: wsk property get | awk '/whisk namespace/{print $3}' APIHOST: sh: wsk property get | awk '/whisk API host/{print $4}' + AUTHB64: + sh: | + if test -e ~/.wskprops + then source ~/.wskprops; echo $(base64 -e $AUTH) + fi diff --git a/web/upload.js b/web/upload.js index 3bbdff5..151b379 100644 --- a/web/upload.js +++ b/web/upload.js @@ -17,7 +17,7 @@ const nuv = require('nuv'); -const contentActionAddr = `${process.env.APIHOST}/api/v1/web/whisk-system/nuv/content/${process.env.NUVUSER}`; +const contentActionAddr = `${process.env.APIHOST}/api/v1/web/whisk-system/nuv/devel_upload/${process.env.MINIO_BUCKET_STATIC}`; // *** Main *** main(); @@ -30,9 +30,8 @@ function main() { let verbose = extractBoolFromParam(verboseParam); let clean = extractBoolFromParam(cleanParam); - let minioKey = `MINIO_SECRET_KEY`; - const minioAuth = process.env[minioKey]; + const minioAuth = process.env.AUTHB64 const pathFoundAsDir = nuv.isDir(path); if (!pathFoundAsDir) { @@ -66,7 +65,7 @@ function main() { function uploadContent(file, minioAuth, fileAddr, verbose) { console.log(`Uploading ${fileAddr}...`); - let res = nuv.nuvExec("curl", "-s", "-X", "PUT", "-T", file, "-H", `minioauth: ${minioAuth}`, `${contentActionAddr}/${fileAddr}`); + let res = nuv.nuvExec("curl", "-s", "-X", "PUT", "-T", file, "-H", `x-impersonate-auth: ${minioAuth}`, `${contentActionAddr}/${fileAddr}`); if (verbose) { console.log(res); } @@ -74,7 +73,7 @@ function uploadContent(file, minioAuth, fileAddr, verbose) { function deleteContent(minioAuth, fileAddr, verbose) { console.log(`Deleting ${fileAddr}...`); - let res = nuv.nuvExec("curl", "-s", "-X", "DELETE", "-H", `minioauth: ${minioAuth}`, `${contentActionAddr}/${fileAddr}`); + let res = nuv.nuvExec("curl", "-s", "-X", "DELETE", "-H", `x-impersonate-auth: ${minioAuth}`, `${contentActionAddr}/${fileAddr}`); if (verbose) { console.log(res); } From cdd04a15bbca066957bf64a7fb1abcb2f8ff1bc6 Mon Sep 17 00:00:00 2001 From: Francesco Timperi Tiberi Date: Fri, 21 Jun 2024 19:36:55 +0100 Subject: [PATCH 5/8] fix: auth entry in config.json was overriding AUTH key for newly created wsku resources --- admin/nuvfile.yml | 2 +- admin/user-crd.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/admin/nuvfile.yml b/admin/nuvfile.yml index c4463c5..9d40ac9 100644 --- a/admin/nuvfile.yml +++ b/admin/nuvfile.yml @@ -46,7 +46,7 @@ tasks: export USERNAME={{._username_}} export EMAIL={{._email_}} export PASSWORD={{._password_}} - export AUTH="$(nuv -random -u):$(nuv -random --str 64)" + export NEW_USER_AUTH="$(nuv -random -u):$(nuv -random --str 64)" # check {{._username_}} is at least 5 chars long if [ ${#USERNAME} -lt 5 ] diff --git a/admin/user-crd.yaml b/admin/user-crd.yaml index 9a027c0..b6573ff 100644 --- a/admin/user-crd.yaml +++ b/admin/user-crd.yaml @@ -25,7 +25,7 @@ spec: email: ${EMAIL} password: ${PASSWORD} namespace: ${USERNAME} - auth: ${AUTH} + auth: ${NEW_USER_AUTH} redis: enabled: ${REDIS_ENABLED} prefix: ${USERNAME} From 11b74d2f4a81d461d019005272b57d818b290968 Mon Sep 17 00:00:00 2001 From: d4rkstar Date: Mon, 27 May 2024 01:32:55 +0200 Subject: [PATCH 6/8] nuvolaris config from package.json --- ide/deploy/scan.py | 42 +++++++++++++++++++++++++++++++++++------- 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/ide/deploy/scan.py b/ide/deploy/scan.py index bb2205a..5382628 100644 --- a/ide/deploy/scan.py +++ b/ide/deploy/scan.py @@ -17,6 +17,7 @@ from glob import glob from .deploy import * +from .client import get_nuvolaris_config def scan(): @@ -25,8 +26,20 @@ def scan(): packages = set() print("> Scan:") - reqs = glob("packages/*/*/requirements.txt") + \ - glob("packages/*/*/package.json") + glob("packages/*/*/composer.json") + + # => REQUIREMENTS + default_reqs_globs = ["packages/*/*/requirements.txt", + "packages/*/*/package.json", + "packages/*/*/composer.json" + ] + package_globs = get_nuvolaris_config("requirements", default_reqs_globs) + reqs = list() + + for pkg_glob in package_globs: + items = glob(pkg_glob) + # extend first list without duplicates + reqs.extend(x for x in items if x not in reqs) + # req = reqs[0] # from util.deploy.deploy import * for req in reqs: @@ -36,8 +49,17 @@ def scan(): deployments.add(act) packages.add(sp[1]) - mains = glob("packages/*/*/index.js") + \ - glob("packages/*/*/__main__.py") + glob("packages/*/*/index.php") + # => MAINS + default_mains_globs = ["packages/*/*/index.js", + "packages/*/*/__main__.py", + "packages/*/*/index.php"] + mains_globs = get_nuvolaris_config("mains", default_mains_globs) + mains = list() + for main_glob in mains_globs: + items = glob(main_glob) + # extend first list without duplicates + mains.extend(x for x in items if x not in mains) + # main = mains[2] for main in mains: print(">> Main:", main) @@ -46,8 +68,15 @@ def scan(): deployments.add(act) packages.add(sp[1]) - singles = glob("packages/*/*.py") + \ - glob("packages/*/*.js") + glob("packages/*/*.php") + # => SINGLES + default_singles_globs = ["packages/*/*.py", + "packages/*/*.js", "packages/*/*.php"] + singles_globs = get_nuvolaris_config("singles", default_singles_globs) + singles = list() + for single_glob in singles_globs: + items = glob(single_glob) + singles.extend(x for x in items if x not in singles) + # single = singles[0] for single in singles: print(">> Action:", single) @@ -56,7 +85,6 @@ def scan(): packages.add(sp[1]) print("> Deploying:") - for package in packages: print(">> Package:", package) deploy_package(package) From b49f4979271c0a090587bd0368b0ac3f0bccac49 Mon Sep 17 00:00:00 2001 From: Michele Sciabarra Date: Mon, 1 Jul 2024 10:53:37 +0200 Subject: [PATCH 7/8] go support --- ide/deploy/deploy.py | 2 +- ide/deploy/scan.py | 12 ++-- ide/go/exec.env | 1 + ide/go/launcher.go | 142 +++++++++++++++++++++++++++++++++++++++++++ ide/go/nuvfile.yml | 52 ++++++++++++++++ ide/nuvfile.yml | 3 + ide/nuvopts.txt | 1 + ide/util/nuvfile.yml | 6 ++ 8 files changed, 214 insertions(+), 5 deletions(-) create mode 100644 ide/go/exec.env create mode 100644 ide/go/launcher.go create mode 100644 ide/go/nuvfile.yml diff --git a/ide/deploy/deploy.py b/ide/deploy/deploy.py index f03583f..e698a15 100644 --- a/ide/deploy/deploy.py +++ b/ide/deploy/deploy.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -MAINS = ["__main__.py", "index.js", "index.php"] +MAINS = ["__main__.py", "index.js", "index.php", "main.go"] import os from os.path import exists, isdir diff --git a/ide/deploy/scan.py b/ide/deploy/scan.py index 5382628..4b33250 100644 --- a/ide/deploy/scan.py +++ b/ide/deploy/scan.py @@ -30,8 +30,9 @@ def scan(): # => REQUIREMENTS default_reqs_globs = ["packages/*/*/requirements.txt", "packages/*/*/package.json", - "packages/*/*/composer.json" - ] + "packages/*/*/composer.json", + "packages/*/*/go.mod"] + package_globs = get_nuvolaris_config("requirements", default_reqs_globs) reqs = list() @@ -52,7 +53,8 @@ def scan(): # => MAINS default_mains_globs = ["packages/*/*/index.js", "packages/*/*/__main__.py", - "packages/*/*/index.php"] + "packages/*/*/index.php", + "packages/*/*/main.go"] mains_globs = get_nuvolaris_config("mains", default_mains_globs) mains = list() for main_glob in mains_globs: @@ -70,7 +72,9 @@ def scan(): # => SINGLES default_singles_globs = ["packages/*/*.py", - "packages/*/*.js", "packages/*/*.php"] + "packages/*/*.js", + "packages/*/*.php", + "packages/*/*.go"] singles_globs = get_nuvolaris_config("singles", default_singles_globs) singles = list() for single_glob in singles_globs: diff --git a/ide/go/exec.env b/ide/go/exec.env new file mode 100644 index 0000000..243c411 --- /dev/null +++ b/ide/go/exec.env @@ -0,0 +1 @@ +nuvolaris/runtime-golang-v1.22 \ No newline at end of file diff --git a/ide/go/launcher.go b/ide/go/launcher.go new file mode 100644 index 0000000..01dfcad --- /dev/null +++ b/ide/go/launcher.go @@ -0,0 +1,142 @@ +// DO NOT EDIT - IMPLEMENTS THE RUNTIME PROTOCOL AN SHOULD NOT BE CHANGED +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + package main + + import ( + "bufio" + "bytes" + "encoding/json" + "fmt" + "io" + "log" + "os" + "reflect" + "strings" + ) + + // OwExecutionEnv is the execution environment set at compile time + var OwExecutionEnv = "" + + func main() { + // check if the execution environment is correct + if OwExecutionEnv != "" && OwExecutionEnv != os.Getenv("__OW_EXECUTION_ENV") { + fmt.Println("Execution Environment Mismatch") + fmt.Println("Expected: ", OwExecutionEnv) + fmt.Println("Actual: ", os.Getenv("__OW_EXECUTION_ENV")) + os.Exit(1) + } + + // debugging + var debug = os.Getenv("OW_DEBUG") != "" + if debug { + f, err := os.OpenFile("/tmp/action.log", os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) + if err == nil { + log.SetOutput(f) + } + log.Printf("Environment: %v", os.Environ()) + } + + resultKind := reflect.TypeOf(Main).Out(0).Kind() + if resultKind != reflect.Map && resultKind != reflect.Slice && resultKind != reflect.Array { + fmt.Println("Support map and slice and array only") + os.Exit(1) + } + + // input + out := os.NewFile(3, "pipe") + defer out.Close() + reader := bufio.NewReader(os.Stdin) + + // acknowledgement of started action + fmt.Fprintf(out, `{ "ok": true}%s`, "\n") + if debug { + log.Println("action started") + } + + // read-eval-print loop + for { + // read one line + inbuf, err := reader.ReadBytes('\n') + if err != nil { + if err != io.EOF { + log.Println(err) + } + break + } + if debug { + log.Printf(">>>'%s'>>>", inbuf) + } + // parse one line + var input map[string]interface{} + err = json.Unmarshal(inbuf, &input) + if err != nil { + log.Println(err.Error()) + fmt.Fprintf(out, "{ error: %q}\n", err.Error()) + continue + } + if debug { + log.Printf("%v\n", input) + } + // set environment variables + for k, v := range input { + if k == "value" { + continue + } + if s, ok := v.(string); ok { + os.Setenv("__OW_"+strings.ToUpper(k), s) + } + } + // get payload if not empty + isJsonObjectParam := true + var payloadForJsonObject map[string]interface{} + var payloadForJsonArray []interface{} + if value, ok := input["value"].(map[string]interface{}); ok { + payloadForJsonObject = value + } else { + if value, ok := input["value"].([]interface{}); ok { + payloadForJsonArray = value + isJsonObjectParam = false + } + } + // process the request + var result interface{} + funcMain := reflect.ValueOf(Main) + if isJsonObjectParam { + param := []reflect.Value{reflect.ValueOf(payloadForJsonObject)} + reflectResult := funcMain.Call(param) + result = reflectResult[0].Interface() + } else { + param := []reflect.Value{reflect.ValueOf(payloadForJsonArray)} + reflectResult := funcMain.Call(param) + result = reflectResult[0].Interface() + } + // encode the answer + output, err := json.Marshal(&result) + if err != nil { + log.Println(err.Error()) + fmt.Fprintf(out, "{ error: %q}\n", err.Error()) + continue + } + output = bytes.Replace(output, []byte("\n"), []byte(""), -1) + if debug { + log.Printf("<<<'%s'<<<", output) + } + fmt.Fprintf(out, "%s\n", output) + } + } \ No newline at end of file diff --git a/ide/go/nuvfile.yml b/ide/go/nuvfile.yml new file mode 100644 index 0000000..e490894 --- /dev/null +++ b/ide/go/nuvfile.yml @@ -0,0 +1,52 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +version: '3' + +tasks: + + clean: + silent: true + desc: clean action DIR + requires: {var: [DIR]} + cmds: + - test -d "{{.DIR}}" || die "{{.DIR}} not found or not a directory" + - /usr/bin/rm -v -f {{.DIR}}/main_.go {{.DIR}}.zip {{.DIR}}/exec + + zip: + silent: true + desc: prepare zip for action in DIR + requires: {var: [DIR]} + cmds: + - test -d "{{.DIR}}" || die "{{.DIR}} not found or not a directory" + - /usr/bin/zip -v "{{.DIR}}.zip" exec.env + - /bin/cp -v launcher.go "{{.DIR}}/main_.go" + generates: + - "{{.DIR}}/main_.go" + + action: + deps: [zip] + requires: {var: [DIR]} + silent: true + desc: compile action in DIR + dir: "{{.DIR}}" + cmds: + - test -d "{{.DIR}}" || die "{{.DIR}} not found or not a directory" + - go build -o exec + - /usr/bin/zip -u "{{.DIR}}.zip" exec + + diff --git a/ide/nuvfile.yml b/ide/nuvfile.yml index 805580f..f61ff15 100644 --- a/ide/nuvfile.yml +++ b/ide/nuvfile.yml @@ -266,3 +266,6 @@ tasks: php: desc: php subcommand + + golang: + desc: go subcommand diff --git a/ide/nuvopts.txt b/ide/nuvopts.txt index bce008f..f20fc7b 100644 --- a/ide/nuvopts.txt +++ b/ide/nuvopts.txt @@ -29,3 +29,4 @@ Commands: ide python python subcommands ide nodejs nodejs subcommands ide php php subcommands + ide go go subcommand diff --git a/ide/util/nuvfile.yml b/ide/util/nuvfile.yml index 6c01e3a..dd2f122 100644 --- a/ide/util/nuvfile.yml +++ b/ide/util/nuvfile.yml @@ -32,6 +32,8 @@ tasks: then nuv ide nodejs zip DIR="$(realpath "packages/{{.A}}")" elif test -e "packages/{{.A}}/composer.json" then nuv ide php zip DIR="$(realpath "packages/{{.A}}")" + elif test -e "packages/{{.A}}/go.mod" + then nuv ide go zip DIR="$(realpath "packages/{{.A}}")" else echo "no zip environment" fi @@ -48,6 +50,8 @@ tasks: then nuv ide nodejs action DIR="$(realpath "packages/{{.A}}")" elif test -e "packages/{{.A}}/index.php" then nuv ide php action DIR="$(realpath "packages/{{.A}}")" + elif test -e "packages/{{.A}}/main.go" + then nuv ide go action DIR="$(realpath "packages/{{.A}}")" else die "*** unknow action type" fi @@ -64,6 +68,8 @@ tasks: then nuv ide nodejs clean DIR="$(realpath "packages/{{.A}}")" elif test -d "packages/{{.A}}/vendor" then nuv ide php clean DIR="$(realpath "packages/{{.A}}")" + elif test -e "packages/{{.A}}/_main.go" + then nuv ide go clean DIR="$(realpath "packages/{{.A}}")" else echo "nothing to clean" fi if test -e "packages/{{.A}}.zip" From 984b82da42766d387e7b29492315c03810f54bcc Mon Sep 17 00:00:00 2001 From: d4rkstar Date: Tue, 9 Jul 2024 01:39:10 +0200 Subject: [PATCH 8/8] feat: moved config to module. added pydoc strings Created a config.py module and moved there all the common defaults params. Added pydoc fucntions documentation --- ide/deploy/client.py | 59 ++++++++++++++++++++---------- ide/deploy/config.py | 65 +++++++++++++++++++++++++++++++++ ide/deploy/deploy.py | 87 +++++++++++++++++++++++++++++++++++++------- ide/deploy/scan.py | 51 +++++++++++++++++--------- ide/deploy/watch.py | 39 +++++++++++++++----- 5 files changed, 239 insertions(+), 62 deletions(-) create mode 100644 ide/deploy/config.py diff --git a/ide/deploy/client.py b/ide/deploy/client.py index 21631fd..895df97 100644 --- a/ide/deploy/client.py +++ b/ide/deploy/client.py @@ -15,47 +15,66 @@ # specific language governing permissions and limitations # under the License. -from pathlib import Path from subprocess import Popen, PIPE -import os, os.path, json +import os +import os.path import threading -import asyncio - -def get_nuvolaris_config(key, default): - try: - dir = os.environ.get("NUV_PWD", "/do_not_exists") - file = f"{dir}/package.json" - info = json.loads(Path(file).read_text()) - return info.get("nuvolaris", {}).get(key, default) - except: - return default - -def readlines(inp): +from typing import IO +from .config import get_nuvolaris_config + + +def readlines(inp: IO[str]): + """Read line from an file descriptor + + Args: + inp (IO[str]): the file descriptor + """ for line in iter(inp.readline, ''): print(line, end='') # serve web area -def launch(key, default): + + +def launch(key: str, default: (str | list)): + """Launch a command in a process, reading. + The command is extracted from nuvolaris config in package.json + or a default command is used + + Args: + key (str): the key from which read the command + default (str | list): the default if the key is not found + """ cmd = get_nuvolaris_config(key, default) proc = Popen( - cmd, shell=True, - cwd=os.environ.get("NUV_PWD"), env=os.environ, + cmd, shell=True, + cwd=os.environ.get("NUV_PWD"), env=os.environ, stdin=PIPE, stdout=PIPE, stderr=PIPE, text=True ) threading.Thread(target=readlines, args=(proc.stdout,)).start() threading.Thread(target=readlines, args=(proc.stderr,)).start() -def serve(): + +def serve(): + """Serve the web area + """ launch("devel", "nuv ide serve") -def logs(): + +def logs(): + """Serve the openwhisk activation's logs + """ launch("logs", "nuv activation poll") # build + + def build(): + """Try to build the frontend application, if the deploy command is set. + + """ deploy = get_nuvolaris_config("deploy", "true") proc = Popen( - deploy, shell=True, + deploy, shell=True, env=os.environ, cwd=os.environ.get("NUV_PWD"), stdin=PIPE, stdout=PIPE, stderr=PIPE, text=True diff --git a/ide/deploy/config.py b/ide/deploy/config.py new file mode 100644 index 0000000..e060e16 --- /dev/null +++ b/ide/deploy/config.py @@ -0,0 +1,65 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import os +import json +from pathlib import Path + +MAINS = ["__main__.py", + "index.js", + "index.php", + "main.go"] + +SKIPDIR = ["virtualenv", + "node_modules", + "__pycache__"] + +DEFAULT_REQ_GLOBS = ["packages/*/*/requirements.txt", + "packages/*/*/package.json", + "packages/*/*/composer.json", + "packages/*/*/go.mod"] + +DEFAULT_MAIN_GLOBS = ["packages/*/*/index.js", + "packages/*/*/__main__.py", + "packages/*/*/index.php", + "packages/*/*/main.go"] + +DEFAULT_SINGLES_GLOBS = ["packages/*/*.py", + "packages/*/*.js", + "packages/*/*.php", + "packages/*/*.go"] + + +def get_nuvolaris_config(key: str, default: list | str) -> (list | str): + """Read package.json if exists and retrieve the required + value for passed key (if defined) + + Args: + key (str): the key of the parameter to retrieve + default (list | str): the default value to return + + Returns: + _type_: the list of values or the single string value. + if not defined, the default value is returned + """ + try: + dir = os.environ.get("NUV_PWD", "/do_not_exists") + file = f"{dir}/package.json" + info = json.loads(Path(file).read_text()) + return info.get("nuvolaris", {}).get(key, default) + except: + return default diff --git a/ide/deploy/deploy.py b/ide/deploy/deploy.py index e698a15..9baa9fc 100644 --- a/ide/deploy/deploy.py +++ b/ide/deploy/deploy.py @@ -15,28 +15,49 @@ # specific language governing permissions and limitations # under the License. -MAINS = ["__main__.py", "index.js", "index.php", "main.go"] - import os -from os.path import exists, isdir +from os.path import exists, isdir from subprocess import Popen +from .config import MAINS dry_run = False -def set_dry_run(b): + +def set_dry_run(b: bool): + """Set global dry run + + Args: + b (bool): true for dry run enabled + """ global dry_run dry_run = b -def exec(cmd): + +def exec(cmd: str): + """Exec a shell command and wait for it to complete. + If dryrun is set, the command is not executed. + + Args: + cmd (str): command line to execue + """ global dry_run print("$", cmd) if not dry_run: Popen(cmd, shell=True, env=os.environ).wait() -def extract_args(files): + +def extract_args(files: list) -> list: + """Extract openwhisk args from files + + Args: + files (list): the list of files to inspect + + Returns: + list: a list of parameters + """ res = [] for file in files: - #if dry_run: + # if dry_run: # print(f": inspecting {file}") if exists(file): with open(file, "r") as f: @@ -47,9 +68,16 @@ def extract_args(files): res.append(line.strip()[2:]) return res + package_done = set() -def deploy_package(package): + +def deploy_package(package: str): + """Deploy a package on nuvolaris + + Args: + package (str): the name of package + """ global package_done # package args ppath = f"packages/{package}.args" @@ -59,15 +87,38 @@ def deploy_package(package): exec(cmd) package_done.add(cmd) -def build_zip(package, action): + +def build_zip(package: str, action: str) -> str: + """Builds a zip for the package / action + + Args: + package (str): package + action (str): action + + Returns: + str: the path of the built zip file + """ exec(f"nuv ide util zip A={package}/{action}") return f"packages/{package}/{action}.zip" -def build_action(package, action): + +def build_action(package: str, action: str) -> str: + """Invoke the nuv ide util action command on package / action + + Args: + package (_type_): _description_ + action (_type_): _description_ + + Returns: + str: the zip with the built action + """ exec(f"nuv ide util action A={package}/{action}") return f"packages/{package}/{action}.zip" -def deploy_action(artifact): + +def deploy_action(artifact: str): + """Deploy an artifact calling nuv action update + """ try: sp = artifact.split("/") [name, typ] = sp[-1].rsplit(".", 1) @@ -83,18 +134,26 @@ def deploy_action(artifact): to_inspect = [f"{base}/{x}" for x in MAINS] else: to_inspect = [artifact] - + args = " ".join(extract_args(to_inspect)) exec(f"nuv action update {package}/{name} {artifact} {args}") + """ file = "packages/deploy/hello.py" file = "packages/deploy/multi.zip" file = "packages/deploy/multi/__main__.py" file = "packages/deploy/multi/requirements.txt" """ -def deploy(file): - #print(f"*** {file}") + + +def deploy(file: str): + """Deploy a package on nuvolaris + + Args: + file (str): the file to deploy + """ + # print(f"*** {file}") if isdir(file): for start in MAINS: sub = f"{file}/{start}" diff --git a/ide/deploy/scan.py b/ide/deploy/scan.py index 4b33250..0be87e7 100644 --- a/ide/deploy/scan.py +++ b/ide/deploy/scan.py @@ -18,22 +18,44 @@ from glob import glob from .deploy import * from .client import get_nuvolaris_config +from .config import DEFAULT_REQ_GLOBS, DEFAULT_MAIN_GLOBS, DEFAULT_SINGLES_GLOBS def scan(): + """This function has two stages:\n + Scan + === + A) Packages requirements\n + Check for package requirements. It will look in a specific requirement file + (from web package.json) or a default set of requirements (requirements.txt, + composer.json etc. etc).\n + For each requirement found, a zip is built calling the task: + ``nuv ide util zip A={package}/{action}``\n + The zip is added to actions (deployments); the directory is added to + packages (packages)\n + B) Mains for packages\n + After that, the scan continue checking the "mains". For each main, an + action is built calling the task:\n + ``nuv ide util action A={package}/{action}``\n + where package is the base directory of main file and the action is the + basename of the main file.\n + C) Singles\n + Finally the scan will take care of single file functions\n + + Deploy + === + Each package is deployed with the command:\n + ``nuv package update {package} {pargs}``\n + Each deployment is deployed with the command:\n + ``nuv action update {package}/{name} {artifact} {args}``\n + """ + # first look for requirements.txt and build the venv (add in set) deployments = set() packages = set() print("> Scan:") - - # => REQUIREMENTS - default_reqs_globs = ["packages/*/*/requirements.txt", - "packages/*/*/package.json", - "packages/*/*/composer.json", - "packages/*/*/go.mod"] - - package_globs = get_nuvolaris_config("requirements", default_reqs_globs) + package_globs = get_nuvolaris_config("requirements", DEFAULT_REQ_GLOBS) reqs = list() for pkg_glob in package_globs: @@ -51,11 +73,7 @@ def scan(): packages.add(sp[1]) # => MAINS - default_mains_globs = ["packages/*/*/index.js", - "packages/*/*/__main__.py", - "packages/*/*/index.php", - "packages/*/*/main.go"] - mains_globs = get_nuvolaris_config("mains", default_mains_globs) + mains_globs = get_nuvolaris_config("mains", DEFAULT_MAIN_GLOBS) mains = list() for main_glob in mains_globs: items = glob(main_glob) @@ -71,11 +89,8 @@ def scan(): packages.add(sp[1]) # => SINGLES - default_singles_globs = ["packages/*/*.py", - "packages/*/*.js", - "packages/*/*.php", - "packages/*/*.go"] - singles_globs = get_nuvolaris_config("singles", default_singles_globs) + + singles_globs = get_nuvolaris_config("singles", DEFAULT_SINGLES_GLOBS) singles = list() for single_glob in singles_globs: items = glob(single_glob) diff --git a/ide/deploy/watch.py b/ide/deploy/watch.py index 99d4234..7559958 100644 --- a/ide/deploy/watch.py +++ b/ide/deploy/watch.py @@ -15,48 +15,66 @@ # specific language governing permissions and limitations # under the License. -SKIPDIR = ["virtualenv", "node_modules", "__pycache__"] - +from typing import Tuple import watchfiles + import asyncio import os.path import signal - +from .config import SKIPDIR from .deploy import deploy from .client import serve, logs -def check_and_deploy(change): + +def check_and_deploy(change: Tuple[watchfiles.Change, str]): + """Deploy a file if the received change is a file modification + + Args: + change (Tuple[watchfiles.Change, str]): the change type and filename + """ change_type, path = change path = os.path.abspath(path) cur_dir_len = len(os.getcwd())+1 src = path[cur_dir_len:] # only modified - if change_type != watchfiles.Change.modified: return + if change_type != watchfiles.Change.modified: + return # no directories - if os.path.isdir(src): return + if os.path.isdir(src): + return # no missing files - if not os.path.exists(src): return + if not os.path.exists(src): + return # no generated directories for dir in src.split("/")[:-1]: - if dir in SKIPDIR: return + if dir in SKIPDIR: + return # no generated files - if src.endswith(".zip"): return + if src.endswith(".zip"): + return # now you can deploy deploy(src) + async def redeploy(): + """Install the filesystem watcher over the packages directory + """ print("> Watching:") iterator = watchfiles.awatch("packages", recursive=True) async for changes in iterator: for change in changes: try: - #print(change) + # print(change) check_and_deploy(change) except Exception as e: print(e) + def watch(): + """This function will start the webserver and show logs. + After that it will watch the filesystem in a loop, until SIGTERM received. + """ # start web server serve() # show logs @@ -64,6 +82,7 @@ def watch(): loop = asyncio.get_event_loop() task = loop.create_task(redeploy()) + def end_loop(): print("Ending task.") task.cancel()