You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by ka...@apache.org on 2023/05/02 07:17:14 UTC
[incubator-devlake] branch main updated: 4986 refactor plugin registration (#5057)
This is an automated email from the ASF dual-hosted git repository.
ka94 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 3554199fc 4986 refactor plugin registration (#5057)
3554199fc is described below
commit 3554199fcf5d194db3877698ee645468cb45d61a
Author: Camille Teruel <ca...@gmail.com>
AuthorDate: Tue May 2 09:17:09 2023 +0200
4986 refactor plugin registration (#5057)
* fix: Replace usage of `match`
We downgraded to python 3.9 that doesn't support `match` statement.
* refactor: Generate swagger doc on go side
The generation of a remote plugin swagger documentation from a template is now
done on go side.
* refactor: Use shell scripts to build and run remote plugins
* refactor: Load remote plugins via `run.sh plugin-info`
* Remove /plugin/register endpoint
* Remote plugins are loaded by looking for `run.sh` files in `REMOTE_PLUGIN_DIR` in loader.go
---------
Co-authored-by: Camille Teruel <ca...@meri.co>
---
.licenserc.yaml | 2 +-
backend/Makefile | 13 ++-
backend/core/config/config_viper.go | 3 +-
.../plugin/plugin_openapi_spec.go} | 23 ++---
backend/core/runner/loader.go | 56 ++++++++++-
backend/python/README.md | 18 ++++
backend/python/build.sh | 36 +++----
.../start.sh => plugins/azuredevops/build.sh} | 18 ++--
.../plugins/{start.sh => azuredevops/run.sh} | 16 +--
backend/python/pydevlake/pydevlake/docgen.py | 39 --------
backend/python/pydevlake/pydevlake/ipc.py | 3 -
backend/python/pydevlake/pydevlake/message.py | 11 ---
backend/python/pydevlake/pydevlake/plugin.py | 25 ++---
backend/python/run_tests.sh | 0
.../{build_tests.sh => test/fakeplugin/build.sh} | 22 +----
backend/python/test/fakeplugin/fakeplugin/main.py | 22 ++---
.../python/test/fakeplugin/{start.sh => run.sh} | 18 ++--
backend/server/api/api.go | 74 +++++++-------
backend/server/api/remote/register.go | 107 ---------------------
backend/server/services/remote/bridge/bootstrap.go | 63 ------------
backend/server/services/remote/bridge/cmd.go | 13 ++-
.../server/services/remote/bridge/python_cmd.go | 38 --------
.../server/services/remote/models/plugin_remote.go | 1 +
.../server/services/remote/plugin/doc/open_api.go | 83 ++++++++++++++++
.../remote/plugin/doc/open_api_spec.json.tmpl} | 20 ++--
backend/server/services/remote/plugin/init.go | 2 +-
.../server/services/remote/plugin/plugin_impl.go | 11 +++
backend/server/services/remote/run/run.go | 2 +-
backend/test/e2e/remote/helper.go | 14 +--
env.example | 1 +
30 files changed, 292 insertions(+), 462 deletions(-)
diff --git a/.licenserc.yaml b/.licenserc.yaml
index 0bb01d9a4..e2da52363 100644
--- a/.licenserc.yaml
+++ b/.licenserc.yaml
@@ -33,7 +33,7 @@ header:
- '**/env.example'
- '**/*.csv'
- '**/*.json'
- - '**/*.template.json'
+ - '**/*.json.tmpl'
- '**/*.sql'
- '**/*.svg'
- '**/*.png'
diff --git a/backend/Makefile b/backend/Makefile
index 8b0a4d02b..a13908eab 100644
--- a/backend/Makefile
+++ b/backend/Makefile
@@ -52,8 +52,7 @@ build-server: swag
build-python: #don't mix this with the other build commands
find ./python/ -name "*.sh" | xargs chmod +x &&\
- ./python/build.sh &&\
- ./python/build_tests.sh
+ sh python/build.sh
build: build-plugin build-server
@@ -69,7 +68,7 @@ run:
worker:
go run worker/*.go
-dev: build-plugin run
+dev: build-plugin build-python run
debug: build-plugin-debug
dlv debug server/main.go
@@ -86,8 +85,12 @@ unit-test: mock unit-test-only python-unit-test
unit-test-only:
set -e; for m in $$(go list ./... | egrep -v 'test|models|e2e'); do echo $$m; go test -timeout 60s -v $$m; done
-python-unit-test:
- sh ./python/build.sh && sh ./python/run_tests.sh
+build-pydevlake:
+ poetry install -C python/pydevlake
+
+python-unit-test: build-pydevlake
+ sh python/build.sh test &&\
+ sh ./python/run_tests.sh
e2e-plugins-test:
export ENV_PATH=$(shell readlink -f .env); set -e; for m in $$(go list ./plugins/... | egrep 'e2e'); do echo $$m; go test -timeout 300s -gcflags=all=-l -v $$m; done
diff --git a/backend/core/config/config_viper.go b/backend/core/config/config_viper.go
index be3106f93..c05165b57 100644
--- a/backend/core/config/config_viper.go
+++ b/backend/core/config/config_viper.go
@@ -75,8 +75,7 @@ func setDefaultValue(v *viper.Viper) {
v.SetDefault("PLUGIN_DIR", "bin/plugins")
v.SetDefault("TEMPORAL_TASK_QUEUE", "DEVLAKE_TASK_QUEUE")
v.SetDefault("TAP_PROPERTIES_DIR", "resources/tap")
- v.SetDefault("ENABLE_REMOTE_PLUGINS", "true")
- v.SetDefault("REMOTE_PLUGINS_STARTUP_PATH", "python/plugins/start.sh")
+ v.SetDefault("REMOTE_PLUGIN_DIR", "python/plugins")
}
// replaceNewEnvItemInOldContent replace old config to new config in env file content
diff --git a/backend/server/api/remote/models.go b/backend/core/plugin/plugin_openapi_spec.go
similarity index 60%
rename from backend/server/api/remote/models.go
rename to backend/core/plugin/plugin_openapi_spec.go
index 079f27cdf..b3c3b2b99 100644
--- a/backend/server/api/remote/models.go
+++ b/backend/core/plugin/plugin_openapi_spec.go
@@ -15,21 +15,12 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
-package remote
+package plugin
-import (
- "encoding/json"
-
- "github.com/apache/incubator-devlake/server/services/remote/models"
-)
-
-type SwaggerDoc struct {
- Name string `json:"name" validate:"required"`
- Resource string `json:"resource" validate:"required"`
- Spec json.RawMessage `json:"spec" validate:"required"`
-}
-
-type PluginDetails struct {
- PluginInfo models.PluginInfo `json:"plugin_info" validate:"required"`
- Swagger SwaggerDoc `json:"swagger" validate:"required"`
+// PluginApiSpec let a plugin document its API with an OpenAPI spec.
+// This is useful for remote plugins because whose standardized API spec template
+// need to be instantiated with remote plugin's specific schemas,
+// something that is not supported by swaggo.
+type PluginOpenApiSpec interface {
+ OpenApiSpec() string
}
diff --git a/backend/core/runner/loader.go b/backend/core/runner/loader.go
index 5c1a7c7a4..327aafe7c 100644
--- a/backend/core/runner/loader.go
+++ b/backend/core/runner/loader.go
@@ -22,22 +22,30 @@ import (
"io/fs"
"path/filepath"
goplugin "plugin"
- "strconv"
"strings"
"github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/server/services/remote"
+ "github.com/apache/incubator-devlake/server/services/remote/bridge"
+ "github.com/apache/incubator-devlake/server/services/remote/models"
)
// LoadPlugins load plugins from local directory
func LoadPlugins(basicRes context.BasicRes) errors.Error {
- remote_plugins_enabled, err := strconv.ParseBool(basicRes.GetConfig("ENABLE_REMOTE_PLUGINS"))
- if err == nil && remote_plugins_enabled {
- remote.Init(basicRes)
+ err := LoadGoPlugins(basicRes)
+ if err != nil {
+ return err
+ }
+ err = LoadRemotePlugins(basicRes)
+ if err != nil {
+ return err
}
+ return nil
+}
+func LoadGoPlugins(basicRes context.BasicRes) errors.Error {
pluginsDir := basicRes.GetConfig("PLUGIN_DIR")
walkErr := filepath.WalkDir(pluginsDir, func(path string, d fs.DirEntry, err error) error {
if err != nil {
@@ -66,7 +74,7 @@ func LoadPlugins(basicRes context.BasicRes) errors.Error {
}
err = plugin.RegisterPlugin(pluginName, pluginMeta)
if err != nil {
- return nil
+ return err
}
basicRes.GetLogger().Info(`plugin loaded %s`, pluginName)
@@ -75,3 +83,41 @@ func LoadPlugins(basicRes context.BasicRes) errors.Error {
})
return errors.Convert(walkErr)
}
+
+func LoadRemotePlugins(basicRes context.BasicRes) errors.Error {
+ remotePluginDir := basicRes.GetConfig("REMOTE_PLUGIN_DIR")
+ if remotePluginDir != "" {
+ basicRes.GetLogger().Info("Loading remote plugins")
+ remote.Init(basicRes)
+ walkErr := filepath.WalkDir(remotePluginDir, func(path string, d fs.DirEntry, err error) error {
+ if err != nil {
+ return err
+ }
+ fileName := d.Name()
+ if fileName == "run.sh" {
+ invoker := bridge.NewCmdInvoker(path)
+ result := invoker.Call("plugin-info", bridge.DefaultContext)
+ if result.Err != nil {
+ return errors.Default.Wrap(result.Err, "Error calling plugin-info")
+ }
+ pluginInfo := &models.PluginInfo{}
+ err := result.Get(pluginInfo)
+ if err != nil {
+ return err
+ }
+ remotePlugin, err := remote.NewRemotePlugin(pluginInfo)
+ if err != nil {
+ return err
+ }
+ err = plugin.RegisterPlugin(pluginInfo.Name, remotePlugin)
+ if err != nil {
+ return err
+ }
+ basicRes.GetLogger().Info(`remote plugin loaded %s`, pluginInfo.Name)
+ }
+ return nil
+ })
+ return errors.Convert(walkErr)
+ }
+ return nil
+}
diff --git a/backend/python/README.md b/backend/python/README.md
index 062f6053e..db99ee5f7 100644
--- a/backend/python/README.md
+++ b/backend/python/README.md
@@ -74,6 +74,24 @@ It specifies three datatypes:
The plugin class declares what are its connection, transformation rule and tool scope types.
It also declares its list of streams, and is responsible to define 4 methods that we'll cover hereafter.
+We also need to create two shell scripts in the plugin root directory to build and run the plugin.
+Create a `build.sh` file with the following content:
+
+```bash
+#!/bin/bash
+
+cd "$(dirname "$0")"
+poetry install
+```
+
+And a `run.sh` file with the following content:
+
+```bash
+#!/bin/bash
+
+cd "$(dirname "$0")"
+poetry run python myplugin/main.py "$@"
+```
### Connection parameters
diff --git a/backend/python/build.sh b/backend/python/build.sh
old mode 100644
new mode 100755
index c8f19d9f2..b61cb8c6d
--- a/backend/python/build.sh
+++ b/backend/python/build.sh
@@ -16,26 +16,18 @@
# limitations under the License.
#
-build() {
- python_dir=$1
- cd "$python_dir" &&\
- poetry install &&\
- cd -
- exit_code=$?
- if [ $exit_code != 0 ]; then
- exit $exit_code
- fi
-}
+if [ -z "$1" ]; then
+ cd "$(dirname "$0")"
+ cd plugins
+else
+ cd "$1"
+fi
-cd "${0%/*}" # make sure we're in the correct dir
-
-poetry config virtualenvs.create true
-
-echo "Installing Python DevLake framework"
-build pydevlake
-
-for plugin_dir in $(ls -d plugins/*/*.toml); do
- plugin_dir=$(dirname $plugin_dir)
- echo "Installing dependencies of python plugin in: $plugin_dir" &&\
- build "$plugin_dir"
-done
\ No newline at end of file
+for plugin_dir in $(ls -d */build.sh); do
+ echo "Building remote plugin: $plugin_dir" &&\
+ $plugin_dir
+ exit_code=$?
+ if [ $exit_code != 0 ]; then
+ exit $exit_code
+ fi
+done
diff --git a/backend/python/test/fakeplugin/start.sh b/backend/python/plugins/azuredevops/build.sh
similarity index 77%
copy from backend/python/test/fakeplugin/start.sh
copy to backend/python/plugins/azuredevops/build.sh
index b49ca6e52..f0db2fed0 100755
--- a/backend/python/test/fakeplugin/start.sh
+++ b/backend/python/plugins/azuredevops/build.sh
@@ -1,26 +1,20 @@
#!/bin/sh
+#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
-
+#
# http://www.apache.org/licenses/LICENSE-2.0
-
+#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+#
-endpoint="$1"
-
-cd "${0%/*}" # make sure we're in the correct dir
-
-echo "Registering fake python plugin"
-poetry run python fakeplugin/main.py startup "$endpoint"
-exit_code=$?
-if [ $exit_code != 0 ]; then
- exit $exit_code
-fi
+cd "$(dirname "$0")"
+poetry install
diff --git a/backend/python/plugins/start.sh b/backend/python/plugins/azuredevops/run.sh
similarity index 72%
rename from backend/python/plugins/start.sh
rename to backend/python/plugins/azuredevops/run.sh
index f5cdd9cf2..e1ea5dd32 100755
--- a/backend/python/plugins/start.sh
+++ b/backend/python/plugins/azuredevops/run.sh
@@ -16,17 +16,5 @@
# limitations under the License.
#
-endpoint="$1"
-
-cd "${0%/*}" # make sure we're in the correct dir
-
-for plugin_dir in $(ls -d */*.toml); do
- plugin_dir=$(dirname $plugin_dir)
- cd $plugin_dir &&\
- poetry run python $plugin_dir/main.py startup "$endpoint" &&\
- cd -
- exit_code=$?
- if [ $exit_code != 0 ]; then
- exit $exit_code
- fi
-done
\ No newline at end of file
+cd "$(dirname "$0")"
+poetry run python azuredevops/main.py "$@"
diff --git a/backend/python/pydevlake/pydevlake/docgen.py b/backend/python/pydevlake/pydevlake/docgen.py
deleted file mode 100644
index 7bc0a5c79..000000000
--- a/backend/python/pydevlake/pydevlake/docgen.py
+++ /dev/null
@@ -1,39 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-
-# http://www.apache.org/licenses/LICENSE-2.0
-
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-from typing import Type
-from pathlib import Path
-from string import Template
-import json
-
-from pydevlake.model import Connection, TransformationRule
-
-
-# TODO: Move swagger documentation generation to GO side along with API implementation
-TEMPLATE_PATH = str(Path(__file__).parent / 'doc.template.json')
-
-def generate_doc(plugin_name: str,
- connection_type: Type[Connection],
- transformation_rule_type: Type[TransformationRule]):
- with open(TEMPLATE_PATH, 'r') as f:
- doc_template = Template(f.read())
- connection_schema = connection_type.schema_json()
- transformation_rule_schema = transformation_rule_type.schema_json() if transformation_rule_type else {}
- doc = doc_template.substitute(
- plugin_name=plugin_name,
- connection_schema=connection_schema,
- transformation_rule_schema=transformation_rule_schema)
- return json.loads(doc)
diff --git a/backend/python/pydevlake/pydevlake/ipc.py b/backend/python/pydevlake/pydevlake/ipc.py
index 7e5ccc56c..7acd95391 100644
--- a/backend/python/pydevlake/pydevlake/ipc.py
+++ b/backend/python/pydevlake/pydevlake/ipc.py
@@ -110,9 +110,6 @@ class PluginCommands:
c = self._plugin.connection_type(**connection)
return self._plugin.make_remote_scopes(c, group_id)
- def startup(self, endpoint: str):
- self._plugin.startup(endpoint)
-
def _mk_context(self, data: dict):
db_url = data['db_url']
scope_dict = data['scope']
diff --git a/backend/python/pydevlake/pydevlake/message.py b/backend/python/pydevlake/pydevlake/message.py
index a0d82eece..7bdda2658 100644
--- a/backend/python/pydevlake/pydevlake/message.py
+++ b/backend/python/pydevlake/pydevlake/message.py
@@ -66,17 +66,6 @@ class PluginInfo(Message):
type: str = "python-poetry"
-class SwaggerDoc(Message):
- name: str
- resource: str
- spec: dict
-
-
-class PluginDetails(Message):
- plugin_info: PluginInfo
- swagger: SwaggerDoc
-
-
class RemoteProgress(Message):
increment: int = 0
current: int = 0
diff --git a/backend/python/pydevlake/pydevlake/plugin.py b/backend/python/pydevlake/pydevlake/plugin.py
index f16ff94f3..d14b9ad5d 100644
--- a/backend/python/pydevlake/pydevlake/plugin.py
+++ b/backend/python/pydevlake/pydevlake/plugin.py
@@ -14,15 +14,15 @@
from typing import Type, Union, Iterable, Optional
-import sys
from abc import ABC, abstractmethod
-import requests
+from pathlib import Path
+import os
+import sys
import fire
import pydevlake.message as msg
from pydevlake.subtasks import Subtask
-from pydevlake.docgen import generate_doc
from pydevlake.ipc import PluginCommands
from pydevlake.context import Context
from pydevlake.stream import Stream, DomainType
@@ -203,19 +203,6 @@ class Plugin(ABC):
raise Exception(f'Unkown stream {stream_name}')
return stream
- def startup(self, endpoint: str):
- details = msg.PluginDetails(
- plugin_info=self.plugin_info(),
- swagger=msg.SwaggerDoc(
- name=self.name,
- resource=self.name,
- spec=generate_doc(self.name, self.connection_type, self.transformation_rule_type)
- )
- )
- resp = requests.post(f"{endpoint}/plugins/register", data=details.json())
- if resp.status_code != 200:
- raise Exception(f"unexpected http status code {resp.status_code}: {resp.content}")
-
def plugin_info(self) -> msg.PluginInfo:
subtask_metas = [
msg.SubtaskMeta(
@@ -249,7 +236,11 @@ class Plugin(ABC):
def _plugin_path(self):
module_name = type(self).__module__
module = sys.modules[module_name]
- return module.__file__
+ pluginMainPath = Path(module.__file__)
+ run_sh_path = pluginMainPath.parent.parent / "run.sh"
+ assert run_sh_path.exists(), f"run.sh not found at {run_sh_path.parent}"
+ assert os.access(run_sh_path, os.X_OK), f"run.sh is not executable"
+ return str(run_sh_path)
@classmethod
def start(cls):
diff --git a/backend/python/run_tests.sh b/backend/python/run_tests.sh
old mode 100644
new mode 100755
diff --git a/backend/python/build_tests.sh b/backend/python/test/fakeplugin/build.sh
similarity index 65%
rename from backend/python/build_tests.sh
rename to backend/python/test/fakeplugin/build.sh
index 59560b867..f0db2fed0 100755
--- a/backend/python/build_tests.sh
+++ b/backend/python/test/fakeplugin/build.sh
@@ -16,23 +16,5 @@
# limitations under the License.
#
-build() {
- python_dir=$1
- cd "$python_dir" &&\
- poetry install &&\
- cd -
- exit_code=$?
- if [ $exit_code != 0 ]; then
- exit $exit_code
- fi
-}
-
-cd "${0%/*}" # make sure we're in the correct dir
-
-poetry config virtualenvs.create true
-
-for plugin_dir in $(find test/ -name "*.toml"); do
- plugin_dir=$(dirname $plugin_dir)
- echo "Building Python test plugin in: $plugin_dir" &&\
- build "$plugin_dir"
-done
\ No newline at end of file
+cd "$(dirname "$0")"
+poetry install
diff --git a/backend/python/test/fakeplugin/fakeplugin/main.py b/backend/python/test/fakeplugin/fakeplugin/main.py
index a14c85d68..6a048ff22 100644
--- a/backend/python/test/fakeplugin/fakeplugin/main.py
+++ b/backend/python/test/fakeplugin/fakeplugin/main.py
@@ -63,20 +63,18 @@ class FakePipelineStream(Stream):
)
def convert_status(self, state: FakePipeline.State):
- match state:
- case FakePipeline.State.FAILURE | FakePipeline.State.SUCCESS:
- return CICDStatus.DONE
- case _:
- return CICDStatus.IN_PROGRESS
+ if state == FakePipeline.State.FAILURE or state == FakePipeline.State.SUCCESS:
+ return CICDStatus.DONE
+ else:
+ return CICDStatus.IN_PROGRESS
def convert_result(self, state: FakePipeline.State):
- match state:
- case FakePipeline.State.SUCCESS:
- return CICDResult.SUCCESS
- case FakePipeline.State.FAILURE:
- return CICDResult.FAILURE
- case _:
- return None
+ if state == FakePipeline.State.SUCCESS:
+ return CICDResult.SUCCESS
+ elif state == FakePipeline.State.FAILURE:
+ return CICDResult.FAILURE
+ else:
+ return None
def duration(self, pipeline: FakePipeline):
if pipeline.finished_at:
diff --git a/backend/python/test/fakeplugin/start.sh b/backend/python/test/fakeplugin/run.sh
similarity index 77%
rename from backend/python/test/fakeplugin/start.sh
rename to backend/python/test/fakeplugin/run.sh
index b49ca6e52..6d287d6ba 100755
--- a/backend/python/test/fakeplugin/start.sh
+++ b/backend/python/test/fakeplugin/run.sh
@@ -1,26 +1,20 @@
#!/bin/sh
+#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
-
+#
# http://www.apache.org/licenses/LICENSE-2.0
-
+#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+#
-endpoint="$1"
-
-cd "${0%/*}" # make sure we're in the correct dir
-
-echo "Registering fake python plugin"
-poetry run python fakeplugin/main.py startup "$endpoint"
-exit_code=$?
-if [ $exit_code != 0 ]; then
- exit $exit_code
-fi
+cd "$(dirname "$0")"
+poetry run python fakeplugin/main.py "$@"
diff --git a/backend/server/api/api.go b/backend/server/api/api.go
index c0d4c1f08..390821d38 100644
--- a/backend/server/api/api.go
+++ b/backend/server/api/api.go
@@ -19,29 +19,28 @@ package api
import (
"fmt"
- "github.com/apache/incubator-devlake/server/api/login"
- "github.com/apache/incubator-devlake/server/api/ping"
- "github.com/apache/incubator-devlake/server/api/version"
- "github.com/apache/incubator-devlake/server/services/auth"
"net/http"
"strconv"
"strings"
"time"
+ "github.com/gin-contrib/cors"
+ "github.com/gin-gonic/gin"
+ ginSwagger "github.com/swaggo/gin-swagger"
+ "github.com/swaggo/gin-swagger/swaggerFiles"
+ "github.com/swaggo/swag"
+
"github.com/apache/incubator-devlake/core/config"
"github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/impls/logruslog"
_ "github.com/apache/incubator-devlake/server/api/docs"
- "github.com/apache/incubator-devlake/server/api/remote"
+ "github.com/apache/incubator-devlake/server/api/login"
+ "github.com/apache/incubator-devlake/server/api/ping"
"github.com/apache/incubator-devlake/server/api/shared"
+ "github.com/apache/incubator-devlake/server/api/version"
"github.com/apache/incubator-devlake/server/services"
- "github.com/apache/incubator-devlake/server/services/remote/bridge"
-
- "github.com/gin-contrib/cors"
- "github.com/gin-gonic/gin"
- "github.com/spf13/viper"
- ginSwagger "github.com/swaggo/gin-swagger"
- "github.com/swaggo/gin-swagger/swaggerFiles"
+ "github.com/apache/incubator-devlake/server/services/auth"
)
const DB_MIGRATION_REQUIRED = `
@@ -73,12 +72,6 @@ func CreateApiService() {
// For both protected and unprotected routes
router.GET("/ping", ping.Get)
router.GET("/version", version.Get)
- // Check if remote plugins are enabled
- remotePluginsEnabled := v.GetBool("ENABLE_REMOTE_PLUGINS")
- if remotePluginsEnabled {
- // Add endpoint to register remote plugins
- router.POST("/plugins/register", remote.RegisterPlugin(router, registerPluginEndpoints))
- }
if awsCognitoEnabled {
// Add login endpoint
@@ -119,8 +112,9 @@ func CreateApiService() {
ctx.Abort()
})
- // Add swagger handler
+ // Add swagger handlers
router.GET("/swagger/*any", ginSwagger.WrapHandler(swaggerFiles.Handler))
+ registerExtraOpenApiSpecs(router)
// Add debug logging for endpoints
gin.DebugPrintRouteFunc = func(httpMethod, absolutePath, handlerName string, nuHandlers int) {
@@ -145,9 +139,6 @@ func CreateApiService() {
// Register API endpoints
RegisterRouter(router)
- if remotePluginsEnabled {
- go bootstrapRemotePlugins(v)
- }
// Get port from config
port := v.GetString("PORT")
// Trim any : from the start
@@ -166,20 +157,29 @@ func CreateApiService() {
}
}
-func bootstrapRemotePlugins(v *viper.Viper) {
- // Get port from config
- port := v.GetString("PORT")
- // Trim any : from the start
- port = strings.TrimLeft(port, ":")
- // Convert to int
- portNum, err := strconv.Atoi(port)
- if err != nil {
- // Panic if PORT is not an int
- panic(fmt.Errorf("PORT [%s] must be int: %s", port, err.Error()))
- }
- // Bootstrap remote plugins
- err = bridge.Bootstrap(v, portNum)
- if err != nil {
- logruslog.Global.Error(err, "")
+func registerExtraOpenApiSpecs(router *gin.Engine) {
+ for name, pluginMeta := range plugin.AllPlugins() {
+ if pluginOpenApiSpec, ok := pluginMeta.(plugin.PluginOpenApiSpec); ok {
+ spec := &swag.Spec{
+ InfoInstanceName: name,
+ SwaggerTemplate: pluginOpenApiSpec.OpenApiSpec(),
+ }
+ swag.Register(name, spec)
+ router.GET(
+ fmt.Sprintf("/plugins/swagger/%s/*any", name),
+ ginSwagger.CustomWrapHandler(
+ &ginSwagger.Config{
+ URL: "doc.json",
+ DocExpansion: "list",
+ InstanceName: name,
+ Title: fmt.Sprintf("%s API", name),
+ DefaultModelsExpandDepth: 1,
+ DeepLinking: true,
+ PersistAuthorization: false,
+ },
+ swaggerFiles.Handler,
+ ),
+ )
+ }
}
}
diff --git a/backend/server/api/remote/register.go b/backend/server/api/remote/register.go
deleted file mode 100644
index 492700cc4..000000000
--- a/backend/server/api/remote/register.go
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package remote
-
-import (
- "fmt"
- "net/http"
-
- "github.com/apache/incubator-devlake/core/errors"
- "github.com/apache/incubator-devlake/core/plugin"
- "github.com/apache/incubator-devlake/server/api/shared"
- "github.com/apache/incubator-devlake/server/services/remote"
-
- "github.com/RaveNoX/go-jsonmerge"
- "github.com/gin-gonic/gin"
- "github.com/go-playground/validator/v10"
- ginSwagger "github.com/swaggo/gin-swagger"
- "github.com/swaggo/gin-swagger/swaggerFiles"
- "github.com/swaggo/swag"
-)
-
-var (
- vld = validator.New()
- cachedDocs = map[string]*swag.Spec{}
-)
-
-type ApiResource struct {
- PluginName string
- Resources map[string]map[string]plugin.ApiResourceHandler
-}
-
-// TODO add swagger doc
-func RegisterPlugin(router *gin.Engine, registerEndpoints func(r *gin.Engine, pluginName string, apiResources map[string]map[string]plugin.ApiResourceHandler)) func(*gin.Context) {
- return func(c *gin.Context) {
- var details PluginDetails
- if err := c.ShouldBindJSON(&details); err != nil {
- shared.ApiOutputError(c, errors.BadInput.Wrap(err, shared.BadRequestBody))
- return
- }
- if err := vld.Struct(&details); err != nil {
- shared.ApiOutputError(c, errors.BadInput.Wrap(err, shared.BadRequestBody))
- return
- }
- remotePlugin, err := remote.NewRemotePlugin(&details.PluginInfo)
- if err != nil {
- shared.ApiOutputError(c, errors.Default.Wrap(err, fmt.Sprintf("plugin %s could not be initialized", details.PluginInfo.Name)))
- return
- }
- resource := ApiResource{
- PluginName: details.PluginInfo.Name,
- Resources: remotePlugin.ApiResources(),
- }
- registerEndpoints(router, resource.PluginName, resource.Resources)
- registerSwagger(router, &details.Swagger)
- shared.ApiOutputSuccess(c, nil, http.StatusOK)
- }
-}
-
-func registerSwagger(router *gin.Engine, doc *SwaggerDoc) {
- if spec, ok := cachedDocs[doc.Name]; ok {
- spec.SwaggerTemplate = combineSpecs(spec.SwaggerTemplate, string(doc.Spec))
- } else {
- spec = &swag.Spec{
- Version: "",
- Host: "",
- BasePath: "",
- Schemes: nil,
- Title: "",
- Description: "",
- InfoInstanceName: doc.Name,
- SwaggerTemplate: string(doc.Spec),
- }
- swag.Register(doc.Name, spec)
- cachedDocs[doc.Name] = spec
- router.GET(fmt.Sprintf("/plugins/swagger/%s/*any", doc.Resource), ginSwagger.CustomWrapHandler(
- &ginSwagger.Config{
- URL: "doc.json",
- DocExpansion: "list",
- InstanceName: doc.Name,
- Title: "",
- DefaultModelsExpandDepth: 1,
- DeepLinking: true,
- PersistAuthorization: false,
- },
- swaggerFiles.Handler))
- }
-}
-
-func combineSpecs(spec1 string, spec2 string) string {
- res, _ := jsonmerge.Merge(spec1, spec2)
- return res.(string)
-}
diff --git a/backend/server/services/remote/bridge/bootstrap.go b/backend/server/services/remote/bridge/bootstrap.go
deleted file mode 100644
index aa5347756..000000000
--- a/backend/server/services/remote/bridge/bootstrap.go
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package bridge
-
-import (
- "fmt"
- "os"
- "os/exec"
- "path/filepath"
-
- "github.com/apache/incubator-devlake/core/errors"
- "github.com/apache/incubator-devlake/core/utils"
- "github.com/apache/incubator-devlake/impls/logruslog"
- "github.com/spf13/viper"
-)
-
-func Bootstrap(cfg *viper.Viper, port int) errors.Error {
- scriptPath := cfg.GetString("REMOTE_PLUGINS_STARTUP_PATH")
- if scriptPath == "" {
- return errors.BadInput.New(fmt.Sprintf("missing env key: %s", "REMOTE_PLUGINS_STARTUP_PATH"))
- }
- absScriptPath := scriptPath
- if !filepath.IsAbs(scriptPath) {
- workingDir, err := errors.Convert01(os.Getwd())
- if err != nil {
- return err
- }
- absScriptPath = filepath.Join(workingDir, scriptPath)
- }
- logruslog.Global.Info("Resolved remote plugins script path: %s", absScriptPath)
- cmd := exec.Command(absScriptPath, fmt.Sprintf("http://127.0.0.1:%d", port)) //expects the plugins to live on the same host
- cmd.Dir = filepath.Dir(absScriptPath)
- result, err := utils.RunProcess(cmd, &utils.RunProcessOptions{
- OnStdout: func(b []byte) {
- logruslog.Global.Info(string(b))
- },
- OnStderr: func(b []byte) {
- logruslog.Global.Error(nil, string(b))
- },
- })
- if err != nil {
- return err
- }
- if result.GetError() != nil {
- logruslog.Global.Error(result.GetError(), "error occurred bootstrapping remote plugins")
- }
- return nil
-}
diff --git a/backend/server/services/remote/bridge/cmd.go b/backend/server/services/remote/bridge/cmd.go
index 4dab3c075..25326ad44 100644
--- a/backend/server/services/remote/bridge/cmd.go
+++ b/backend/server/services/remote/bridge/cmd.go
@@ -21,6 +21,7 @@ import (
"encoding/json"
"fmt"
"os/exec"
+ "path"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/log"
@@ -34,10 +35,18 @@ type CmdInvoker struct {
workingPath string
}
-func NewCmdInvoker(workingPath string, resolveCmd func(methodName string, args ...string) (string, []string)) *CmdInvoker {
+func NewCmdInvoker(execPath string) *CmdInvoker {
+ // Split the path into dir and file
+ dir, file := path.Split(execPath)
+ resolveCmd := func(methodName string, args ...string) (string, []string) {
+ allArgs := []string{methodName}
+ allArgs = append(allArgs, args...)
+ return fmt.Sprintf("./%s", file), allArgs
+ }
+
return &CmdInvoker{
resolveCmd: resolveCmd,
- workingPath: workingPath,
+ workingPath: dir,
}
}
diff --git a/backend/server/services/remote/bridge/python_cmd.go b/backend/server/services/remote/bridge/python_cmd.go
deleted file mode 100644
index bdcebcd82..000000000
--- a/backend/server/services/remote/bridge/python_cmd.go
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package bridge
-
-import (
- "path/filepath"
- "strings"
-)
-
-const (
- poetryExec = "poetry"
- pythonExec = "python"
-)
-
-func NewPythonPoetryCmdInvoker(scriptPath string) *CmdInvoker {
- tomlPath := filepath.Dir(filepath.Dir(scriptPath)) //the main entrypoint expected to be at toplevel
- scriptPath = strings.TrimPrefix(scriptPath, tomlPath+"/")
- return NewCmdInvoker(tomlPath, func(methodName string, args ...string) (string, []string) {
- allArgs := []string{"run", pythonExec, scriptPath, methodName}
- allArgs = append(allArgs, args...)
- return poetryExec, allArgs
- })
-}
diff --git a/backend/server/services/remote/models/plugin_remote.go b/backend/server/services/remote/models/plugin_remote.go
index 72f4a45bf..a8984ea56 100644
--- a/backend/server/services/remote/models/plugin_remote.go
+++ b/backend/server/services/remote/models/plugin_remote.go
@@ -27,5 +27,6 @@ type RemotePlugin interface {
plugin.PluginApi
plugin.PluginTask
plugin.PluginMeta
+ plugin.PluginOpenApiSpec
RunMigrations(forceMigrate bool) errors.Error
}
diff --git a/backend/server/services/remote/plugin/doc/open_api.go b/backend/server/services/remote/plugin/doc/open_api.go
new file mode 100644
index 000000000..811c19309
--- /dev/null
+++ b/backend/server/services/remote/plugin/doc/open_api.go
@@ -0,0 +1,83 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package doc
+
+import (
+ "encoding/json"
+ "io"
+ "os"
+ "path"
+ "runtime"
+ "strings"
+ "text/template"
+
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/server/services/remote/models"
+)
+
+func GenerateOpenApiSpec(pluginInfo *models.PluginInfo) (*string, errors.Error) {
+ connectionSchema, err := json.Marshal(pluginInfo.ConnectionModelInfo.JsonSchema)
+ if err != nil {
+ return nil, errors.Default.Wrap(err, "connection schema is not valid JSON")
+ }
+ scopeSchema, err := json.Marshal(pluginInfo.ScopeModelInfo.JsonSchema)
+ if err != nil {
+ return nil, errors.Default.Wrap(err, "scope schema is not valid JSON")
+ }
+ txRuleSchema, err := json.Marshal(pluginInfo.TransformationRuleModelInfo.JsonSchema)
+ if err != nil {
+ return nil, errors.Default.Wrap(err, "transformation rule schema is not valid JSON")
+ }
+ specTemplate, tmplErr := specTemplate()
+ if tmplErr != nil {
+ return nil, tmplErr
+ }
+ writer := &strings.Builder{}
+ err = specTemplate.Execute(writer, map[string]interface{}{
+ "PluginName": pluginInfo.Name,
+ "ConnectionSchema": string(connectionSchema),
+ "ScopeSchema": string(scopeSchema),
+ "TransformationRuleSchema": string(txRuleSchema),
+ })
+ if err != nil {
+ return nil, errors.Default.Wrap(err, "could not execute swagger doc template")
+ }
+ doc := writer.String()
+ return &doc, nil
+}
+
+func specTemplate() (*template.Template, errors.Error) {
+ file, err := os.Open(specTemplatePath())
+ if err != nil {
+ return nil, errors.Default.Wrap(err, "could not open swagger doc template")
+ }
+ contents, err := io.ReadAll(file)
+ if err != nil {
+ return nil, errors.Default.Wrap(err, "could not read swagger doc template")
+ }
+ specTemplate, err := template.New("doc").Parse(string(contents))
+ if err != nil {
+ return nil, errors.Default.Wrap(err, "could not parse swagger doc template")
+ }
+ return specTemplate, nil
+}
+
+func specTemplatePath() string {
+ _, currentFile, _, _ := runtime.Caller(0)
+ return path.Join(path.Dir(currentFile), "open_api_spec.json.tmpl")
+}
diff --git a/backend/python/pydevlake/pydevlake/doc.template.json b/backend/server/services/remote/plugin/doc/open_api_spec.json.tmpl
similarity index 95%
rename from backend/python/pydevlake/pydevlake/doc.template.json
rename to backend/server/services/remote/plugin/doc/open_api_spec.json.tmpl
index 10cccd2c1..eda7eb40a 100644
--- a/backend/python/pydevlake/pydevlake/doc.template.json
+++ b/backend/server/services/remote/plugin/doc/open_api_spec.json.tmpl
@@ -1,11 +1,11 @@
{
"info": {
- "title": "$plugin_name plugin documentation",
+ "title": "{{.PluginName}} plugin documentation",
"version": "1.0.0"
},
"openapi": "3.0.2",
"paths": {
- "/plugins/$plugin_name/connections/{connectionId}": {
+ "/plugins/{{.PluginName}}/connections/{connectionId}": {
"get": {
"description": "Get a connection",
"parameters": [
@@ -72,7 +72,7 @@
}
}
},
- "/plugins/$plugin_name/connections": {
+ "/plugins/{{.PluginName}}/connections": {
"get": {
"description": "Get all connections",
"responses": {
@@ -115,7 +115,7 @@
}
}
},
- "/plugins/$plugin_name/test": {
+ "/plugins/{{.PluginName}}/test": {
"post": {
"description": "Test if a connection is valid",
"parameters": [
@@ -138,7 +138,7 @@
}
}
},
- "/plugins/$plugin_name/connections/{connectionId}/scopes/{scopeId}": {
+ "/plugins/{{.PluginName}}/connections/{connectionId}/scopes/{scopeId}": {
"get": {
"description": "Get a scope",
"parameters": [
@@ -192,7 +192,7 @@
}
}
},
- "/plugins/$plugin_name/connections/{connectionId}/scopes": {
+ "/plugins/{{.PluginName}}/connections/{connectionId}/scopes": {
"get": {
"description": "Get all scopes",
"parameters": [
@@ -249,7 +249,7 @@
}
}
},
- "/plugins/$plugin_name/connections/{connectionId}/transformation_rules": {
+ "/plugins/{{.PluginName}}/connections/{connectionId}/transformation_rules": {
"get": {
"description": "Get all transformation rules",
"parameters": [
@@ -300,7 +300,7 @@
}
}
},
- "/plugins/$plugin_name/connections/{connectionId}/transformation_rules/{ruleId}": {
+ "/plugins/{{.PluginName}}/connections/{connectionId}/transformation_rules/{ruleId}": {
"get": {
"description": "Get a transformation rule",
"parameters": [
@@ -351,8 +351,8 @@
},
"components": {
"schemas": {
- "connection": $connection_schema,
- "transformationRule": $transformation_rule_schema,
+ "connection": {{.ConnectionSchema}},
+ "transformationRule": {{.TransformationRuleSchema}},
"scope": {
"title": "Scope",
"type": "object",
diff --git a/backend/server/services/remote/plugin/init.go b/backend/server/services/remote/plugin/init.go
index cd75b2740..6698a83d1 100644
--- a/backend/server/services/remote/plugin/init.go
+++ b/backend/server/services/remote/plugin/init.go
@@ -42,7 +42,7 @@ func Init(br context.BasicRes) {
}
func NewRemotePlugin(info *models.PluginInfo) (models.RemotePlugin, errors.Error) {
- invoker := bridge.NewPythonPoetryCmdInvoker(info.PluginPath)
+ invoker := bridge.NewCmdInvoker(info.PluginPath)
plugin, err := newPlugin(info, invoker)
if err != nil {
diff --git a/backend/server/services/remote/plugin/plugin_impl.go b/backend/server/services/remote/plugin/plugin_impl.go
index 8790bd634..e8f563690 100644
--- a/backend/server/services/remote/plugin/plugin_impl.go
+++ b/backend/server/services/remote/plugin/plugin_impl.go
@@ -28,6 +28,7 @@ import (
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/server/services/remote/bridge"
"github.com/apache/incubator-devlake/server/services/remote/models"
+ "github.com/apache/incubator-devlake/server/services/remote/plugin/doc"
)
type (
@@ -41,6 +42,7 @@ type (
scopeTabler *coreModels.DynamicTabler
transformationRuleTabler *coreModels.DynamicTabler
resources map[string]map[string]plugin.ApiResourceHandler
+ openApiSpec string
}
RemotePluginTaskData struct {
DbUrl string `json:"db_url"`
@@ -68,6 +70,10 @@ func newPlugin(info *models.PluginInfo, invoker bridge.Invoker) (*remotePluginIm
if err != nil {
return nil, errors.Default.Wrap(err, fmt.Sprintf("Couldn't load Scope type for plugin %s", info.Name))
}
+ openApiSpec, err := doc.GenerateOpenApiSpec(info)
+ if err != nil {
+ return nil, errors.Default.Wrap(err, fmt.Sprintf("Couldn't generate OpenAPI spec for plugin %s", info.Name))
+ }
p := remotePluginImpl{
name: info.Name,
invoker: invoker,
@@ -77,6 +83,7 @@ func newPlugin(info *models.PluginInfo, invoker bridge.Invoker) (*remotePluginIm
scopeTabler: scopeTabler,
transformationRuleTabler: txRuleTabler,
resources: GetDefaultAPI(invoker, connectionTabler, txRuleTabler, scopeTabler, connectionHelper),
+ openApiSpec: *openApiSpec,
}
remoteBridge := bridge.NewBridge(invoker)
for _, subtask := range info.SubtaskMetas {
@@ -195,4 +202,8 @@ func (p *remotePluginImpl) RunMigrations(forceMigrate bool) errors.Error {
return err
}
+func (p *remotePluginImpl) OpenApiSpec() string {
+ return p.openApiSpec
+}
+
var _ models.RemotePlugin = (*remotePluginImpl)(nil)
diff --git a/backend/server/services/remote/run/run.go b/backend/server/services/remote/run/run.go
index b774ab1f8..81c656bc1 100644
--- a/backend/server/services/remote/run/run.go
+++ b/backend/server/services/remote/run/run.go
@@ -38,7 +38,7 @@ func main() {
_ = cmd.MarkFlagRequired("connectionId")
cmd.Run = func(cmd *cobra.Command, args []string) {
- invoker := bridge.NewPythonPoetryCmdInvoker(*pluginPath)
+ invoker := bridge.NewCmdInvoker(*pluginPath)
pluginInfo := models.PluginInfo{}
err := invoker.Call("plugin-info", bridge.DefaultContext).Get(&pluginInfo)
diff --git a/backend/test/e2e/remote/helper.go b/backend/test/e2e/remote/helper.go
index e7521cc9c..e62cbef1b 100644
--- a/backend/test/e2e/remote/helper.go
+++ b/backend/test/e2e/remote/helper.go
@@ -53,17 +53,6 @@ type (
}
)
-func SetupEnv() {
- fmt.Println("Setup test env")
- path := filepath.Join(helper.ProjectRoot, FAKE_PLUGIN_DIR, "start.sh")
- _, err := os.Stat(path)
- if err != nil {
- panic(err)
- }
- _ = os.Setenv("REMOTE_PLUGINS_STARTUP_PATH", path)
- _ = os.Setenv("ENABLE_REMOTE_PLUGINS", "true")
-}
-
func ConnectLocalServer(t *testing.T) *helper.DevlakeClient {
fmt.Println("Connect to server")
client := helper.StartDevLakeServer(t, nil)
@@ -72,7 +61,8 @@ func ConnectLocalServer(t *testing.T) *helper.DevlakeClient {
}
func CreateClient(t *testing.T) *helper.DevlakeClient {
- SetupEnv()
+ path := filepath.Join(helper.ProjectRoot, FAKE_PLUGIN_DIR)
+ _ = os.Setenv("REMOTE_PLUGIN_DIR", path)
client := ConnectLocalServer(t)
client.AwaitPluginAvailability(PLUGIN_NAME, 60*time.Second)
return client
diff --git a/env.example b/env.example
index ee4e1bb95..8c043663a 100644
--- a/env.example
+++ b/env.example
@@ -4,6 +4,7 @@
# Lake plugin dir, absolute path or relative path
PLUGIN_DIR=bin/plugins
+REMOTE_PLUGIN_DIR=python/plugins
# Lake Database Connection String
DB_URL=mysql://merico:merico@mysql:3306/lake?charset=utf8mb4&parseTime=True