You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@submarine.apache.org by ku...@apache.org on 2021/12/27 15:19:02 UTC
[submarine] branch master updated: SUBMARINE-1135. Implement CLI Config Command
This is an automated email from the ASF dual-hosted git repository.
kuanhsun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/submarine.git
The following commit(s) were added to refs/heads/master by this push:
new e68cf27 SUBMARINE-1135. Implement CLI Config Command
e68cf27 is described below
commit e68cf27da85451a2c4d14eca366a0c125e939972
Author: atosystem <at...@hotmail.com>
AuthorDate: Sat Dec 25 12:35:22 2021 +0800
SUBMARINE-1135. Implement CLI Config Command
### What is this PR for?
Implement `submarine config` command to set *_port_* and *_hostname_* for API connection.
Usage:
`submarine config init` : init config file (restore to default)
`submarine config list` : list config content
`submarine config set <param> <value>` : set param = value in config
`submarine config get <param>` : get param
Examples:
1. Change API port to `8080`
`submarine config set connection.port 8080`
2. Change hostname to `127.0.0.1`
`submarine config set connection.hostname 127.0.0.1`
### What type of PR is it?
[Feature]
### Todos
None
### What is the Jira issue?
https://issues.apache.org/jira/projects/SUBMARINE/issues/SUBMARINE-1135
### How should this be tested?
python unit test is implemented
### Screenshots (if appropriate)
### Questions:
* Do the license files need updating? No
* Are there breaking changes for older versions? No
* Does this need new documentation? Yes
Author: atosystem <at...@hotmail.com>
Signed-off-by: kuanhsun <ku...@apache.org>
Closes #847 from atosystem/SUBMARINE-1135 and squashes the following commits:
e96646f3 [atosystem] SUBMARINE-1135. Add config init unit test
22beebbc [atosystem] SUBMARINE-1135. Change init connection.port
e49ab80c [atosystem] SUBMARINE-1135. Fix dependency for python3.7
bd63dd4d [atosystem] SUBMARINE-1135. Add rich in setup.py
1566e582 [atosystem] SUBMARINE-1135. Implement CLI Config and unit tests
---
.../style-check/python/mypy-requirements.txt | 2 +
submarine-sdk/pysubmarine/setup.py | 5 +
.../pysubmarine/submarine/cli/config/__init__.py | 20 ++--
.../submarine/cli/config/cli_config.yaml | 3 +
.../pysubmarine/submarine/cli/config/command.py | 82 +++++++++++++
.../pysubmarine/submarine/cli/config/config.py | 127 +++++++++++++++++++++
submarine-sdk/pysubmarine/submarine/cli/main.py | 12 ++
submarine-sdk/pysubmarine/tests/cli/test_config.py | 62 ++++++++++
8 files changed, 304 insertions(+), 9 deletions(-)
diff --git a/dev-support/style-check/python/mypy-requirements.txt b/dev-support/style-check/python/mypy-requirements.txt
index 369325a..41d5e73 100644
--- a/dev-support/style-check/python/mypy-requirements.txt
+++ b/dev-support/style-check/python/mypy-requirements.txt
@@ -18,4 +18,6 @@ types-requests==2.25.6
types-certifi==2020.4.0
types-six==1.16.1
types-python-dateutil==2.8.0
+types-dataclasses==0.6.1
+types-PyYAML==6.0.1
sqlalchemy[mypy]
diff --git a/submarine-sdk/pysubmarine/setup.py b/submarine-sdk/pysubmarine/setup.py
index 1e72068..720a561 100644
--- a/submarine-sdk/pysubmarine/setup.py
+++ b/submarine-sdk/pysubmarine/setup.py
@@ -26,6 +26,7 @@ setup(
long_description_content_type="text/markdown",
url="https://github.com/apache/submarine",
packages=find_packages(exclude=["tests", "tests.*"]),
+ package_data={"submarine.cli.config": ["cli_config.yaml"]},
install_requires=[
"six>=1.10.0",
"numpy==1.19.2",
@@ -41,6 +42,10 @@ setup(
"mlflow>=1.15.0",
"boto3>=1.17.58",
"click==8.0.3",
+ "rich==10.15.2",
+ "dacite==1.6.0",
+ "dataclasses>=0.6",
+ "pyaml==21.10.1",
],
extras_require={
"tf": ["tensorflow==1.15.0"],
diff --git a/dev-support/style-check/python/mypy-requirements.txt b/submarine-sdk/pysubmarine/submarine/cli/config/__init__.py
similarity index 62%
copy from dev-support/style-check/python/mypy-requirements.txt
copy to submarine-sdk/pysubmarine/submarine/cli/config/__init__.py
index 369325a..4103e81 100644
--- a/dev-support/style-check/python/mypy-requirements.txt
+++ b/submarine-sdk/pysubmarine/submarine/cli/config/__init__.py
@@ -1,11 +1,11 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
+# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
+# the License. You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
@@ -13,9 +13,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-mypy==0.930
-types-requests==2.25.6
-types-certifi==2020.4.0
-types-six==1.16.1
-types-python-dateutil==2.8.0
-sqlalchemy[mypy]
+from submarine.cli.config.command import get_config, init_config, list_config, set_config
+
+__all__ = [
+ "list_config",
+ "get_config",
+ "set_config",
+ "init_config",
+]
diff --git a/submarine-sdk/pysubmarine/submarine/cli/config/cli_config.yaml b/submarine-sdk/pysubmarine/submarine/cli/config/cli_config.yaml
new file mode 100644
index 0000000..84dc9ed
--- /dev/null
+++ b/submarine-sdk/pysubmarine/submarine/cli/config/cli_config.yaml
@@ -0,0 +1,3 @@
+connection:
+ hostname: "localhost"
+ port: 32080
diff --git a/submarine-sdk/pysubmarine/submarine/cli/config/command.py b/submarine-sdk/pysubmarine/submarine/cli/config/command.py
new file mode 100644
index 0000000..8927bc6
--- /dev/null
+++ b/submarine-sdk/pysubmarine/submarine/cli/config/command.py
@@ -0,0 +1,82 @@
+"""
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+"""
+
+from dataclasses import asdict
+from typing import Union
+
+import click
+from rich.console import Console
+from rich.json import JSON as richJSON
+from rich.panel import Panel
+
+from submarine.cli.config.config import initConfig, loadConfig, rgetattr, rsetattr, saveConfig
+
+
+@click.command("list")
+def list_config():
+ """List Submarine CLI Config"""
+ console = Console()
+ _config = loadConfig()
+ json_data = richJSON.from_data({**asdict(_config)})
+ console.print(Panel(json_data, title="SubmarineCliConfig"))
+
+
+@click.command("get")
+@click.argument("param")
+def get_config(param):
+ """Get Submarine CLI Config"""
+ _config = loadConfig()
+ try:
+ click.echo("{}={}".format(param, rgetattr(_config, param)))
+ except AttributeError as err:
+ click.echo(err)
+
+
+@click.command("set")
+@click.argument("param")
+@click.argument("value")
+def set_config(param, value):
+ """Set Submarine CLI Config"""
+ _config = loadConfig()
+ _paramField = rgetattr(_config, param)
+ # define types that can be cast from command line input
+ primitive = (int, str, bool)
+
+ def is_primitiveType(_type):
+ return _type in primitive
+
+ # cast type
+ if type(_paramField) == type(Union) and is_primitiveType(type(_paramField).__args__[0]):
+ value = type(_paramField).__args__[0](value)
+ elif is_primitiveType(type(_paramField)):
+ value = type(_paramField)(value)
+
+ try:
+ rsetattr(_config, param, value)
+ except TypeError as err:
+ click.echo(err)
+ saveConfig(_config)
+
+
+@click.command("init")
+def init_config():
+ """Init Submarine CLI Config"""
+ try:
+ initConfig()
+ click.echo("Submarine CLI Config initialized")
+ except AttributeError as err:
+ click.echo(err)
diff --git a/submarine-sdk/pysubmarine/submarine/cli/config/config.py b/submarine-sdk/pysubmarine/submarine/cli/config/config.py
new file mode 100644
index 0000000..dada3db
--- /dev/null
+++ b/submarine-sdk/pysubmarine/submarine/cli/config/config.py
@@ -0,0 +1,127 @@
+"""
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+"""
+
+import functools
+import os
+from dataclasses import asdict, dataclass, field
+from typing import Optional, Union
+
+import click
+import dacite
+import yaml
+
+CONFIG_YAML_PATH = os.path.join(os.path.dirname(__file__), "cli_config.yaml")
+
+
+@dataclass
+class BaseConfig:
+ def __setattr__(self, __name, __value) -> None:
+ """
+ Override __setattr__ for custom type checking
+ """
+ # ignore this line for mypy checking, since there is some errors for mypy to check dataclass
+ _field = self.__dataclass_fields__[__name] # type: ignore
+ if hasattr(_field.type, "__origin__") and _field.type.__origin__ == Union:
+ if not isinstance(__value, _field.type.__args__):
+ msg = (
+ "Field `{0.name}` is of type {1}, should be one of the type: {0.type.__args__}"
+ .format(_field, type(__value))
+ )
+ raise TypeError(msg)
+ else:
+ if not type(__value) == _field.type:
+ msg = "Field {0.name} is of type {1}, should be {0.type}".format(
+ _field, type(__value)
+ )
+ raise TypeError(msg)
+
+ super().__setattr__(__name, __value)
+
+
+@dataclass
+class ConnectionConfig(BaseConfig):
+ hostname: Optional[str] = field(
+ default="localhost",
+ metadata={"help": "Hostname for submarine CLI to connect"},
+ )
+
+ port: Optional[int] = field(
+ default=32080,
+ metadata={"help": "Port for submarine CLI to connect"},
+ )
+
+
+@dataclass
+class SubmarineCliConfig(BaseConfig):
+ connection: ConnectionConfig = field(
+ default_factory=lambda: ConnectionConfig(),
+ metadata={"help": "Port for submarine CLI to connect"},
+ )
+
+
+def rgetattr(obj, attr, *args):
+ """
+ Recursive get attr
+ Example:
+ rgetattr(obj,"a.b.c") is equivalent to obj.a.b.c
+ """
+
+ def _getattr(obj, attr):
+ return getattr(obj, attr, *args)
+
+ return functools.reduce(_getattr, [obj] + attr.split("."))
+
+
+def rsetattr(obj, attr, val):
+ """
+ Recursive set attr
+ Example:
+ rsetattr(obj,"a.b.c",val) is equivalent to obj.a.b.c = val
+ """
+ pre, _, post = attr.rpartition(".")
+ if pre:
+ _r = rgetattr(obj, pre)
+ return setattr(_r, post, val)
+ else:
+ return setattr(obj, post, val)
+
+
+def loadConfig(config_path: str = CONFIG_YAML_PATH) -> Optional[SubmarineCliConfig]:
+ with open(config_path, "r") as stream:
+ try:
+ parsed_yaml: dict = yaml.safe_load(stream)
+ return_config: SubmarineCliConfig = dacite.from_dict(
+ data_class=SubmarineCliConfig, data=parsed_yaml
+ )
+ return return_config
+ except yaml.YAMLError as exc:
+ click.echo("Error Reading Config")
+ click.echo(exc)
+ return None
+
+
+def saveConfig(config: SubmarineCliConfig, config_path: str = CONFIG_YAML_PATH):
+ with open(config_path, "w") as stream:
+ try:
+ yaml.safe_dump({**asdict(config)}, stream)
+ except yaml.YAMLError as exc:
+ click.echo("Error Saving Config")
+ click.echo(exc)
+
+
+def initConfig(config_path: str = CONFIG_YAML_PATH):
+ saveConfig(SubmarineCliConfig(), config_path)
diff --git a/submarine-sdk/pysubmarine/submarine/cli/main.py b/submarine-sdk/pysubmarine/submarine/cli/main.py
index b8a699b..2998f9a 100644
--- a/submarine-sdk/pysubmarine/submarine/cli/main.py
+++ b/submarine-sdk/pysubmarine/submarine/cli/main.py
@@ -17,6 +17,7 @@
import click
+from submarine.cli.config import command as config_cmd
from submarine.cli.environment import command as environment_cmd
from submarine.cli.experiment import command as experiment_cmd
from submarine.cli.notebook import command as notebook_cmd
@@ -49,6 +50,11 @@ def cmdgrp_sandbox():
pass
+@entry_point.group("config")
+def cmdgrp_config():
+ pass
+
+
# experiment
cmdgrp_list.add_command(experiment_cmd.list_experiment)
cmdgrp_get.add_command(experiment_cmd.get_experiment)
@@ -65,3 +71,9 @@ cmdgrp_delete.add_command(environment_cmd.delete_environment)
# sandbox
cmdgrp_sandbox.add_command(sandbox_cmd.start_sandbox)
cmdgrp_sandbox.add_command(sandbox_cmd.delete_sandbox)
+
+# config
+cmdgrp_config.add_command(config_cmd.set_config)
+cmdgrp_config.add_command(config_cmd.list_config)
+cmdgrp_config.add_command(config_cmd.get_config)
+cmdgrp_config.add_command(config_cmd.init_config)
diff --git a/submarine-sdk/pysubmarine/tests/cli/test_config.py b/submarine-sdk/pysubmarine/tests/cli/test_config.py
new file mode 100644
index 0000000..23563d4
--- /dev/null
+++ b/submarine-sdk/pysubmarine/tests/cli/test_config.py
@@ -0,0 +1,62 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from click.testing import CliRunner
+
+from submarine.cli import main
+from submarine.cli.config.config import SubmarineCliConfig, initConfig, loadConfig
+
+
+def test_list_config():
+ initConfig()
+ runner = CliRunner()
+ result = runner.invoke(main.entry_point, ["config", "list"])
+ _config = loadConfig()
+ assert result.exit_code == 0
+ assert "SubmarineCliConfig" in result.output
+ assert '"hostname": "{}"'.format(_config.connection.hostname) in result.output
+ assert '"port": {}'.format(_config.connection.port) in result.output
+
+
+def test_init_config():
+ runner = CliRunner()
+ result = runner.invoke(main.entry_point, ["config", "init"])
+ result = runner.invoke(main.entry_point, ["config", "list"])
+ _default_config = SubmarineCliConfig()
+ assert result.exit_code == 0
+ assert '"hostname": "{}"'.format(_default_config.connection.hostname) in result.output
+ assert '"port": {}'.format(_default_config.connection.port) in result.output
+
+
+def test_get_set_experiment():
+ initConfig()
+ mock_hostname = "mockhost"
+ runner = CliRunner()
+ result = runner.invoke(main.entry_point, ["config", "get", "connection.hostname"])
+ assert result.exit_code == 0
+ _config = loadConfig()
+ assert "connection.hostname={}".format(_config.connection.hostname) in result.output
+
+ result = runner.invoke(
+ main.entry_point, ["config", "set", "connection.hostname", mock_hostname]
+ )
+ assert result.exit_code == 0
+
+ result = runner.invoke(main.entry_point, ["config", "get", "connection.hostname"])
+ assert result.exit_code == 0
+ _config = loadConfig()
+ assert "connection.hostname={}".format(mock_hostname) in result.output
+ assert mock_hostname == _config.connection.hostname
+ initConfig()
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@submarine.apache.org
For additional commands, e-mail: dev-help@submarine.apache.org