You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@submarine.apache.org by pi...@apache.org on 2022/01/21 18:03:45 UTC
[submarine] branch master updated: SUBMARINE-1133. Connect API for CLI Notebooks
This is an automated email from the ASF dual-hosted git repository.
pingsutw pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/submarine.git
The following commit(s) were added to refs/heads/master by this push:
new 071e1ea SUBMARINE-1133. Connect API for CLI Notebooks
071e1ea is described below
commit 071e1ea0384d3eb86d2ce21f5c2353cb0ce066c2
Author: atosystem <at...@hotmail.com>
AuthorDate: Tue Jan 18 13:26:02 2022 +0800
SUBMARINE-1133. Connect API for CLI Notebooks
### What is this PR for?
Implement
```bash=
submarine list notebook
submarine get notebook <id>
submarine delete notebook <id>
```
### What type of PR is it?
[Feature]
### Todos
None
### What is the Jira issue?
https://issues.apache.org/jira/browse/SUBMARINE-1133
### How should this be tested?
python e2e tests are implemented
### Screenshots (if appropriate)
None
### Questions:
* Do the license files need updating? No
* Are there breaking changes for older versions? No
* Does this need new documentation? Yes
Author: atosystem <at...@hotmail.com>
Signed-off-by: Kevin <pi...@apache.org>
Closes #871 from atosystem/SUBMARINE-1133 and squashes the following commits:
94d792fa [atosystem] SUBMARINE-1133. remove redundant error handling
782e1a60 [atosystem] SUBMARINE-1116. add link
8b36faad [atosystem] SUBMARINE-1133. implement notebook CLI
cc73adf4 [atosystem] SUBMARINE-1133. Add notebook client
---
.../pysubmarine/submarine/cli/config/config.py | 4 +-
.../submarine/cli/environment/command.py | 3 +-
.../submarine/cli/experiment/command.py | 3 +-
.../pysubmarine/submarine/cli/notebook/command.py | 114 ++++++++++++++++++++-
.../submarine/client/api/notebook_client.py | 113 ++++++++++++++++++++
.../pysubmarine/tests/cli/test_notebook.py | 69 ++++++++++---
.../tests/notebook/test_notebook_client.py | 43 ++++++++
7 files changed, 325 insertions(+), 24 deletions(-)
diff --git a/submarine-sdk/pysubmarine/submarine/cli/config/config.py b/submarine-sdk/pysubmarine/submarine/cli/config/config.py
index dada3db..056961b 100644
--- a/submarine-sdk/pysubmarine/submarine/cli/config/config.py
+++ b/submarine-sdk/pysubmarine/submarine/cli/config/config.py
@@ -100,7 +100,7 @@ def rsetattr(obj, attr, val):
return setattr(obj, post, val)
-def loadConfig(config_path: str = CONFIG_YAML_PATH) -> Optional[SubmarineCliConfig]:
+def loadConfig(config_path: str = CONFIG_YAML_PATH) -> SubmarineCliConfig:
with open(config_path, "r") as stream:
try:
parsed_yaml: dict = yaml.safe_load(stream)
@@ -111,7 +111,7 @@ def loadConfig(config_path: str = CONFIG_YAML_PATH) -> Optional[SubmarineCliConf
except yaml.YAMLError as exc:
click.echo("Error Reading Config")
click.echo(exc)
- return None
+ exit(1)
def saveConfig(config: SubmarineCliConfig, config_path: str = CONFIG_YAML_PATH):
diff --git a/submarine-sdk/pysubmarine/submarine/cli/environment/command.py b/submarine-sdk/pysubmarine/submarine/cli/environment/command.py
index a545e98..5b0ace9 100644
--- a/submarine-sdk/pysubmarine/submarine/cli/environment/command.py
+++ b/submarine-sdk/pysubmarine/submarine/cli/environment/command.py
@@ -29,8 +29,7 @@ from submarine.client.api.environment_client import EnvironmentClient
from submarine.client.exceptions import ApiException
submarineCliConfig = loadConfig()
-if submarineCliConfig is None:
- exit(1)
+
environmentClient = EnvironmentClient(
host="http://{}:{}".format(
submarineCliConfig.connection.hostname, submarineCliConfig.connection.port
diff --git a/submarine-sdk/pysubmarine/submarine/cli/experiment/command.py b/submarine-sdk/pysubmarine/submarine/cli/experiment/command.py
index 27b1e87..277c5a0 100644
--- a/submarine-sdk/pysubmarine/submarine/cli/experiment/command.py
+++ b/submarine-sdk/pysubmarine/submarine/cli/experiment/command.py
@@ -29,8 +29,7 @@ from submarine.client.api.experiment_client import ExperimentClient
from submarine.client.exceptions import ApiException
submarineCliConfig = loadConfig()
-if submarineCliConfig is None:
- exit(1)
+
experimentClient = ExperimentClient(
host="http://{}:{}".format(
submarineCliConfig.connection.hostname, submarineCliConfig.connection.port
diff --git a/submarine-sdk/pysubmarine/submarine/cli/notebook/command.py b/submarine-sdk/pysubmarine/submarine/cli/notebook/command.py
index 8deb3cb..c6a1d1a 100644
--- a/submarine-sdk/pysubmarine/submarine/cli/notebook/command.py
+++ b/submarine-sdk/pysubmarine/submarine/cli/notebook/command.py
@@ -15,24 +15,132 @@
under the License.
"""
+import json
+import time
+
import click
+from rich.console import Console
+from rich.json import JSON as richJSON
+from rich.panel import Panel
+from rich.table import Table
+
+from submarine.cli.config.config import loadConfig
+from submarine.client.api.notebook_client import NotebookClient
+from submarine.client.exceptions import ApiException
+
+submarineCliConfig = loadConfig()
+
+notebookClient = NotebookClient(
+ host="http://{}:{}".format(
+ submarineCliConfig.connection.hostname, submarineCliConfig.connection.port
+ )
+)
+
+POLLING_INTERVAL = 1 # sec
+TIMEOUT = 30 # sec
@click.command("notebook")
def list_notebook():
"""List notebooks"""
- click.echo("list notebook!")
+ COLS_TO_SHOW = ["Name", "ID", "Environment", "Resources", "Status"]
+ console = Console()
+ # using user_id hard coded in SysUserRestApi.java
+ # https://github.com/apache/submarine/blob/5040068d7214a46c52ba87e10e9fa64411293cf7/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/rest/SysUserRestApi.java#L228
+ try:
+ thread = notebookClient.list_notebooks_async(user_id="4291d7da9005377ec9aec4a71ea837f")
+ timeout = time.time() + TIMEOUT
+ with console.status("[bold green] Fetching Notebook..."):
+ while not thread.ready():
+ time.sleep(POLLING_INTERVAL)
+ if time.time() > timeout:
+ console.print("[bold red] Timeout!")
+ return
+
+ result = thread.get()
+ results = result.result
+
+ results = list(
+ map(
+ lambda r: [
+ r["name"],
+ r["notebookId"],
+ r["spec"]["environment"]["name"],
+ r["spec"]["spec"]["resources"],
+ r["status"],
+ ],
+ results,
+ )
+ )
+
+ table = Table(title="List of Notebooks")
+
+ for col in COLS_TO_SHOW:
+ table.add_column(col, overflow="fold")
+ for res in results:
+ table.add_row(*res)
+
+ console.print(table)
+
+ except ApiException as err:
+ if err.body is not None:
+ errbody = json.loads(err.body)
+ click.echo("[Api Error] {}".format(errbody["message"]))
+ else:
+ click.echo("[Api Error] {}".format(err))
@click.command("notebook")
@click.argument("id")
def get_notebook(id):
"""Get notebooks"""
- click.echo("get notebook! id={}".format(id))
+ console = Console()
+ try:
+ thread = notebookClient.get_notebook_async(id)
+ timeout = time.time() + TIMEOUT
+ with console.status("[bold green] Fetching Notebook(id = {} )...".format(id)):
+ while not thread.ready():
+ time.sleep(POLLING_INTERVAL)
+ if time.time() > timeout:
+ console.print("[bold red] Timeout!")
+ return
+
+ result = thread.get()
+ result = result.result
+
+ json_data = richJSON.from_data(result)
+ console.print(Panel(json_data, title="Notebook(id = {} )".format(id)))
+ except ApiException as err:
+ if err.body is not None:
+ errbody = json.loads(err.body)
+ click.echo("[Api Error] {}".format(errbody["message"]))
+ else:
+ click.echo("[Api Error] {}".format(err))
@click.command("notebook")
@click.argument("id")
def delete_notebook(id):
"""Delete notebook"""
- click.echo("delete notebook! id={}".format(id))
+ console = Console()
+ try:
+ thread = notebookClient.delete_notebook_async(id)
+ timeout = time.time() + TIMEOUT
+ with console.status("[bold green] Deleting Notebook(id = {} )...".format(id)):
+ while not thread.ready():
+ time.sleep(POLLING_INTERVAL)
+ if time.time() > timeout:
+ console.print("[bold red] Timeout!")
+ return
+
+ result = thread.get()
+ result = result.result
+
+ console.print("[bold green] Notebook(id = {} ) deleted".format(id))
+
+ except ApiException as err:
+ if err.body is not None:
+ errbody = json.loads(err.body)
+ click.echo("[Api Error] {}".format(errbody["message"]))
+ else:
+ click.echo("[Api Error] {}".format(err))
diff --git a/submarine-sdk/pysubmarine/submarine/client/api/notebook_client.py b/submarine-sdk/pysubmarine/submarine/client/api/notebook_client.py
new file mode 100644
index 0000000..0d388c6
--- /dev/null
+++ b/submarine-sdk/pysubmarine/submarine/client/api/notebook_client.py
@@ -0,0 +1,113 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import os
+
+from submarine.client.api.notebook_api import NotebookApi
+from submarine.client.api_client import ApiClient
+from submarine.client.configuration import Configuration
+
+logger = logging.getLogger(__name__)
+logging.basicConfig(format="%(message)s")
+logging.getLogger().setLevel(logging.INFO)
+
+
+def generate_host():
+ """
+ Generate submarine host
+ :return: submarine host
+ """
+ submarine_server_dns_name = str(os.environ.get("SUBMARINE_SERVER_DNS_NAME"))
+ submarine_server_port = str(os.environ.get("SUBMARINE_SERVER_PORT"))
+ host = "http://" + submarine_server_dns_name + ":" + submarine_server_port
+ return host
+
+
+class NotebookClient:
+ def __init__(self, host: str = generate_host()):
+ """
+ Submarine notebook client constructor
+ :param host: An HTTP URI like http://submarine-server:8080.
+ """
+ # TODO(pingsutw): support authentication for talking to the submarine server
+ self.host = host
+ configuration = Configuration()
+ configuration.host = host + "/api"
+ api_client = ApiClient(configuration=configuration)
+ self.notebook_api = NotebookApi(api_client=api_client)
+
+ def create_notebook(self, notebook_spec):
+ """
+ Create an notebook
+ :param notebook_spec: submarine notebook spec
+ :return: submarine notebook
+ """
+ response = self.notebook_api.create_notebook(notebook_spec=notebook_spec)
+ return response.result
+
+ def get_notebook(self, id):
+ """
+ Get the notebook's detailed info by id
+ :param id: submarine notebook id
+ :return: submarine notebook
+ """
+ response = self.notebook_api.get_notebook(id=id)
+ return response.result
+
+ def get_notebook_async(self, id):
+ """
+ Get the notebook's detailed info by id (async)
+ :param id: submarine notebook id
+ :return: multiprocessing.pool.ApplyResult
+ """
+ thread = self.notebook_api.get_notebook(id=id, async_req=True)
+ return thread
+
+ def list_notebooks(self, user_id):
+ """
+ List notebook instances which belong to user
+ :param user_id
+ :return: List of submarine notebooks
+ """
+ response = self.notebook_api.list_notebooks(id=user_id)
+ return response.result
+
+ def list_notebooks_async(self, user_id):
+ """
+ List notebook instances which belong to user (async)
+ :param user_id:
+ :return: multiprocessing.pool.ApplyResult
+ """
+ thread = self.notebook_api.list_notebooks(id=user_id, async_req=True)
+ return thread
+
+ def delete_notebook(self, id):
+ """
+ Delete the Submarine notebook
+ :param id: Submarine notebook id
+ :return: The detailed info about deleted submarine notebook
+ """
+ response = self.notebook_api.delete_notebook(id)
+ return response.result
+
+ def delete_notebook_async(self, id):
+ """
+ Delete the Submarine notebook (async)
+ :param id: Submarine notebook id
+ :return: The detailed info about deleted submarine notebook
+ """
+ thread = self.notebook_api.delete_notebook(id, async_req=True)
+ return thread
diff --git a/submarine-sdk/pysubmarine/tests/cli/test_notebook.py b/submarine-sdk/pysubmarine/tests/cli/test_notebook.py
index 83bb47a..eff73da 100644
--- a/submarine-sdk/pysubmarine/tests/cli/test_notebook.py
+++ b/submarine-sdk/pysubmarine/tests/cli/test_notebook.py
@@ -13,29 +13,68 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+import pytest
from click.testing import CliRunner
from submarine.cli import main
+from submarine.client.api.notebook_client import NotebookClient
+from submarine.client.models.environment_spec import EnvironmentSpec
+from submarine.client.models.notebook_meta import NotebookMeta
+from submarine.client.models.notebook_pod_spec import NotebookPodSpec
+from submarine.client.models.notebook_spec import NotebookSpec
+TEST_CONSOLE_WIDTH = 191
-def test_list_notebook():
- runner = CliRunner()
- result = runner.invoke(main.entry_point, ["list", "notebook"])
+
+@pytest.mark.e2e
+def test_all_notbook_e2e():
+ """E2E Test for using submarine CLI to access submarine notebook
+ To run this test, you should first set
+ your submarine CLI config `port` to 8080 and `hostname` to localhost
+ i.e. please execute the commands in your terminal:
+ submarine config set connection.hostname localhost
+ submarine config set connection.port 8080
+ """
+ # set env to display full table
+ runner = CliRunner(env={"COLUMNS": str(TEST_CONSOLE_WIDTH)})
+ # check if cli config is correct for testing
+ result = runner.invoke(main.entry_point, ["config", "get", "connection.port"])
assert result.exit_code == 0
- assert "list notebook!" in result.output
+ assert "connection.port={}".format(8080) in result.output
+ submarine_client = NotebookClient(host="http://localhost:8080")
-def test_get_notebook():
- mock_notebook_id = "0"
- runner = CliRunner()
- result = runner.invoke(main.entry_point, ["get", "notebook", mock_notebook_id])
- assert result.exit_code == 0
- assert "get notebook! id={}".format(mock_notebook_id) in result.output
+ mock_user_id = "4291d7da9005377ec9aec4a71ea837f"
+
+ notebook_meta = NotebookMeta(name="test-nb", namespace="default", owner_id=mock_user_id)
+ environment = EnvironmentSpec(name="notebook-env")
+ notebook_podSpec = NotebookPodSpec(
+ env_vars={"TEST_ENV": "test"}, resources="cpu=1,memory=1.0Gi"
+ )
+ notebookSpec = NotebookSpec(meta=notebook_meta, environment=environment, spec=notebook_podSpec)
+ notebook = submarine_client.create_notebook(notebookSpec)
+ notebookId = notebook["notebookId"]
-def test_delete_notebook():
- mock_notebook_id = "0"
- runner = CliRunner()
- result = runner.invoke(main.entry_point, ["delete", "notebook", mock_notebook_id])
+ # test list notebook
+ result = runner.invoke(main.entry_point, ["list", "notebook"])
assert result.exit_code == 0
- assert "delete notebook! id={}".format(mock_notebook_id) in result.output
+ assert "List of Notebooks" in result.output
+ assert notebook["name"] in result.output
+ assert notebook["notebookId"] in result.output
+ assert notebook["spec"]["environment"]["name"] in result.output
+ assert notebook["spec"]["spec"]["resources"] in result.output
+ # no need to check status (we do not wait for the notbook to run)
+
+ # test get notebook
+ result = runner.invoke(main.entry_point, ["get", "notebook", notebookId])
+ assert "Notebook(id = {} )".format(notebookId) in result.output
+ assert notebook["spec"]["environment"]["name"] in result.output
+
+ # test delete notebook
+ result = runner.invoke(main.entry_point, ["delete", "notebook", notebookId])
+ assert "Notebook(id = {} ) deleted".format(notebookId) in result.output
+
+ # test get environment fail after delete
+ result = runner.invoke(main.entry_point, ["get", "notebook", notebookId])
+ assert "[Api Error] Notebook not found." in result.output
diff --git a/submarine-sdk/pysubmarine/tests/notebook/test_notebook_client.py b/submarine-sdk/pysubmarine/tests/notebook/test_notebook_client.py
new file mode 100644
index 0000000..0e124c1
--- /dev/null
+++ b/submarine-sdk/pysubmarine/tests/notebook/test_notebook_client.py
@@ -0,0 +1,43 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pytest
+
+from submarine.client.api.notebook_client import NotebookClient
+from submarine.client.models.environment_spec import EnvironmentSpec
+from submarine.client.models.notebook_meta import NotebookMeta
+from submarine.client.models.notebook_pod_spec import NotebookPodSpec
+from submarine.client.models.notebook_spec import NotebookSpec
+
+
+@pytest.mark.e2e
+def test_notebook_e2e():
+ submarine_client = NotebookClient(host="http://localhost:8080")
+
+ mock_user_id = "4291d7da9005377ec9aec4a71ea837f"
+
+ notebook_meta = NotebookMeta(name="test-nb", namespace="default", owner_id=mock_user_id)
+ environment = EnvironmentSpec(name="notebook-env")
+ notebook_podSpec = NotebookPodSpec(
+ env_vars={"TEST_ENV": "test"}, resources="cpu=1,memory=1.0Gi"
+ )
+ notebookSpec = NotebookSpec(meta=notebook_meta, environment=environment, spec=notebook_podSpec)
+
+ notebook = submarine_client.create_notebook(notebookSpec)
+
+ notebookId = notebook["notebookId"]
+ submarine_client.get_notebook(notebookId)
+ submarine_client.list_notebooks(user_id=mock_user_id)
+ submarine_client.delete_notebook(notebookId)
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@submarine.apache.org
For additional commands, e-mail: dev-help@submarine.apache.org