You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kyuubi.apache.org by fe...@apache.org on 2023/05/28 01:32:33 UTC
[kyuubi] branch master updated: [KYUUBI #4889] Admin command line supports list server command
This is an automated email from the ASF dual-hosted git repository.
feiwang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new 52d3bf25e [KYUUBI #4889] Admin command line supports list server command
52d3bf25e is described below
commit 52d3bf25ed5690f30ddfffcd01e2fc43e73223ce
Author: fwang12 <fw...@ebay.com>
AuthorDate: Sun May 28 09:32:23 2023 +0800
[KYUUBI #4889] Admin command line supports list server command
### _Why are the changes needed?_
Support to list server with kyuubi-admin/kyuubi rest client.
### _How was this patch tested?_
- [x] Add some test cases that check the changes thoroughly including negative and positive cases if possible
- UT for AdminRestAPI
- UT for AminCtlArgument
- UT for AdminCtl
- [ ] Add screenshots for manual tests if appropriate
- [x] [Run test](https://kyuubi.readthedocs.io/en/master/develop_tools/testing.html#running-tests) locally before make a pull request
Closes #4889 from turboFei/list_server.
Closes #4889
bfd13fbde [fwang12] nit
0a0131552 [fwang12] list server
Authored-by: fwang12 <fw...@ebay.com>
Signed-off-by: fwang12 <fw...@ebay.com>
---
docs/tools/kyuubi-admin.rst | 9 +++++
.../kyuubi/ctl/cli/AdminControlCliArguments.scala | 8 +++-
.../ctl/cmd/list/AdminListServerCommand.scala | 44 ++++++++++++++++++++++
.../apache/kyuubi/ctl/opt/AdminCommandLine.scala | 8 +++-
.../scala/org/apache/kyuubi/ctl/util/Render.scala | 15 +++++++-
.../kyuubi/ctl/AdminControlCliArgumentsSuite.scala | 11 +++++-
.../org/apache/kyuubi/client/AdminRestApi.java | 8 ++++
.../kyuubi/client/api/v1/dto/ServerData.java | 2 +
.../kyuubi/server/rest/client/AdminCtlSuite.scala | 18 ++++++++-
.../server/rest/client/AdminRestApiSuite.scala | 25 +++++++++++-
10 files changed, 142 insertions(+), 6 deletions(-)
diff --git a/docs/tools/kyuubi-admin.rst b/docs/tools/kyuubi-admin.rst
index 606396593..7fd07e973 100644
--- a/docs/tools/kyuubi-admin.rst
+++ b/docs/tools/kyuubi-admin.rst
@@ -98,6 +98,15 @@ Usage: ``bin/kyuubi-admin list engine [options]``
* - --hs2ProxyUser
- The proxy user to impersonate. When specified, it will list engines for the hs2ProxyUser.
+.. _list_server:
+
+List Servers
+-------------------------------------
+
+Prints a table of the key information about the servers.
+
+Usage: ``bin/kyuubi-admin list server``
+
.. _delete_engine:
Delete an Engine
diff --git a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cli/AdminControlCliArguments.scala b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cli/AdminControlCliArguments.scala
index 4bc1e1317..5a45630c6 100644
--- a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cli/AdminControlCliArguments.scala
+++ b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cli/AdminControlCliArguments.scala
@@ -22,7 +22,7 @@ import scopt.OParser
import org.apache.kyuubi.KyuubiException
import org.apache.kyuubi.ctl.cmd.Command
import org.apache.kyuubi.ctl.cmd.delete.AdminDeleteEngineCommand
-import org.apache.kyuubi.ctl.cmd.list.AdminListEngineCommand
+import org.apache.kyuubi.ctl.cmd.list.{AdminListEngineCommand, AdminListServerCommand}
import org.apache.kyuubi.ctl.cmd.refresh.RefreshConfigCommand
import org.apache.kyuubi.ctl.opt.{AdminCommandLine, CliConfig, ControlAction, ControlObject}
@@ -37,6 +37,7 @@ class AdminControlCliArguments(args: Seq[String], env: Map[String, String] = sys
cliConfig.action match {
case ControlAction.LIST => cliConfig.resource match {
case ControlObject.ENGINE => new AdminListEngineCommand(cliConfig)
+ case ControlObject.SERVER => new AdminListServerCommand(cliConfig)
case _ => throw new KyuubiException(s"Invalid resource: ${cliConfig.resource}")
}
case ControlAction.DELETE => cliConfig.resource match {
@@ -61,6 +62,11 @@ class AdminControlCliArguments(args: Seq[String], env: Map[String, String] = sys
| sharelevel ${cliConfig.engineOpts.engineShareLevel}
| sharesubdomain ${cliConfig.engineOpts.engineSubdomain}
""".stripMargin
+ case ControlObject.SERVER =>
+ s"""Parsed arguments:
+ | action ${cliConfig.action}
+ | resource ${cliConfig.resource}
+ """.stripMargin
case ControlObject.CONFIG =>
s"""Parsed arguments:
| action ${cliConfig.action}
diff --git a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/list/AdminListServerCommand.scala b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/list/AdminListServerCommand.scala
new file mode 100644
index 000000000..a8e7e5d06
--- /dev/null
+++ b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/list/AdminListServerCommand.scala
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kyuubi.ctl.cmd.list
+
+import scala.collection.JavaConverters._
+
+import org.apache.kyuubi.client.AdminRestApi
+import org.apache.kyuubi.client.api.v1.dto.ServerData
+import org.apache.kyuubi.ctl.RestClientFactory.withKyuubiRestClient
+import org.apache.kyuubi.ctl.cmd.AdminCtlCommand
+import org.apache.kyuubi.ctl.opt.CliConfig
+import org.apache.kyuubi.ctl.util.Render
+
+class AdminListServerCommand(cliConfig: CliConfig)
+ extends AdminCtlCommand[Seq[ServerData]](cliConfig) {
+
+ override def validate(): Unit = {}
+
+ override def doRun(): Seq[ServerData] = {
+ withKyuubiRestClient(normalizedCliConfig, null, conf) { kyuubiRestClient =>
+ val adminRestApi = new AdminRestApi(kyuubiRestClient)
+ adminRestApi.listServers().asScala
+ }
+ }
+
+ override def render(resp: Seq[ServerData]): Unit = {
+ info(Render.renderServerNodesInfo(resp))
+ }
+}
diff --git a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/opt/AdminCommandLine.scala b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/opt/AdminCommandLine.scala
index b1a70935b..71e4068e5 100644
--- a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/opt/AdminCommandLine.scala
+++ b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/opt/AdminCommandLine.scala
@@ -64,7 +64,8 @@ object AdminCommandLine extends CommonCommandLine {
.text("\tList information about resources.")
.action((_, c) => c.copy(action = ControlAction.LIST))
.children(
- engineCmd(builder).text("\tList all the engine nodes for a user")))
+ engineCmd(builder).text("\tList all the engine nodes for a user"),
+ serverCmd(builder).text("\tList all the server nodes")))
}
@@ -94,6 +95,11 @@ object AdminCommandLine extends CommonCommandLine {
.text("The engine share level this engine belong to."))
}
+ private def serverCmd(builder: OParserBuilder[CliConfig]): OParser[_, CliConfig] = {
+ import builder._
+ cmd("server").action((_, c) => c.copy(resource = ControlObject.SERVER))
+ }
+
private def refreshConfigCmd(builder: OParserBuilder[CliConfig]): OParser[_, CliConfig] = {
import builder._
cmd("config").action((_, c) => c.copy(resource = ControlObject.CONFIG))
diff --git a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/util/Render.scala b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/util/Render.scala
index 2d4879e42..bb6e3529d 100644
--- a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/util/Render.scala
+++ b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/util/Render.scala
@@ -19,7 +19,7 @@ package org.apache.kyuubi.ctl.util
import scala.collection.JavaConverters._
import scala.collection.mutable.ListBuffer
-import org.apache.kyuubi.client.api.v1.dto.{Batch, Engine, GetBatchesResponse, SessionData}
+import org.apache.kyuubi.client.api.v1.dto.{Batch, Engine, GetBatchesResponse, ServerData, SessionData}
import org.apache.kyuubi.ctl.util.DateTimeUtils._
import org.apache.kyuubi.ha.client.ServiceNodeInfo
@@ -45,6 +45,19 @@ private[ctl] object Render {
Tabulator.format(title, header, rows)
}
+ def renderServerNodesInfo(serverNodesInfo: Seq[ServerData]): String = {
+ val title = s"Server Node List (total ${serverNodesInfo.size})"
+ val header = Array("Namespace", "Instance", "Attributes", "Status")
+ val rows = serverNodesInfo.map { server =>
+ Array(
+ server.getNamespace,
+ server.getInstance,
+ server.getAttributes.asScala.map { case (k, v) => s"$k=$v" }.mkString("\n"),
+ server.getStatus)
+ }.toArray
+ Tabulator.format(title, header, rows)
+ }
+
def renderSessionDataListInfo(sessions: Seq[SessionData]): String = {
val title = s"Live Session List (total ${sessions.size})"
val header = Array(
diff --git a/kyuubi-ctl/src/test/scala/org/apache/kyuubi/ctl/AdminControlCliArgumentsSuite.scala b/kyuubi-ctl/src/test/scala/org/apache/kyuubi/ctl/AdminControlCliArgumentsSuite.scala
index dab796127..fdadd011b 100644
--- a/kyuubi-ctl/src/test/scala/org/apache/kyuubi/ctl/AdminControlCliArgumentsSuite.scala
+++ b/kyuubi-ctl/src/test/scala/org/apache/kyuubi/ctl/AdminControlCliArgumentsSuite.scala
@@ -115,6 +115,13 @@ class AdminControlCliArgumentsSuite extends KyuubiFunSuite with TestPrematureExi
}
}
+ test("test list server") {
+ val args = Array("list", "server")
+ val opArgs = new AdminControlCliArguments(args)
+ assert(opArgs.cliConfig.action.toString === "LIST")
+ assert(opArgs.cliConfig.resource.toString === "SERVER")
+ }
+
test("test --help") {
// scalastyle:off
val helpString =
@@ -130,7 +137,7 @@ class AdminControlCliArgumentsSuite extends KyuubiFunSuite with TestPrematureExi
| --hs2ProxyUser <value> The value of hive.server2.proxy.user config.
| --conf <value> Kyuubi config property pair, formatted key=value.
|
- |Command: list [engine]
+ |Command: list [engine|server]
| List information about resources.
|Command: list engine [options]
| List all the engine nodes for a user
@@ -140,6 +147,8 @@ class AdminControlCliArgumentsSuite extends KyuubiFunSuite with TestPrematureExi
| The engine subdomain this engine belong to.
| -esl, --engine-share-level <value>
| The engine share level this engine belong to.
+ |Command: list server
+ | List all the server nodes
|
|Command: delete [engine]
| Delete resources.
diff --git a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/AdminRestApi.java b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/AdminRestApi.java
index c81af593a..a983827c8 100644
--- a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/AdminRestApi.java
+++ b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/AdminRestApi.java
@@ -23,6 +23,7 @@ import java.util.List;
import java.util.Map;
import org.apache.kyuubi.client.api.v1.dto.Engine;
import org.apache.kyuubi.client.api.v1.dto.OperationData;
+import org.apache.kyuubi.client.api.v1.dto.ServerData;
import org.apache.kyuubi.client.api.v1.dto.SessionData;
public class AdminRestApi {
@@ -99,6 +100,13 @@ public class AdminRestApi {
return this.getClient().delete(url, null, client.getAuthHeader());
}
+ public List<ServerData> listServers() {
+ ServerData[] result =
+ this.getClient()
+ .get(API_BASE_PATH + "/server", null, ServerData[].class, client.getAuthHeader());
+ return Arrays.asList(result);
+ }
+
private IRestClient getClient() {
return this.client.getHttpClient();
}
diff --git a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/ServerData.java b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/ServerData.java
index d64d43a72..6fe036162 100644
--- a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/ServerData.java
+++ b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/ServerData.java
@@ -29,6 +29,8 @@ public class ServerData {
private Map<String, String> attributes;
private String status;
+ public ServerData() {}
+
public ServerData(
String nodeName,
String namespace,
diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/server/rest/client/AdminCtlSuite.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/server/rest/client/AdminCtlSuite.scala
index 389b67e47..986b171c1 100644
--- a/kyuubi-server/src/test/scala/org/apache/kyuubi/server/rest/client/AdminCtlSuite.scala
+++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/server/rest/client/AdminCtlSuite.scala
@@ -19,13 +19,16 @@ package org.apache.kyuubi.server.rest.client
import java.util.UUID
+import org.mockito.Mockito.lenient
+import org.scalatestplus.mockito.MockitoSugar.mock
+
import org.apache.kyuubi.{KYUUBI_VERSION, RestClientTestHelper}
import org.apache.kyuubi.config.KyuubiConf
import org.apache.kyuubi.ctl.{CtlConf, TestPrematureExit}
import org.apache.kyuubi.engine.EngineRef
import org.apache.kyuubi.ha.HighAvailabilityConf
+import org.apache.kyuubi.ha.client.{DiscoveryPaths, ServiceDiscovery}
import org.apache.kyuubi.ha.client.DiscoveryClientProvider.withDiscoveryClient
-import org.apache.kyuubi.ha.client.DiscoveryPaths
import org.apache.kyuubi.plugin.PluginLoader
class AdminCtlSuite extends RestClientTestHelper with TestPrematureExit {
@@ -102,4 +105,17 @@ class AdminCtlSuite extends RestClientTestHelper with TestPrematureExit {
args,
"Engine Node List (total 0)")
}
+
+ test("list server") {
+ // Mock Kyuubi Server
+ val serverDiscovery = mock[ServiceDiscovery]
+ lenient.when(serverDiscovery.fe).thenReturn(fe)
+ val namespace = conf.get(HighAvailabilityConf.HA_NAMESPACE)
+ withDiscoveryClient(conf) { client =>
+ client.registerService(conf, namespace, serverDiscovery)
+
+ val args = Array("list", "server", "--authSchema", "spnego")
+ testPrematureExitForAdminControlCli(args, "Server Node List (total 1)")
+ }
+ }
}
diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/server/rest/client/AdminRestApiSuite.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/server/rest/client/AdminRestApiSuite.scala
index b79e62a12..91cd33e58 100644
--- a/kyuubi-server/src/test/scala/org/apache/kyuubi/server/rest/client/AdminRestApiSuite.scala
+++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/server/rest/client/AdminRestApiSuite.scala
@@ -22,14 +22,16 @@ import java.util.UUID
import scala.collection.JavaConverters.asScalaBufferConverter
import org.apache.hive.service.rpc.thrift.TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V2
+import org.mockito.Mockito.lenient
+import org.scalatestplus.mockito.MockitoSugar.mock
import org.apache.kyuubi.{KYUUBI_VERSION, RestClientTestHelper}
import org.apache.kyuubi.client.{AdminRestApi, KyuubiRestClient}
import org.apache.kyuubi.config.{KyuubiConf, KyuubiReservedKeys}
import org.apache.kyuubi.engine.EngineRef
import org.apache.kyuubi.ha.HighAvailabilityConf
+import org.apache.kyuubi.ha.client.{DiscoveryPaths, ServiceDiscovery}
import org.apache.kyuubi.ha.client.DiscoveryClientProvider.withDiscoveryClient
-import org.apache.kyuubi.ha.client.DiscoveryPaths
import org.apache.kyuubi.plugin.PluginLoader
class AdminRestApiSuite extends RestClientTestHelper {
@@ -148,4 +150,25 @@ class AdminRestApiSuite extends RestClientTestHelper {
assert(!operations.map(op => op.getIdentifier).contains(operation.identifier.toString))
}
+
+ test("list server") {
+ val spnegoKyuubiRestClient: KyuubiRestClient =
+ KyuubiRestClient.builder(baseUri.toString)
+ .authHeaderMethod(KyuubiRestClient.AuthHeaderMethod.SPNEGO)
+ .spnegoHost("localhost")
+ .build()
+ val adminRestApi = new AdminRestApi(spnegoKyuubiRestClient)
+
+ // Mock Kyuubi Server
+ val serverDiscovery = mock[ServiceDiscovery]
+ lenient.when(serverDiscovery.fe).thenReturn(fe)
+ val namespace = conf.get(HighAvailabilityConf.HA_NAMESPACE)
+ withDiscoveryClient(conf) { client =>
+ client.registerService(conf, namespace, serverDiscovery)
+
+ val servers = adminRestApi.listServers().asScala
+ assert(servers.nonEmpty)
+ assert(servers.map(s => s.getInstance()).contains(server.frontendServices.last.connectionUrl))
+ }
+ }
}