You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uniffle.apache.org by ro...@apache.org on 2023/06/19 14:33:45 UTC
[incubator-uniffle] branch master updated: [#768] feat(cli): Cli method for blacklist update (#931)
This is an automated email from the ASF dual-hosted git repository.
roryqi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-uniffle.git
The following commit(s) were added to refs/heads/master by this push:
new f9369ab0 [#768] feat(cli): Cli method for blacklist update (#931)
f9369ab0 is described below
commit f9369ab03e459a6c3462b754b6ef3333e6a52563
Author: Junbo wang <10...@qq.com>
AuthorDate: Mon Jun 19 22:33:39 2023 +0800
[#768] feat(cli): Cli method for blacklist update (#931)
### What changes were proposed in this pull request?
I created a command called uniffle cli to update blacklist, the following is the case of using the command.
```
Usage:
Optional
-a,--admin <arg> This is an admin command that will print args.
-c,--cli <arg> This is an client cli command that will print args.
-h,--help Help for the Uniffle CLI.
-rc,--refreshChecker This is an admin command that will refresh access checker..
-cc,--checkerClass <arg> This is an client cli command that will print args.
-s,--help <arg> This is coordinator server host.
-p,--port <arg> This is coordinator server port.
```
> Run the example-cli command
uniffle admin-cli -rc -cc org.apache.uniffle.test.AccessClusterTest$MockedAccessChecker -s localhost -p 12345
### Why are the changes needed?
We use refreshChecker cli to update blacklist.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Unit test verification.
---
bin/uniffle | 2 +-
cli/pom.xml | 8 ++
.../java/org/apache/uniffle/api/AdminRestApi.java | 30 +++--
.../org/apache/uniffle/cli/UniffleAdminCLI.java | 129 +++++++++++++++++++++
.../apache/uniffle/client/HttpClientFactory.java | 60 ++++++++++
.../java/org/apache/uniffle/client/RestClient.java | 13 +--
.../org/apache/uniffle/client/RestClientConf.java | 57 +++++++++
.../org/apache/uniffle/client/RestClientImpl.java | 127 ++++++++++++++++++++
.../apache/uniffle/client/UniffleRestClient.java | 104 +++++++++++++++++
.../client/exception/UniffleRestException.java | 13 +--
.../org/apache/uniffle/cli/AdminRestApiTest.java | 58 +++++++++
.../apache/uniffle/cli/UniffleTestAdminCLI.java | 57 +++++++++
.../uniffle/coordinator/CoordinatorServer.java | 4 +
.../access/checker/AbstractAccessChecker.java | 7 +-
.../coordinator/access/checker/AccessChecker.java | 2 +
.../web/servlet/admin/RefreshCheckerServlet.java | 50 ++++++++
integration-test/common/pom.xml | 5 +
.../org/apache/uniffle/test/AccessClusterTest.java | 5 +
.../uniffle/test/CoordinatorAdminServiceTest.java | 71 ++++++++++++
pom.xml | 28 +++++
20 files changed, 804 insertions(+), 26 deletions(-)
diff --git a/bin/uniffle b/bin/uniffle
index 5b6fdd7b..977ab9b1 100755
--- a/bin/uniffle
+++ b/bin/uniffle
@@ -36,7 +36,7 @@ function uniffle_cmd_case
UNIFFLE_CLASSNAME=org.apache.uniffle.cli.UniffleCLI
;;
admin-cli)
- UNIFFLE_CLASSNAME=org.apache.uniffle.cli.UniffleCLI
+ UNIFFLE_CLASSNAME=org.apache.uniffle.cli.UniffleAdminCLI
;;
*)
UNIFFLE_CLASSNAME="${subcmd}"
diff --git a/cli/pom.xml b/cli/pom.xml
index a8eaf2d9..8914df73 100644
--- a/cli/pom.xml
+++ b/cli/pom.xml
@@ -41,5 +41,13 @@
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.uniffle</groupId>
+ <artifactId>rss-internal-client</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ </dependency>
</dependencies>
</project>
diff --git a/coordinator/src/main/java/org/apache/uniffle/coordinator/access/checker/AbstractAccessChecker.java b/cli/src/main/java/org/apache/uniffle/api/AdminRestApi.java
similarity index 57%
copy from coordinator/src/main/java/org/apache/uniffle/coordinator/access/checker/AbstractAccessChecker.java
copy to cli/src/main/java/org/apache/uniffle/api/AdminRestApi.java
index 677890c5..50d26ea0 100644
--- a/coordinator/src/main/java/org/apache/uniffle/coordinator/access/checker/AbstractAccessChecker.java
+++ b/cli/src/main/java/org/apache/uniffle/api/AdminRestApi.java
@@ -15,16 +15,30 @@
* limitations under the License.
*/
-package org.apache.uniffle.coordinator.access.checker;
+package org.apache.uniffle.api;
-import org.apache.uniffle.coordinator.AccessManager;
+import java.util.HashMap;
+import java.util.Map;
-/**
- * Abstract class for checking the access info from the client-side.
- */
-public abstract class AbstractAccessChecker implements AccessChecker {
+import org.apache.uniffle.client.RestClient;
+import org.apache.uniffle.client.UniffleRestClient;
+
+public class AdminRestApi {
+ private UniffleRestClient client;
+
+ private AdminRestApi() {
+ }
+
+ public AdminRestApi(UniffleRestClient client) {
+ this.client = client;
+ }
+
+ public String refreshAccessChecker() {
+ Map<String, Object> params = new HashMap<>();
+ return this.getClient().get("/api/admin/refreshChecker", params, null);
+ }
- protected AbstractAccessChecker(AccessManager accessManager) throws Exception {
-
+ private RestClient getClient() {
+ return this.client.getHttpClient();
}
}
diff --git a/cli/src/main/java/org/apache/uniffle/cli/UniffleAdminCLI.java b/cli/src/main/java/org/apache/uniffle/cli/UniffleAdminCLI.java
new file mode 100644
index 00000000..67d41463
--- /dev/null
+++ b/cli/src/main/java/org/apache/uniffle/cli/UniffleAdminCLI.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.uniffle.cli;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.uniffle.AbstractCustomCommandLine;
+import org.apache.uniffle.UniffleCliArgsException;
+import org.apache.uniffle.api.AdminRestApi;
+import org.apache.uniffle.client.UniffleRestClient;
+
+public class UniffleAdminCLI extends AbstractCustomCommandLine {
+
+ private static final Logger LOG = LoggerFactory.getLogger(UniffleAdminCLI.class);
+
+ private final Options allOptions;
+ private final Option refreshCheckerCli;
+ private final Option coordinatorHost;
+ private final Option coordPort;
+ private final Option ssl;
+
+ private final Option help;
+ protected UniffleRestClient client;
+
+ public UniffleAdminCLI(String shortPrefix, String longPrefix) {
+ allOptions = new Options();
+ refreshCheckerCli = new Option(shortPrefix + "r", longPrefix + "refreshChecker",
+ false, "This is an admin command that will refresh access checker.");
+ help = new Option(shortPrefix + "h", longPrefix + "help",
+ false, "Help for the Uniffle Admin CLI.");
+ coordinatorHost = new Option(shortPrefix + "s", longPrefix + "coordinatorHost",
+ true, "This is coordinator server host.");
+ coordPort = new Option(shortPrefix + "p", longPrefix + "port",
+ true, "This is coordinator server port.");
+ ssl = new Option(null, longPrefix + "ssl", false, "use SSL");
+
+ allOptions.addOption(refreshCheckerCli);
+ allOptions.addOption(coordinatorHost);
+ allOptions.addOption(coordPort);
+ allOptions.addOption(ssl);
+ allOptions.addOption(help);
+ }
+
+ public UniffleAdminCLI(String shortPrefix, String longPrefix, UniffleRestClient client) {
+ this(shortPrefix, longPrefix);
+ this.client = client;
+ }
+
+ public int run(String[] args) throws UniffleCliArgsException {
+ final CommandLine cmd = parseCommandLineOptions(args, true);
+
+ if (cmd.hasOption(help.getOpt())) {
+ printUsage();
+ return 0;
+ }
+
+ if (cmd.hasOption(coordinatorHost.getOpt()) && cmd.hasOption(coordPort.getOpt())) {
+ String host = cmd.getOptionValue(coordinatorHost.getOpt()).trim();
+ int port = Integer.parseInt(cmd.getOptionValue(coordPort.getOpt()).trim());
+ String hostUrl;
+ if (cmd.hasOption(ssl.getOpt())) {
+ hostUrl = String.format("https://%s:%d", host, port);
+ } else {
+ hostUrl = String.format("http://%s:%d", host, port);
+ }
+ LOG.info("connected to coordinator: {}.", hostUrl);
+ client = UniffleRestClient.builder(hostUrl).build();
+ }
+
+ if (cmd.hasOption(refreshCheckerCli.getOpt())) {
+ LOG.info("uniffle-admin-cli : refresh coordinator access checker!");
+ refreshAccessChecker();
+ return 0;
+ }
+ return 1;
+ }
+
+ private String refreshAccessChecker() throws UniffleCliArgsException {
+ if (client == null) {
+ throw new UniffleCliArgsException("Missing Coordinator host address and grpc port parameters.");
+ }
+ AdminRestApi adminRestApi = new AdminRestApi(client);
+ return adminRestApi.refreshAccessChecker();
+ }
+
+ @Override
+ public void addRunOptions(Options baseOptions) {
+ baseOptions.addOption(refreshCheckerCli);
+ baseOptions.addOption(coordinatorHost);
+ baseOptions.addOption(coordPort);
+ }
+
+ @Override
+ public void addGeneralOptions(Options baseOptions) {
+ baseOptions.addOption(help);
+ }
+
+ public static void main(String[] args) {
+ int retCode;
+ try {
+ final UniffleAdminCLI cli = new UniffleAdminCLI("", "");
+ retCode = cli.run(args);
+ } catch (UniffleCliArgsException e) {
+ retCode = AbstractCustomCommandLine.handleCliArgsException(e, LOG);
+ } catch (Exception e) {
+ retCode = AbstractCustomCommandLine.handleError(e, LOG);
+ }
+ System.exit(retCode);
+ }
+}
diff --git a/cli/src/main/java/org/apache/uniffle/client/HttpClientFactory.java b/cli/src/main/java/org/apache/uniffle/client/HttpClientFactory.java
new file mode 100644
index 00000000..dd0bae3c
--- /dev/null
+++ b/cli/src/main/java/org/apache/uniffle/client/HttpClientFactory.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.uniffle.client;
+
+import javax.net.ssl.SSLContext;
+
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.conn.ssl.NoopHostnameVerifier;
+import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
+import org.apache.http.conn.ssl.TrustStrategy;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.http.ssl.SSLContexts;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.uniffle.client.exception.UniffleRestException;
+
+
+public class HttpClientFactory {
+ private static final Logger LOG = LoggerFactory.getLogger(HttpClientFactory.class);
+
+ public static CloseableHttpClient createHttpClient(RestClientConf conf) {
+ RequestConfig requestConfig =
+ RequestConfig.custom()
+ .setSocketTimeout(conf.getSocketTimeout())
+ .setConnectTimeout(conf.getConnectTimeout())
+ .build();
+ SSLConnectionSocketFactory sslSocketFactory;
+ try {
+ TrustStrategy acceptingTrustStrategy = (cert, authType) -> true;
+ SSLContext sslContext =
+ SSLContexts.custom().loadTrustMaterial(null, acceptingTrustStrategy).build();
+ sslSocketFactory = new SSLConnectionSocketFactory(sslContext, NoopHostnameVerifier.INSTANCE);
+ } catch (Exception e) {
+ LOG.error("Error: ", e);
+ throw new UniffleRestException("Failed to create HttpClient", e);
+ }
+
+ return HttpClientBuilder.create()
+ .setDefaultRequestConfig(requestConfig)
+ .setSSLSocketFactory(sslSocketFactory)
+ .build();
+ }
+}
diff --git a/coordinator/src/main/java/org/apache/uniffle/coordinator/access/checker/AbstractAccessChecker.java b/cli/src/main/java/org/apache/uniffle/client/RestClient.java
similarity index 70%
copy from coordinator/src/main/java/org/apache/uniffle/coordinator/access/checker/AbstractAccessChecker.java
copy to cli/src/main/java/org/apache/uniffle/client/RestClient.java
index 677890c5..b867121d 100644
--- a/coordinator/src/main/java/org/apache/uniffle/coordinator/access/checker/AbstractAccessChecker.java
+++ b/cli/src/main/java/org/apache/uniffle/client/RestClient.java
@@ -15,16 +15,15 @@
* limitations under the License.
*/
-package org.apache.uniffle.coordinator.access.checker;
+package org.apache.uniffle.client;
-import org.apache.uniffle.coordinator.AccessManager;
+import java.util.Map;
/**
- * Abstract class for checking the access info from the client-side.
+ * A underlying http client interface for common rest request.
*/
-public abstract class AbstractAccessChecker implements AccessChecker {
+public interface RestClient extends AutoCloseable, Cloneable {
+
+ String get(String path, Map<String, Object> params, String authHeader);
- protected AbstractAccessChecker(AccessManager accessManager) throws Exception {
-
- }
}
diff --git a/cli/src/main/java/org/apache/uniffle/client/RestClientConf.java b/cli/src/main/java/org/apache/uniffle/client/RestClientConf.java
new file mode 100644
index 00000000..2224c28a
--- /dev/null
+++ b/cli/src/main/java/org/apache/uniffle/client/RestClientConf.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.uniffle.client;
+
+public class RestClientConf {
+ private int maxAttempts;
+ private int attemptWaitTime;
+ private int socketTimeout;
+ private int connectTimeout;
+
+ public int getMaxAttempts() {
+ return maxAttempts;
+ }
+
+ public void setMaxAttempts(int maxAttempts) {
+ this.maxAttempts = maxAttempts;
+ }
+
+ public int getAttemptWaitTime() {
+ return attemptWaitTime;
+ }
+
+ public void setAttemptWaitTime(int attemptWaitTime) {
+ this.attemptWaitTime = attemptWaitTime;
+ }
+
+ public int getSocketTimeout() {
+ return socketTimeout;
+ }
+
+ public void setSocketTimeout(int socketTimeout) {
+ this.socketTimeout = socketTimeout;
+ }
+
+ public int getConnectTimeout() {
+ return connectTimeout;
+ }
+
+ public void setConnectTimeout(int connectTimeout) {
+ this.connectTimeout = connectTimeout;
+ }
+}
diff --git a/cli/src/main/java/org/apache/uniffle/client/RestClientImpl.java b/cli/src/main/java/org/apache/uniffle/client/RestClientImpl.java
new file mode 100644
index 00000000..87bf8952
--- /dev/null
+++ b/cli/src/main/java/org/apache/uniffle/client/RestClientImpl.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.uniffle.client;
+
+import java.net.ConnectException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.Map;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpHeaders;
+import org.apache.http.NoHttpResponseException;
+import org.apache.http.client.HttpResponseException;
+import org.apache.http.client.ResponseHandler;
+import org.apache.http.client.methods.HttpUriRequest;
+import org.apache.http.client.methods.RequestBuilder;
+import org.apache.http.client.utils.URIBuilder;
+import org.apache.http.conn.ConnectTimeoutException;
+import org.apache.http.entity.ContentType;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.util.EntityUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.uniffle.client.exception.UniffleRestException;
+
+
+public class RestClientImpl implements RestClient {
+ private static final Logger LOG = LoggerFactory.getLogger(RestClientImpl.class);
+ private CloseableHttpClient httpclient;
+ private String baseUrl;
+
+ public RestClientImpl(String baseUrl, CloseableHttpClient httpclient) {
+ this.httpclient = httpclient;
+ this.baseUrl = baseUrl;
+ }
+
+ @Override
+ public void close() throws Exception {
+ if (httpclient != null) {
+ httpclient.close();
+ }
+ }
+
+ @Override
+ public String get(String path, Map<String, Object> params, String authHeader) {
+ return doRequest(buildURI(path, params), authHeader, RequestBuilder.get());
+ }
+
+ private String doRequest(URI uri, String authHeader, RequestBuilder requestBuilder) {
+ String response;
+ try {
+ if (requestBuilder.getFirstHeader(HttpHeaders.CONTENT_TYPE) == null) {
+ requestBuilder.setHeader(
+ HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON.getMimeType());
+ }
+ if (StringUtils.isNotBlank(authHeader)) {
+ requestBuilder.setHeader(HttpHeaders.AUTHORIZATION, authHeader);
+ }
+ HttpUriRequest httpRequest = requestBuilder.setUri(uri).build();
+
+ LOG.debug("Executing {} request: {}", httpRequest.getMethod(), uri);
+
+ ResponseHandler<String> responseHandler =
+ resp -> {
+ int status = resp.getStatusLine().getStatusCode();
+ HttpEntity entity = resp.getEntity();
+ String entityStr = entity != null ? EntityUtils.toString(entity) : null;
+ if (status >= 200 && status < 300) {
+ return entityStr;
+ } else {
+ throw new HttpResponseException(status, entityStr);
+ }
+ };
+
+ response = httpclient.execute(httpRequest, responseHandler);
+ LOG.debug("Response: {}", response);
+ } catch (ConnectException | ConnectTimeoutException | NoHttpResponseException e) {
+ // net exception can be retried by connecting to other Kyuubi server
+ throw new UniffleRestException("Api request failed for " + uri.toString(), e);
+ } catch (UniffleRestException rethrow) {
+ throw rethrow;
+ } catch (Exception e) {
+ LOG.error("Error: ", e);
+ throw new UniffleRestException("Api request failed for " + uri.toString(), e);
+ }
+
+ return response;
+ }
+
+
+ private URI buildURI(String path, Map<String, Object> params) {
+ URI uri;
+ try {
+ String url = StringUtils.isNotBlank(path) ? this.baseUrl + "/" + path : this.baseUrl;
+ URIBuilder builder = new URIBuilder(url);
+
+ if (!params.isEmpty()) {
+ for (Map.Entry<String, Object> entry : params.entrySet()) {
+ if (entry.getValue() != null) {
+ builder.addParameter(entry.getKey(), entry.getValue().toString());
+ }
+ }
+ }
+ uri = builder.build();
+ } catch (URISyntaxException e) {
+ throw new UniffleRestException("invalid URI.", e);
+ }
+ return uri;
+ }
+}
diff --git a/cli/src/main/java/org/apache/uniffle/client/UniffleRestClient.java b/cli/src/main/java/org/apache/uniffle/client/UniffleRestClient.java
new file mode 100644
index 00000000..eb45f4eb
--- /dev/null
+++ b/cli/src/main/java/org/apache/uniffle/client/UniffleRestClient.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.uniffle.client;
+
+import org.apache.http.impl.client.CloseableHttpClient;
+
+public class UniffleRestClient implements AutoCloseable {
+ private RestClient restClient;
+ private RestClientConf conf;
+ // This is the base host URL for the coordinator server.
+ // It should be in the format of "https://server:port" or "http://server:port".
+ private String hostUrl;
+
+ @Override
+ public void close() throws Exception {
+ if (restClient != null) {
+ restClient.close();
+ }
+ }
+
+ private UniffleRestClient(Builder builder) {
+ this.hostUrl = builder.hostUrl;
+
+ RestClientConf conf = new RestClientConf();
+ conf.setConnectTimeout(builder.connectTimeout);
+ conf.setSocketTimeout(builder.socketTimeout);
+ conf.setMaxAttempts(builder.maxAttempts);
+ conf.setAttemptWaitTime(builder.attemptWaitTime);
+ this.conf = conf;
+ CloseableHttpClient httpclient = HttpClientFactory.createHttpClient(conf);
+ this.restClient = new RestClientImpl(hostUrl, httpclient);
+ }
+
+ public RestClient getHttpClient() {
+ return restClient;
+ }
+
+ public RestClientConf getConf() {
+ return conf;
+ }
+
+ public static Builder builder(String hostUrl) {
+ return new Builder(hostUrl);
+ }
+
+ public static class Builder {
+
+ private String hostUrl;
+
+ // 2 minutes
+ private int socketTimeout = 2 * 60 * 1000;
+
+ // 30s
+ private int connectTimeout = 30 * 1000;
+
+ private int maxAttempts = 3;
+
+ // 3s
+ private int attemptWaitTime = 3 * 1000;
+
+ public Builder(String hostUrl) {
+ this.hostUrl = hostUrl;
+ }
+
+ public Builder socketTimeout(int socketTimeout) {
+ this.socketTimeout = socketTimeout;
+ return this;
+ }
+
+ public Builder connectionTimeout(int connectTimeout) {
+ this.connectTimeout = connectTimeout;
+ return this;
+ }
+
+ public Builder maxAttempts(int maxAttempts) {
+ this.maxAttempts = maxAttempts;
+ return this;
+ }
+
+ public Builder attemptWaitTime(int attemptWaitTime) {
+ this.attemptWaitTime = attemptWaitTime;
+ return this;
+ }
+
+ public UniffleRestClient build() {
+ return new UniffleRestClient(this);
+ }
+ }
+}
diff --git a/coordinator/src/main/java/org/apache/uniffle/coordinator/access/checker/AbstractAccessChecker.java b/cli/src/main/java/org/apache/uniffle/client/exception/UniffleRestException.java
similarity index 70%
copy from coordinator/src/main/java/org/apache/uniffle/coordinator/access/checker/AbstractAccessChecker.java
copy to cli/src/main/java/org/apache/uniffle/client/exception/UniffleRestException.java
index 677890c5..10b3cec2 100644
--- a/coordinator/src/main/java/org/apache/uniffle/coordinator/access/checker/AbstractAccessChecker.java
+++ b/cli/src/main/java/org/apache/uniffle/client/exception/UniffleRestException.java
@@ -15,16 +15,11 @@
* limitations under the License.
*/
-package org.apache.uniffle.coordinator.access.checker;
+package org.apache.uniffle.client.exception;
-import org.apache.uniffle.coordinator.AccessManager;
+public class UniffleRestException extends RuntimeException {
-/**
- * Abstract class for checking the access info from the client-side.
- */
-public abstract class AbstractAccessChecker implements AccessChecker {
-
- protected AbstractAccessChecker(AccessManager accessManager) throws Exception {
-
+ public UniffleRestException(String message, Throwable cause) {
+ super(message, cause);
}
}
diff --git a/cli/src/test/java/org/apache/uniffle/cli/AdminRestApiTest.java b/cli/src/test/java/org/apache/uniffle/cli/AdminRestApiTest.java
new file mode 100644
index 00000000..cb400a0a
--- /dev/null
+++ b/cli/src/test/java/org/apache/uniffle/cli/AdminRestApiTest.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.uniffle.cli;
+
+import java.util.HashMap;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.Answers;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.MockitoAnnotations;
+
+import org.apache.uniffle.UniffleCliArgsException;
+import org.apache.uniffle.api.AdminRestApi;
+import org.apache.uniffle.client.UniffleRestClient;
+
+public class AdminRestApiTest {
+
+ @Mock(answer = Answers.RETURNS_DEEP_STUBS)
+ private UniffleRestClient uniffleRestClient;
+
+ @InjectMocks
+ private AdminRestApi adminRestApi;
+
+ @BeforeEach
+ public void setup() throws Exception {
+ MockitoAnnotations.initMocks(this);
+ }
+
+ @Test
+ public void testRunRefreshAccessChecker() throws UniffleCliArgsException {
+ Mockito.when(uniffleRestClient.getHttpClient()
+ .get(Mockito.anyString(), Mockito.anyMap(), Mockito.anyString()))
+ .thenReturn("OK");
+ String result = adminRestApi.refreshAccessChecker();
+ Mockito.verify(uniffleRestClient.getHttpClient(),
+ Mockito.times(1)).get("/api/admin/refreshChecker",
+ new HashMap<>(), null);
+ }
+
+}
diff --git a/cli/src/test/java/org/apache/uniffle/cli/UniffleTestAdminCLI.java b/cli/src/test/java/org/apache/uniffle/cli/UniffleTestAdminCLI.java
new file mode 100644
index 00000000..60fc390c
--- /dev/null
+++ b/cli/src/test/java/org/apache/uniffle/cli/UniffleTestAdminCLI.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.uniffle.cli;
+
+import java.io.IOException;
+import java.util.HashMap;
+
+import org.junit.jupiter.api.Test;
+import org.mockito.Answers;
+import org.mockito.Mockito;
+
+import org.apache.uniffle.UniffleCliArgsException;
+import org.apache.uniffle.client.UniffleRestClient;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+public class UniffleTestAdminCLI {
+
+ @Test
+ public void testAdminRefreshCLI() throws UniffleCliArgsException, IOException {
+
+ UniffleRestClient client = Mockito.mock(UniffleRestClient.class, Answers.RETURNS_DEEP_STUBS);
+ Mockito.when(client.getHttpClient().get(Mockito.anyString(), Mockito.anyMap(), Mockito.anyString()))
+ .thenReturn("OK");
+ UniffleAdminCLI uniffleAdminCLI = new UniffleAdminCLI("", "", client);
+ String[] args = {"-r"};
+ assertEquals(0, uniffleAdminCLI.run(args));
+ Mockito.verify(client.getHttpClient(),
+ Mockito.times(1)).get("/api/admin/refreshChecker",
+ new HashMap<>(), null);
+ }
+
+
+ @Test
+ public void testMissingClientCLI() throws UniffleCliArgsException, IOException {
+ UniffleAdminCLI uniffleAdminCLI = new UniffleAdminCLI("", "");
+ String[] args = {"-r"};
+ assertThrows(UniffleCliArgsException.class, () -> uniffleAdminCLI.run(args));
+ }
+
+}
diff --git a/coordinator/src/main/java/org/apache/uniffle/coordinator/CoordinatorServer.java b/coordinator/src/main/java/org/apache/uniffle/coordinator/CoordinatorServer.java
index 3fb185f4..286ea179 100644
--- a/coordinator/src/main/java/org/apache/uniffle/coordinator/CoordinatorServer.java
+++ b/coordinator/src/main/java/org/apache/uniffle/coordinator/CoordinatorServer.java
@@ -45,6 +45,7 @@ import org.apache.uniffle.coordinator.util.CoordinatorUtils;
import org.apache.uniffle.coordinator.web.servlet.CancelDecommissionServlet;
import org.apache.uniffle.coordinator.web.servlet.DecommissionServlet;
import org.apache.uniffle.coordinator.web.servlet.NodesServlet;
+import org.apache.uniffle.coordinator.web.servlet.admin.RefreshCheckerServlet;
import static org.apache.uniffle.common.config.RssBaseConf.RSS_SECURITY_HADOOP_KERBEROS_ENABLE;
import static org.apache.uniffle.common.config.RssBaseConf.RSS_SECURITY_HADOOP_KERBEROS_KEYTAB_FILE;
@@ -201,6 +202,9 @@ public class CoordinatorServer extends ReconfigurableBase {
jettyServer.addServlet(
new CancelDecommissionServlet(this),
"/api/server/cancelDecommission");
+ jettyServer.addServlet(
+ new RefreshCheckerServlet(this),
+ "/api/admin/refreshChecker");
}
private void registerMetrics() throws Exception {
diff --git a/coordinator/src/main/java/org/apache/uniffle/coordinator/access/checker/AbstractAccessChecker.java b/coordinator/src/main/java/org/apache/uniffle/coordinator/access/checker/AbstractAccessChecker.java
index 677890c5..d3e0eb44 100644
--- a/coordinator/src/main/java/org/apache/uniffle/coordinator/access/checker/AbstractAccessChecker.java
+++ b/coordinator/src/main/java/org/apache/uniffle/coordinator/access/checker/AbstractAccessChecker.java
@@ -25,6 +25,11 @@ import org.apache.uniffle.coordinator.AccessManager;
public abstract class AbstractAccessChecker implements AccessChecker {
protected AbstractAccessChecker(AccessManager accessManager) throws Exception {
-
+
+ }
+
+ @Override
+ public void refreshAccessChecker() {
+
}
}
diff --git a/coordinator/src/main/java/org/apache/uniffle/coordinator/access/checker/AccessChecker.java b/coordinator/src/main/java/org/apache/uniffle/coordinator/access/checker/AccessChecker.java
index 71a380a2..53235383 100644
--- a/coordinator/src/main/java/org/apache/uniffle/coordinator/access/checker/AccessChecker.java
+++ b/coordinator/src/main/java/org/apache/uniffle/coordinator/access/checker/AccessChecker.java
@@ -34,4 +34,6 @@ public interface AccessChecker extends Closeable {
* @return access check result
*/
AccessCheckResult check(AccessInfo accessInfo);
+
+ void refreshAccessChecker();
}
diff --git a/coordinator/src/main/java/org/apache/uniffle/coordinator/web/servlet/admin/RefreshCheckerServlet.java b/coordinator/src/main/java/org/apache/uniffle/coordinator/web/servlet/admin/RefreshCheckerServlet.java
new file mode 100644
index 00000000..a61f1976
--- /dev/null
+++ b/coordinator/src/main/java/org/apache/uniffle/coordinator/web/servlet/admin/RefreshCheckerServlet.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.uniffle.coordinator.web.servlet.admin;
+
+import java.util.List;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.uniffle.coordinator.CoordinatorServer;
+import org.apache.uniffle.coordinator.access.checker.AccessChecker;
+import org.apache.uniffle.coordinator.web.Response;
+import org.apache.uniffle.coordinator.web.servlet.BaseServlet;
+
+public class RefreshCheckerServlet extends BaseServlet {
+
+ private static final Logger LOG = LoggerFactory.getLogger(RefreshCheckerServlet.class);
+ private final CoordinatorServer coordinator;
+
+ public RefreshCheckerServlet(CoordinatorServer coordinator) {
+ this.coordinator = coordinator;
+ }
+
+ @Override
+ protected Response handleGet(HttpServletRequest req, HttpServletResponse resp) {
+ List<AccessChecker> accessCheckers = coordinator.getAccessManager().getAccessCheckers();
+ LOG.info(
+ "The access checker {} has been refreshed, you can add the checker via rss.coordinator.access.checkers.",
+ accessCheckers);
+ accessCheckers.forEach(AccessChecker::refreshAccessChecker);
+ return Response.success(null);
+ }
+}
diff --git a/integration-test/common/pom.xml b/integration-test/common/pom.xml
index 8fb44c0f..952774a8 100644
--- a/integration-test/common/pom.xml
+++ b/integration-test/common/pom.xml
@@ -50,6 +50,11 @@
<artifactId>shuffle-server</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.uniffle</groupId>
+ <artifactId>cli</artifactId>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>org.apache.uniffle</groupId>
<artifactId>shuffle-server</artifactId>
diff --git a/integration-test/common/src/test/java/org/apache/uniffle/test/AccessClusterTest.java b/integration-test/common/src/test/java/org/apache/uniffle/test/AccessClusterTest.java
index 4e167e04..02a8a834 100644
--- a/integration-test/common/src/test/java/org/apache/uniffle/test/AccessClusterTest.java
+++ b/integration-test/common/src/test/java/org/apache/uniffle/test/AccessClusterTest.java
@@ -69,6 +69,11 @@ public class AccessClusterTest extends CoordinatorTestBase {
return new AccessCheckResult(false, "");
}
+ @Override
+ public void refreshAccessChecker() {
+ // ignore.
+ }
+
@Override
public void close() throws IOException {
// ignore.
diff --git a/integration-test/common/src/test/java/org/apache/uniffle/test/CoordinatorAdminServiceTest.java b/integration-test/common/src/test/java/org/apache/uniffle/test/CoordinatorAdminServiceTest.java
new file mode 100644
index 00000000..0aab33bc
--- /dev/null
+++ b/integration-test/common/src/test/java/org/apache/uniffle/test/CoordinatorAdminServiceTest.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.uniffle.test;
+
+import java.util.HashMap;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import org.apache.uniffle.api.AdminRestApi;
+import org.apache.uniffle.client.UniffleRestClient;
+import org.apache.uniffle.common.config.RssBaseConf;
+import org.apache.uniffle.coordinator.CoordinatorConf;
+import org.apache.uniffle.coordinator.web.Response;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+public class CoordinatorAdminServiceTest extends IntegrationTestBase {
+
+ private static final Integer JETTY_HTTP_PORT = 12345;
+ private static final String accessChecker = "org.apache.uniffle.test.AccessClusterTest$MockedAccessChecker";
+
+ private ObjectMapper objectMapper = new ObjectMapper();
+
+ protected AdminRestApi adminRestApi;
+
+
+ @BeforeAll
+ public static void setUp() throws Exception {
+ CoordinatorConf coordinatorConf = new CoordinatorConf();
+ coordinatorConf.set(RssBaseConf.JETTY_HTTP_PORT, JETTY_HTTP_PORT);
+ coordinatorConf.set(RssBaseConf.JETTY_CORE_POOL_SIZE, 128);
+ coordinatorConf.set(RssBaseConf.RPC_SERVER_PORT, 12346);
+ coordinatorConf.setString(CoordinatorConf.COORDINATOR_ACCESS_CHECKERS.key(), accessChecker);
+ createCoordinatorServer(coordinatorConf);
+ startServers();
+ }
+
+
+ @BeforeEach
+ public void createClient() {
+ String hostUrl = String.format("http://%s:%d", LOCALHOST, JETTY_HTTP_PORT);
+ adminRestApi = new AdminRestApi(UniffleRestClient.builder(hostUrl).build());
+ }
+
+ @Test
+ public void test() throws Exception {
+ String content = adminRestApi.refreshAccessChecker();
+ Response<HashMap> response = objectMapper.readValue(content,
+ new TypeReference<Response<HashMap>>() { });
+ assertEquals(0, response.getCode());
+ }
+}
diff --git a/pom.xml b/pom.xml
index 670e6862..64704b8f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -269,6 +269,12 @@
<artifactId>shuffle-server</artifactId>
<version>${project.version}</version>
</dependency>
+
+ <dependency>
+ <groupId>org.apache.uniffle</groupId>
+ <artifactId>cli</artifactId>
+ <version>${project.version}</version>
+ </dependency>
<dependency>
<groupId>org.apache.uniffle</groupId>
<artifactId>coordinator</artifactId>
@@ -356,6 +362,10 @@
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-json</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -468,9 +478,19 @@
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ </exclusion>
</exclusions>
</dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ <version>${httpclient.version}</version>
+ </dependency>
+
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
@@ -587,6 +607,10 @@
<groupId>javax.servlet</groupId>
<artifactId>servlet-api</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ </exclusion>
</exclusions>
</dependency>
@@ -1700,6 +1724,10 @@
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-json</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>