You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kyuubi.apache.org by bo...@apache.org on 2023/02/02 09:46:24 UTC
[kyuubi] branch master updated: [KYUUBI #4221] REST client for creating batch with uploading resource file
This is an automated email from the ASF dual-hosted git repository.
bowenliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new 37cf4cb29 [KYUUBI #4221] REST client for creating batch with uploading resource file
37cf4cb29 is described below
commit 37cf4cb29ec61c82ed022b81fc11c6908ea785e5
Author: liangbowen <li...@gf.com.cn>
AuthorDate: Thu Feb 2 17:46:14 2023 +0800
[KYUUBI #4221] REST client for creating batch with uploading resource file
### _Why are the changes needed?_
- REST client support for creating batch with uploading resource file (https://kyuubi.readthedocs.io/en/master/client/rest/rest_api.html#post-batches-with-uploading-resource) introduced in #4144
### _How was this patch tested?_
- [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible
- [ ] Add screenshots for manual tests if appropriate
- [x] [Run test](https://kyuubi.readthedocs.io/en/master/develop_tools/testing.html#running-tests) locally before make a pull request
Closes #4221 from bowenliang123/restclient-batch-upload.
Closes #4221
9ec83766 [liangbowen] add MultiPart class to avoid exposing httpclient classes to RestClient interface
83051843 [liangbowen] fix content type
056edb84 [liangbowen] nit
74ad5642 [liangbowen] style
68dcdb38 [liangbowen] update dependencyList
a274d392 [liangbowen] client for creating batch with uploading resource file
Authored-by: liangbowen <li...@gf.com.cn>
Signed-off-by: liangbowen <li...@gf.com.cn>
---
dev/dependencyList | 1 +
kyuubi-rest-client/pom.xml | 5 +++
.../org/apache/kyuubi/client/BatchRestApi.java | 14 ++++---
.../java/org/apache/kyuubi/client/IRestClient.java | 3 ++
.../java/org/apache/kyuubi/client/RestClient.java | 48 +++++++++++++++++++---
.../v1/dto/MultiPart.java} | 34 ++++++++++-----
.../server/rest/client/BatchRestApiSuite.scala | 20 +++++++++
pom.xml | 6 +++
8 files changed, 111 insertions(+), 20 deletions(-)
diff --git a/dev/dependencyList b/dev/dependencyList
index 268d10ca2..1c73c13e2 100644
--- a/dev/dependencyList
+++ b/dev/dependencyList
@@ -61,6 +61,7 @@ hk2-locator/2.6.1//hk2-locator-2.6.1.jar
hk2-utils/2.6.1//hk2-utils-2.6.1.jar
httpclient/4.5.13//httpclient-4.5.13.jar
httpcore/4.4.15//httpcore-4.4.15.jar
+httpmime/4.5.13//httpmime-4.5.13.jar
j2objc-annotations/1.3//j2objc-annotations-1.3.jar
jackson-annotations/2.14.1//jackson-annotations-2.14.1.jar
jackson-core/2.14.1//jackson-core-2.14.1.jar
diff --git a/kyuubi-rest-client/pom.xml b/kyuubi-rest-client/pom.xml
index 6e07b6267..28b1670f1 100644
--- a/kyuubi-rest-client/pom.xml
+++ b/kyuubi-rest-client/pom.xml
@@ -59,6 +59,11 @@
<artifactId>httpclient</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpmime</artifactId>
+ </dependency>
+
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client-api</artifactId>
diff --git a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/BatchRestApi.java b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/BatchRestApi.java
index a5f27590e..a09ab562b 100644
--- a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/BatchRestApi.java
+++ b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/BatchRestApi.java
@@ -17,13 +17,10 @@
package org.apache.kyuubi.client;
+import java.io.File;
import java.util.HashMap;
import java.util.Map;
-import org.apache.kyuubi.client.api.v1.dto.Batch;
-import org.apache.kyuubi.client.api.v1.dto.BatchRequest;
-import org.apache.kyuubi.client.api.v1.dto.CloseBatchResponse;
-import org.apache.kyuubi.client.api.v1.dto.GetBatchesResponse;
-import org.apache.kyuubi.client.api.v1.dto.OperationLog;
+import org.apache.kyuubi.client.api.v1.dto.*;
import org.apache.kyuubi.client.util.JsonUtils;
public class BatchRestApi {
@@ -43,6 +40,13 @@ public class BatchRestApi {
return this.getClient().post(API_BASE_PATH, requestBody, Batch.class, client.getAuthHeader());
}
+ public Batch createBatch(BatchRequest request, File resourceFile) {
+ Map<String, MultiPart> multiPartMap = new HashMap<>();
+ multiPartMap.put("batchRequest", new MultiPart(MultiPart.MultiPartType.JSON, request));
+ multiPartMap.put("resourceFile", new MultiPart(MultiPart.MultiPartType.FILE, resourceFile));
+ return this.getClient().post(API_BASE_PATH, multiPartMap, Batch.class, client.getAuthHeader());
+ }
+
public Batch getBatchById(String batchId) {
String path = String.format("%s/%s", API_BASE_PATH, batchId);
return this.getClient().get(path, null, Batch.class, client.getAuthHeader());
diff --git a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/IRestClient.java b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/IRestClient.java
index 50436ef73..66897df54 100644
--- a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/IRestClient.java
+++ b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/IRestClient.java
@@ -18,6 +18,7 @@
package org.apache.kyuubi.client;
import java.util.Map;
+import org.apache.kyuubi.client.api.v1.dto.MultiPart;
/** A underlying http client interface for common rest request. */
public interface IRestClient extends AutoCloseable {
@@ -27,6 +28,8 @@ public interface IRestClient extends AutoCloseable {
<T> T post(String path, String body, Class<T> type, String authHeader);
+ <T> T post(String path, Map<String, MultiPart> multiPartMap, Class<T> type, String authHeader);
+
String post(String path, String body, String authHeader);
<T> T delete(String path, Map<String, Object> params, Class<T> type, String authHeader);
diff --git a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/RestClient.java b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/RestClient.java
index 7b93f559e..fa3544726 100644
--- a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/RestClient.java
+++ b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/RestClient.java
@@ -17,6 +17,7 @@
package org.apache.kyuubi.client;
+import java.io.File;
import java.net.ConnectException;
import java.net.URI;
import java.net.URISyntaxException;
@@ -32,9 +33,15 @@ import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.methods.RequestBuilder;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.conn.ConnectTimeoutException;
+import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
+import org.apache.http.entity.mime.MultipartEntityBuilder;
+import org.apache.http.entity.mime.content.ContentBody;
+import org.apache.http.entity.mime.content.FileBody;
+import org.apache.http.entity.mime.content.StringBody;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.util.EntityUtils;
+import org.apache.kyuubi.client.api.v1.dto.MultiPart;
import org.apache.kyuubi.client.exception.KyuubiRestException;
import org.apache.kyuubi.client.exception.RetryableKyuubiRestException;
import org.apache.kyuubi.client.util.JsonUtils;
@@ -87,6 +94,37 @@ public class RestClient implements IRestClient {
return doRequest(buildURI(path), authHeader, postRequestBuilder);
}
+ @Override
+ public <T> T post(
+ String path, Map<String, MultiPart> multiPartMap, Class<T> type, String authHeader) {
+ MultipartEntityBuilder entityBuilder =
+ MultipartEntityBuilder.create().setCharset(StandardCharsets.UTF_8);
+ multiPartMap.forEach(
+ (s, multiPart) -> {
+ ContentBody contentBody;
+ Object payload = multiPart.getPayload();
+ switch (multiPart.getType()) {
+ case JSON:
+ String string =
+ (payload instanceof String) ? (String) payload : JsonUtils.toJson(payload);
+ contentBody = new StringBody(string, ContentType.APPLICATION_JSON);
+ break;
+ case FILE:
+ contentBody = new FileBody((File) payload);
+ break;
+ default:
+ throw new RuntimeException("Unsupported multi part type:" + multiPart);
+ }
+ entityBuilder.addPart(s, contentBody);
+ });
+ HttpEntity httpEntity = entityBuilder.build();
+ RequestBuilder postRequestBuilder = RequestBuilder.post(buildURI(path));
+ postRequestBuilder.setHeader(httpEntity.getContentType());
+ postRequestBuilder.setEntity(httpEntity);
+ String responseJson = doRequest(buildURI(path), authHeader, postRequestBuilder);
+ return JsonUtils.fromJson(responseJson, type);
+ }
+
@Override
public <T> T delete(String path, Map<String, Object> params, Class<T> type, String authHeader) {
String responseJson = delete(path, params, authHeader);
@@ -101,14 +139,14 @@ public class RestClient implements IRestClient {
private String doRequest(URI uri, String authHeader, RequestBuilder requestBuilder) {
String response;
try {
+ if (requestBuilder.getFirstHeader(HttpHeaders.CONTENT_TYPE) == null) {
+ requestBuilder.setHeader(
+ HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON.getMimeType());
+ }
if (StringUtils.isNotBlank(authHeader)) {
requestBuilder.setHeader(HttpHeaders.AUTHORIZATION, authHeader);
}
- HttpUriRequest httpRequest =
- requestBuilder
- .setUri(uri)
- .setHeader(HttpHeaders.CONTENT_TYPE, "application/json")
- .build();
+ HttpUriRequest httpRequest = requestBuilder.setUri(uri).build();
LOG.debug("Executing {} request: {}", httpRequest.getMethod(), uri);
diff --git a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/IRestClient.java b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/MultiPart.java
similarity index 55%
copy from kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/IRestClient.java
copy to kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/MultiPart.java
index 50436ef73..997a629fe 100644
--- a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/IRestClient.java
+++ b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/MultiPart.java
@@ -15,21 +15,35 @@
* limitations under the License.
*/
-package org.apache.kyuubi.client;
+package org.apache.kyuubi.client.api.v1.dto;
-import java.util.Map;
+public class MultiPart {
+ private MultiPartType type;
+ private Object payload;
-/** A underlying http client interface for common rest request. */
-public interface IRestClient extends AutoCloseable {
- <T> T get(String path, Map<String, Object> params, Class<T> type, String authHeader);
+ public enum MultiPartType {
+ FILE,
+ JSON
+ }
- String get(String path, Map<String, Object> params, String authHeader);
+ public MultiPart(MultiPartType type, Object obj) {
+ this.type = type;
+ this.payload = obj;
+ }
- <T> T post(String path, String body, Class<T> type, String authHeader);
+ public Object getPayload() {
+ return payload;
+ }
- String post(String path, String body, String authHeader);
+ public void setPayload(Object payload) {
+ this.payload = payload;
+ }
- <T> T delete(String path, Map<String, Object> params, Class<T> type, String authHeader);
+ public MultiPartType getType() {
+ return type;
+ }
- String delete(String path, Map<String, Object> params, String authHeader);
+ public void setType(MultiPartType type) {
+ this.type = type;
+ }
}
diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/server/rest/client/BatchRestApiSuite.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/server/rest/client/BatchRestApiSuite.scala
index b425f62d6..6110ccb1e 100644
--- a/kyuubi-server/src/test/scala/org/apache/kyuubi/server/rest/client/BatchRestApiSuite.scala
+++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/server/rest/client/BatchRestApiSuite.scala
@@ -17,6 +17,7 @@
package org.apache.kyuubi.server.rest.client
+import java.nio.file.Paths
import java.util.Base64
import org.scalatest.time.SpanSugar.convertIntToGrainOfTime
@@ -83,6 +84,25 @@ class BatchRestApiSuite extends RestClientTestHelper with BatchTestHelper {
basicKyuubiRestClient.close()
}
+ test("basic batch rest client with uploading resource file") {
+ val basicKyuubiRestClient: KyuubiRestClient =
+ KyuubiRestClient.builder(baseUri.toString)
+ .authHeaderMethod(KyuubiRestClient.AuthHeaderMethod.BASIC)
+ .username(ldapUser)
+ .password(ldapUserPasswd)
+ .socketTimeout(30000)
+ .build()
+ val batchRestApi: BatchRestApi = new BatchRestApi(basicKyuubiRestClient)
+
+ val requestObj = newSparkBatchRequest(Map("spark.master" -> "local"))
+ val exampleJarFile = Paths.get(sparkBatchTestResource.get).toFile
+ val batch: Batch = batchRestApi.createBatch(requestObj, exampleJarFile)
+
+ assert(batch.getKyuubiInstance === fe.connectionUrl)
+ assert(batch.getBatchType === "SPARK")
+ basicKyuubiRestClient.close()
+ }
+
test("basic batch rest client with invalid user") {
val totalConnections =
MetricsSystem.counterValue(MetricsConstants.REST_CONN_TOTAL).getOrElse(0L)
diff --git a/pom.xml b/pom.xml
index b13dae18c..405065dc9 100644
--- a/pom.xml
+++ b/pom.xml
@@ -807,6 +807,12 @@
</exclusions>
</dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpmime</artifactId>
+ <version>${httpclient.version}</version>
+ </dependency>
+
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpcore</artifactId>