You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kyuubi.apache.org by fe...@apache.org on 2023/02/20 03:18:42 UTC
[kyuubi] branch branch-1.7 updated: [KYUUBI #4216] Support to transfer client version for kyuubi hive jdbc and rest client sdk
This is an automated email from the ASF dual-hosted git repository.
feiwang pushed a commit to branch branch-1.7
in repository https://gitbox.apache.org/repos/asf/kyuubi.git
The following commit(s) were added to refs/heads/branch-1.7 by this push:
new c14e914f0 [KYUUBI #4216] Support to transfer client version for kyuubi hive jdbc and rest client sdk
c14e914f0 is described below
commit c14e914f0cb9b72d74842bd7c77a320c83e9b5cb
Author: fwang12 <fw...@ebay.com>
AuthorDate: Mon Feb 20 11:18:21 2023 +0800
[KYUUBI #4216] Support to transfer client version for kyuubi hive jdbc and rest client sdk
### _Why are the changes needed?_
To close #4216
### _How was this patch tested?_
- [x] Add some test cases that check the changes thoroughly including negative and positive cases if possible
- [ ] Add screenshots for manual tests if appropriate
- [x] [Run test](https://kyuubi.readthedocs.io/en/master/develop_tools/testing.html#running-tests) locally before make a pull request
Closes #4370 from turboFei/client_version.
Closes #4216
efd934c6b [fwang12] save
cf8acd54a [fwang12] version properties
Authored-by: fwang12 <fw...@ebay.com>
Signed-off-by: fwang12 <fw...@ebay.com>
(cherry picked from commit 3f6ba776fbda59e0c424283ababc4b6b18912b21)
Signed-off-by: fwang12 <fw...@ebay.com>
---
.../apache/kyuubi/config/KyuubiReservedKeys.scala | 1 +
kyuubi-hive-jdbc/pom.xml | 8 ++++
.../apache/kyuubi/jdbc/hive/KyuubiConnection.java | 1 +
.../java/org/apache/kyuubi/jdbc/hive/Utils.java | 17 +++++++++
.../src/main/resources/version.properties | 18 +++++++++
.../org/apache/kyuubi/jdbc/hive/UtilsTest.java | 7 ++++
kyuubi-rest-client/pom.xml | 7 ++++
.../org/apache/kyuubi/client/BatchRestApi.java | 12 ++++++
.../apache/kyuubi/client/util/VersionUtils.java | 43 ++++++++++++++++++++++
.../src/main/resources/version.properties | 18 +++++++++
.../kyuubi/client/util/VersionUtilsTest.java | 30 +++++++++++++++
.../KyuubiOperationPerConnectionSuite.scala | 12 +++++-
.../server/rest/client/BatchRestApiSuite.scala | 21 ++++++++++-
13 files changed, 192 insertions(+), 3 deletions(-)
diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiReservedKeys.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiReservedKeys.scala
index 6036af855..335253925 100644
--- a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiReservedKeys.scala
+++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiReservedKeys.scala
@@ -19,6 +19,7 @@ package org.apache.kyuubi.config
object KyuubiReservedKeys {
final val KYUUBI_CLIENT_IP_KEY = "kyuubi.client.ipAddress"
+ final val KYUUBI_CLIENT_VERSION_KEY = "kyuubi.client.version"
final val KYUUBI_SERVER_IP_KEY = "kyuubi.server.ipAddress"
final val KYUUBI_SESSION_USER_KEY = "kyuubi.session.user"
final val KYUUBI_SESSION_SIGN_PUBLICKEY = "kyuubi.session.sign.publickey"
diff --git a/kyuubi-hive-jdbc/pom.xml b/kyuubi-hive-jdbc/pom.xml
index 4d9648e75..e471c28ca 100644
--- a/kyuubi-hive-jdbc/pom.xml
+++ b/kyuubi-hive-jdbc/pom.xml
@@ -171,6 +171,14 @@
</dependencies>
<build>
+
+ <resources>
+ <resource>
+ <filtering>true</filtering>
+ <directory>src/main/resources</directory>
+ </resource>
+ </resources>
+
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
diff --git a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiConnection.java b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiConnection.java
index 9931dcec2..0932ea565 100644
--- a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiConnection.java
+++ b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiConnection.java
@@ -738,6 +738,7 @@ public class KyuubiConnection implements SQLConnection, KyuubiLoggable {
} catch (UnknownHostException e) {
LOG.debug("Error getting Kyuubi session local client ip address", e);
}
+ openConf.put(Utils.KYUUBI_CLIENT_VERSION_KEY, Utils.getVersion());
openReq.setConfiguration(openConf);
// Store the user name in the open request in case no non-sasl authentication
diff --git a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/Utils.java b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/Utils.java
index 59cc67f9d..1daa322ec 100644
--- a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/Utils.java
+++ b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/Utils.java
@@ -486,4 +486,21 @@ public class Utils {
public static boolean isKyuubiOperationHint(String hint) {
return KYUUBI_OPERATION_HINT_PATTERN.matcher(hint).matches();
}
+
+ public static final String KYUUBI_CLIENT_VERSION_KEY = "kyuubi.client.version";
+ private static String KYUUBI_CLIENT_VERSION;
+
+ public static synchronized String getVersion() {
+ if (KYUUBI_CLIENT_VERSION == null) {
+ try {
+ Properties prop = new Properties();
+ prop.load(Utils.class.getClassLoader().getResourceAsStream("version.properties"));
+ KYUUBI_CLIENT_VERSION = prop.getProperty(KYUUBI_CLIENT_VERSION_KEY, "unknown");
+ } catch (Exception e) {
+ LOG.error("Error getting kyuubi client version", e);
+ KYUUBI_CLIENT_VERSION = "unknown";
+ }
+ }
+ return KYUUBI_CLIENT_VERSION;
+ }
}
diff --git a/kyuubi-hive-jdbc/src/main/resources/version.properties b/kyuubi-hive-jdbc/src/main/resources/version.properties
new file mode 100644
index 000000000..82ae50cfb
--- /dev/null
+++ b/kyuubi-hive-jdbc/src/main/resources/version.properties
@@ -0,0 +1,18 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+kyuubi.client.version = ${project.version}
diff --git a/kyuubi-hive-jdbc/src/test/java/org/apache/kyuubi/jdbc/hive/UtilsTest.java b/kyuubi-hive-jdbc/src/test/java/org/apache/kyuubi/jdbc/hive/UtilsTest.java
index e0594dde0..b01957b3e 100644
--- a/kyuubi-hive-jdbc/src/test/java/org/apache/kyuubi/jdbc/hive/UtilsTest.java
+++ b/kyuubi-hive-jdbc/src/test/java/org/apache/kyuubi/jdbc/hive/UtilsTest.java
@@ -29,6 +29,7 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.Map;
import java.util.Properties;
+import java.util.regex.Pattern;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
@@ -138,4 +139,10 @@ public class UtilsTest {
assertEquals(expectedDb, jdbcConnectionParams1.getDbName());
assertEquals(expectedHiveConf, jdbcConnectionParams1.getHiveConfs());
}
+
+ @Test
+ public void testGetVersion() {
+ Pattern pattern = Pattern.compile("^\\d+\\.\\d+\\.\\d+.*");
+ assert pattern.matcher(Utils.getVersion()).matches();
+ }
}
diff --git a/kyuubi-rest-client/pom.xml b/kyuubi-rest-client/pom.xml
index 28b1670f1..993265aef 100644
--- a/kyuubi-rest-client/pom.xml
+++ b/kyuubi-rest-client/pom.xml
@@ -115,6 +115,13 @@
<build>
+ <resources>
+ <resource>
+ <filtering>true</filtering>
+ <directory>src/main/resources</directory>
+ </resource>
+ </resources>
+
<plugins>
<plugin>
<groupId>net.alchim31.maven</groupId>
diff --git a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/BatchRestApi.java b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/BatchRestApi.java
index a09ab562b..f5099568b 100644
--- a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/BatchRestApi.java
+++ b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/BatchRestApi.java
@@ -22,6 +22,7 @@ import java.util.HashMap;
import java.util.Map;
import org.apache.kyuubi.client.api.v1.dto.*;
import org.apache.kyuubi.client.util.JsonUtils;
+import org.apache.kyuubi.client.util.VersionUtils;
public class BatchRestApi {
@@ -36,11 +37,13 @@ public class BatchRestApi {
}
public Batch createBatch(BatchRequest request) {
+ setClientVersion(request);
String requestBody = JsonUtils.toJson(request);
return this.getClient().post(API_BASE_PATH, requestBody, Batch.class, client.getAuthHeader());
}
public Batch createBatch(BatchRequest request, File resourceFile) {
+ setClientVersion(request);
Map<String, MultiPart> multiPartMap = new HashMap<>();
multiPartMap.put("batchRequest", new MultiPart(MultiPart.MultiPartType.JSON, request));
multiPartMap.put("resourceFile", new MultiPart(MultiPart.MultiPartType.FILE, resourceFile));
@@ -96,4 +99,13 @@ public class BatchRestApi {
private IRestClient getClient() {
return this.client.getHttpClient();
}
+
+ private void setClientVersion(BatchRequest request) {
+ if (request != null) {
+ Map<String, String> newConf = new HashMap<>();
+ newConf.putAll(request.getConf());
+ newConf.put(VersionUtils.KYUUBI_CLIENT_VERSION_KEY, VersionUtils.getVersion());
+ request.setConf(newConf);
+ }
+ }
}
diff --git a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/util/VersionUtils.java b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/util/VersionUtils.java
new file mode 100644
index 000000000..bcabca5b9
--- /dev/null
+++ b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/util/VersionUtils.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kyuubi.client.util;
+
+import java.util.Properties;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class VersionUtils {
+ static final Logger LOG = LoggerFactory.getLogger(VersionUtils.class);
+
+ public static final String KYUUBI_CLIENT_VERSION_KEY = "kyuubi.client.version";
+ private static String KYUUBI_CLIENT_VERSION;
+
+ public static synchronized String getVersion() {
+ if (KYUUBI_CLIENT_VERSION == null) {
+ try {
+ Properties prop = new Properties();
+ prop.load(VersionUtils.class.getClassLoader().getResourceAsStream("version.properties"));
+ KYUUBI_CLIENT_VERSION = prop.getProperty(KYUUBI_CLIENT_VERSION_KEY, "unknown");
+ } catch (Exception e) {
+ LOG.error("Error getting kyuubi client version", e);
+ KYUUBI_CLIENT_VERSION = "unknown";
+ }
+ }
+ return KYUUBI_CLIENT_VERSION;
+ }
+}
diff --git a/kyuubi-rest-client/src/main/resources/version.properties b/kyuubi-rest-client/src/main/resources/version.properties
new file mode 100644
index 000000000..82ae50cfb
--- /dev/null
+++ b/kyuubi-rest-client/src/main/resources/version.properties
@@ -0,0 +1,18 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+kyuubi.client.version = ${project.version}
diff --git a/kyuubi-rest-client/src/test/java/org/apache/kyuubi/client/util/VersionUtilsTest.java b/kyuubi-rest-client/src/test/java/org/apache/kyuubi/client/util/VersionUtilsTest.java
new file mode 100644
index 000000000..d4675f340
--- /dev/null
+++ b/kyuubi-rest-client/src/test/java/org/apache/kyuubi/client/util/VersionUtilsTest.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kyuubi.client.util;
+
+import java.util.regex.Pattern;
+import org.junit.Test;
+
+public class VersionUtilsTest {
+
+ @Test
+ public void testGetClientVersion() {
+ Pattern pattern = Pattern.compile("^\\d+\\.\\d+\\.\\d+.*");
+ assert pattern.matcher(VersionUtils.getVersion()).matches();
+ }
+}
diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/KyuubiOperationPerConnectionSuite.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/KyuubiOperationPerConnectionSuite.scala
index 341f00639..669475b6c 100644
--- a/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/KyuubiOperationPerConnectionSuite.scala
+++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/KyuubiOperationPerConnectionSuite.scala
@@ -26,8 +26,8 @@ import scala.collection.JavaConverters._
import org.apache.hive.service.rpc.thrift._
import org.scalatest.time.SpanSugar.convertIntToGrainOfTime
-import org.apache.kyuubi.WithKyuubiServer
-import org.apache.kyuubi.config.KyuubiConf
+import org.apache.kyuubi.{KYUUBI_VERSION, WithKyuubiServer}
+import org.apache.kyuubi.config.{KyuubiConf, KyuubiReservedKeys}
import org.apache.kyuubi.config.KyuubiConf.SESSION_CONF_ADVISOR
import org.apache.kyuubi.engine.ApplicationState
import org.apache.kyuubi.jdbc.KyuubiHiveDriver
@@ -272,6 +272,14 @@ class KyuubiOperationPerConnectionSuite extends WithKyuubiServer with HiveJDBCTe
assert(MetricsSystem.counterValue(connFailedMetric).getOrElse(0L) > connFailedCount)
}
}
+
+ test("support to transfer client version when opening jdbc connection") {
+ withJdbcStatement() { stmt =>
+ val rs = stmt.executeQuery(s"set spark.${KyuubiReservedKeys.KYUUBI_CLIENT_VERSION_KEY}")
+ assert(rs.next())
+ assert(rs.getString(2) === KYUUBI_VERSION)
+ }
+ }
}
class TestSessionConfAdvisor extends SessionConfAdvisor {
diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/server/rest/client/BatchRestApiSuite.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/server/rest/client/BatchRestApiSuite.scala
index 6110ccb1e..cb7905286 100644
--- a/kyuubi-server/src/test/scala/org/apache/kyuubi/server/rest/client/BatchRestApiSuite.scala
+++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/server/rest/client/BatchRestApiSuite.scala
@@ -22,10 +22,11 @@ import java.util.Base64
import org.scalatest.time.SpanSugar.convertIntToGrainOfTime
-import org.apache.kyuubi.{BatchTestHelper, RestClientTestHelper}
+import org.apache.kyuubi.{BatchTestHelper, KYUUBI_VERSION, RestClientTestHelper}
import org.apache.kyuubi.client.{BatchRestApi, KyuubiRestClient}
import org.apache.kyuubi.client.api.v1.dto.Batch
import org.apache.kyuubi.client.exception.KyuubiRestException
+import org.apache.kyuubi.config.KyuubiReservedKeys
import org.apache.kyuubi.metrics.{MetricsConstants, MetricsSystem}
import org.apache.kyuubi.session.{KyuubiSession, SessionHandle}
@@ -215,4 +216,22 @@ class BatchRestApiSuite extends RestClientTestHelper with BatchTestHelper {
batchRestApi.listBatches(null, null, null, 0, 0, 0, 1)
batchRestApi.listBatches(null, null, null, 0, 0, 0, 1)
}
+
+ test("support to transfer client version when creating batch") {
+ val spnegoKyuubiRestClient: KyuubiRestClient =
+ KyuubiRestClient.builder(baseUri.toString)
+ .authHeaderMethod(KyuubiRestClient.AuthHeaderMethod.SPNEGO)
+ .spnegoHost("localhost")
+ .build()
+ val batchRestApi: BatchRestApi = new BatchRestApi(spnegoKyuubiRestClient)
+ // create batch
+ val requestObj =
+ newSparkBatchRequest(Map("spark.master" -> "local"))
+
+ val batch = batchRestApi.createBatch(requestObj)
+ val batchSession =
+ server.backendService.sessionManager.getSession(SessionHandle.fromUUID(batch.getId))
+ assert(
+ batchSession.conf.get(KyuubiReservedKeys.KYUUBI_CLIENT_VERSION_KEY) == Some(KYUUBI_VERSION))
+ }
}