[incubator-kyuubi] branch master updated: [KYUUBI #3794][FOLLOWUP] Expose `kyuubi.operation.result.codec` to KyuubiConf

[ch...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

chengpan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-kyuubi.git


The following commit(s) were added to refs/heads/master by this push:
     new 081e03de4 [KYUUBI #3794][FOLLOWUP] Expose `kyuubi.operation.result.codec` to KyuubiConf
081e03de4 is described below

commit 081e03de4ff3a364eb1caa37f1c162fb71d225b5
Author: Fu Chen <cf...@gmail.com>
AuthorDate: Mon Nov 28 18:48:50 2022 +0800

    [KYUUBI #3794][FOLLOWUP] Expose `kyuubi.operation.result.codec` to KyuubiConf
    
    ### _Why are the changes needed?_
    
    Expose `kyuubi.operation.result.codec` to KyuubiConf
    
    ### _How was this patch tested?_
    - [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible
    
    - [ ] Add screenshots for manual tests if appropriate
    
    - [ ] [Run test](https://kyuubi.apache.org/docs/latest/develop_tools/testing.html#running-tests) locally before make a pull request
    
    Closes #3866 from cfmcgrady/arrow.
    
    Closes #3794
    
    2047f364 [Fu Chen] update
    6f3fe24d [Fu Chen] rename
    20748ca7 [Fu Chen] expose `kyuubi.operation.result.codec` to KyuubiConf
    
    Authored-by: Fu Chen <cf...@gmail.com>
    Signed-off-by: Cheng Pan <ch...@apache.org>
---
 docs/deployment/settings.md                                 |  1 +
 .../spark/operation/SparkArrowbasedOperationSuite.scala     | 10 +++++++---
 .../main/scala/org/apache/kyuubi/config/KyuubiConf.scala    | 13 +++++++++++++
 3 files changed, 21 insertions(+), 3 deletions(-)

diff --git a/docs/deployment/settings.md b/docs/deployment/settings.md
index c9638af83..e8d9b17a6 100644
--- a/docs/deployment/settings.md
+++ b/docs/deployment/settings.md
@@ -437,6 +437,7 @@ kyuubi.operation.plan.only.mode|none|Configures the statement performed mode, Th
 kyuubi.operation.plan.only.output.style|plain|Configures the planOnly output style, The value can be 'plain' and 'json', default value is 'plain', this configuration supports only the output styles of the Spark engine|string|1.7.0
 kyuubi.operation.progress.enabled|false|Whether to enable the operation progress. When true, the operation progress will be returned in `GetOperationStatus`.|boolean|1.6.0
 kyuubi.operation.query.timeout|&lt;undefined&gt;|Timeout for query executions at server-side, take affect with client-side timeout(`java.sql.Statement.setQueryTimeout`) together, a running query will be cancelled automatically if timeout. It's off by default, which means only client-side take fully control whether the query should timeout or not. If set, client-side timeout capped at this point. To cancel the queries right away without waiting task to finish, consider enabling kyuubi.ope [...]
+kyuubi.operation.result.codec|simple|Specify the result codec, available configs are: <ul> <li>SIMPLE: the result will convert to TRow at the engine driver side. </li> <li>ARROW: the result will be encoded as Arrow at the executor side before collecting by the driver, and deserialized at the client side. note that it only takes effect for kyuubi-hive-jdbc clients now.</li></ul>|string|1.7.0
 kyuubi.operation.result.max.rows|0|Max rows of Spark query results. Rows that exceeds the limit would be ignored. By setting this value to 0 to disable the max rows limit.|int|1.6.0
 kyuubi.operation.scheduler.pool|&lt;undefined&gt;|The scheduler pool of job. Note that, this config should be used after change Spark config spark.scheduler.mode=FAIR.|string|1.1.1
 kyuubi.operation.spark.listener.enabled|true|When set to true, Spark engine registers a SQLOperationListener before executing the statement, logs a few summary statistics when each stage completes.|boolean|1.6.0
diff --git a/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/operation/SparkArrowbasedOperationSuite.scala b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/operation/SparkArrowbasedOperationSuite.scala
index 7145add65..b976f3aac 100644
--- a/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/operation/SparkArrowbasedOperationSuite.scala
+++ b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/operation/SparkArrowbasedOperationSuite.scala
@@ -17,6 +17,7 @@
 
 package org.apache.kyuubi.engine.spark.operation
 
+import org.apache.kyuubi.config.KyuubiConf
 import org.apache.kyuubi.engine.spark.WithSparkSQLEngine
 import org.apache.kyuubi.operation.SparkDataTypeTests
 
@@ -27,15 +28,18 @@ class SparkArrowbasedOperationSuite extends WithSparkSQLEngine with SparkDataTyp
   override def withKyuubiConf: Map[String, String] = Map.empty
 
   override def jdbcVars: Map[String, String] = {
-    Map("kyuubi.operation.result.codec" -> resultCodec)
+    Map(KyuubiConf.OPERATION_RESULT_CODEC.key -> resultCodec)
   }
 
   override def resultCodec: String = "arrow"
 
   test("make sure kyuubi.operation.result.codec=arrow") {
     withJdbcStatement() { statement =>
-      val resultSet =
-        statement.executeQuery("SELECT '${hivevar:kyuubi.operation.result.codec}' AS col")
+      val query =
+        s"""
+           |SELECT '$${hivevar:${KyuubiConf.OPERATION_RESULT_CODEC.key}}' AS col
+           |""".stripMargin
+      val resultSet = statement.executeQuery(query)
       assert(resultSet.next())
       assert(resultSet.getString("col") === "arrow")
     }
diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
index b61a32747..4b6e1d510 100644
--- a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
+++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
@@ -1444,6 +1444,19 @@ object KyuubiConf {
       .booleanConf
       .createWithDefault(false)
 
+  val OPERATION_RESULT_CODEC: ConfigEntry[String] =
+    buildConf("kyuubi.operation.result.codec")
+      .doc("Specify the result codec, available configs are: <ul>" +
+        " <li>SIMPLE: the result will convert to TRow at the engine driver side. </li>" +
+        " <li>ARROW: the result will be encoded as Arrow at the executor side before collecting" +
+        " by the driver, and deserialized at the client side. note that it only takes effect for" +
+        " kyuubi-hive-jdbc clients now.</li></ul>")
+      .version("1.7.0")
+      .stringConf
+      .checkValues(Set("arrow", "simple"))
+      .transform(_.toLowerCase(Locale.ROOT))
+      .createWithDefault("simple")
+
   val OPERATION_RESULT_MAX_ROWS: ConfigEntry[Int] =
     buildConf("kyuubi.operation.result.max.rows")
       .doc("Max rows of Spark query results. Rows that exceeds the limit would be ignored. " +