You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kyuubi.apache.org by ch...@apache.org on 2022/08/11 07:32:08 UTC
[incubator-kyuubi] branch master updated: [KYUUBI #3206] Change Flink default version to 1.15
This is an automated email from the ASF dual-hosted git repository.
chengpan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new 86964fefb [KYUUBI #3206] Change Flink default version to 1.15
86964fefb is described below
commit 86964fefbe794c8553cfa0d567a3cf3e43ef4a96
Author: Luning Wang <wa...@gmail.com>
AuthorDate: Thu Aug 11 15:31:59 2022 +0800
[KYUUBI #3206] Change Flink default version to 1.15
### _Why are the changes needed?_
### _How was this patch tested?_
- [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible
- [ ] Add screenshots for manual tests if appropriate
- [ ] [Run test](https://kyuubi.apache.org/docs/latest/develop_tools/testing.html#running-tests) locally before make a pull request
Closes #3207 from deadwind4/flink-default-version.
Closes #3206
4c9b5d6c [Luning Wang] Fix test case to fit flink 1.14 and 1.15
09c854e2 [Luning Wang] change sql-client.jar name to fit flink 1.15
eef06d1e [Luning Wang] change the suffix to fit 1.15
93fb44f4 [Luning Wang] Comment out cross validation
8ea10218 [Luning Wang] Fix cross validation
a5e5cdb9 [Luning Wang] Fix github CI Flink version
1ee8a5d9 [Luning Wang] [KYUUBI #3206] Change Flink default version to 1.15
Authored-by: Luning Wang <wa...@gmail.com>
Signed-off-by: Cheng Pan <ch...@apache.org>
---
.github/workflows/master.yml | 11 +++++----
docs/quick_start/quick_start.md | 2 +-
.../engine/flink/FlinkProcessBuilderSuite.scala | 27 ++++++++++++----------
pom.xml | 4 ++--
4 files changed, 24 insertions(+), 20 deletions(-)
diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
index e8c982ca7..0c6ed989b 100644
--- a/.github/workflows/master.yml
+++ b/.github/workflows/master.yml
@@ -111,11 +111,12 @@ jobs:
- '1.15'
flink-archive: [ "" ]
comment: [ "normal" ]
- include:
- - java: 8
- flink: '1.15'
- flink-archive: '-Dflink.archive.mirror=https://archive.apache.org/dist/flink/flink-1.15.1 -Dflink.archive.name=flink-1.15.1-bin-scala_2.12.tgz'
- comment: 'verify-flink-1.15'
+ # FIXME: Cross Flink versions verification is not supported yet
+ # include:
+ # - java: 8
+ # flink: '1.15'
+ # flink-archive: '-Dflink.archive.mirror=https://archive.apache.org/dist/flink/flink-1.14.5 -Dflink.archive.name=flink-1.14.5-bin-scala_2.12.tgz'
+ # comment: 'verify-flink-1.14'
steps:
- uses: actions/checkout@v2
- name: Tune Runner VM
diff --git a/docs/quick_start/quick_start.md b/docs/quick_start/quick_start.md
index e0dd99b1b..2099ed352 100644
--- a/docs/quick_start/quick_start.md
+++ b/docs/quick_start/quick_start.md
@@ -154,7 +154,7 @@ Similar to `JAVA_HOME`, you can also set `FLINK_HOME` in different ways. However
For example,
```bash
-FLINK_HOME=/Downloads/flink-1.14.4
+FLINK_HOME=/Downloads/flink-1.15.1
```
#### Setup Kyuubi Flink Configration
diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/flink/FlinkProcessBuilderSuite.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/flink/FlinkProcessBuilderSuite.scala
index 211105dea..79f23c7c6 100644
--- a/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/flink/FlinkProcessBuilderSuite.scala
+++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/flink/FlinkProcessBuilderSuite.scala
@@ -22,8 +22,9 @@ import java.util
import scala.collection.JavaConverters._
import scala.collection.immutable.ListMap
+import scala.util.matching.Regex
-import org.apache.kyuubi.{FLINK_COMPILE_VERSION, KyuubiException, KyuubiFunSuite, SCALA_COMPILE_VERSION}
+import org.apache.kyuubi.{KyuubiException, KyuubiFunSuite}
import org.apache.kyuubi.config.KyuubiConf
import org.apache.kyuubi.config.KyuubiConf.{ENGINE_FLINK_EXTRA_CLASSPATH, ENGINE_FLINK_JAVA_OPTIONS, ENGINE_FLINK_MEMORY}
import org.apache.kyuubi.engine.flink.FlinkProcessBuilder._
@@ -45,16 +46,18 @@ class FlinkProcessBuilderSuite extends KyuubiFunSuite {
(FLINK_HADOOP_CLASSPATH_KEY -> s"${File.separator}hadoop")
private def confStr: String = {
conf.clone.set("yarn.tags", "KYUUBI").getAll
- .map { case (k, v) => s"\\\n\t--conf $k=$v" }
+ .map { case (k, v) => s"\\\\\\n\\t--conf $k=$v" }
.mkString(" ")
}
- private def compareActualAndExpected(builder: FlinkProcessBuilder): Unit = {
+ private def matchActualAndExpected(builder: FlinkProcessBuilder): Unit = {
val actualCommands = builder.toString
val classpathStr = constructClasspathStr(builder)
val expectedCommands =
s"$javaPath -Xmx512m -agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5005 " +
- s"-cp $classpathStr $mainClassStr \\\n\t--conf kyuubi.session.user=vinoyang $confStr"
- assert(actualCommands == expectedCommands)
+ s"-cp $classpathStr $mainClassStr \\\\\\n\\t--conf kyuubi.session.user=vinoyang $confStr"
+ val regex = new Regex(expectedCommands)
+ val matcher = regex.pattern.matcher(actualCommands)
+ assert(matcher.matches())
}
private def constructClasspathStr(builder: FlinkProcessBuilder) = {
@@ -62,8 +65,8 @@ class FlinkProcessBuilderSuite extends KyuubiFunSuite {
builder.mainResource.foreach(classpathEntries.add)
val flinkHome = builder.flinkHome
- classpathEntries.add(s"$flinkHome$flinkSqlClientJarPathSuffix")
- classpathEntries.add(s"$flinkHome$flinkLibPathSuffix")
+ classpathEntries.add(s"$flinkHome$flinkSqlClientJarPathSuffixRegex")
+ classpathEntries.add(s"$flinkHome$flinkLibPathSuffixRegex")
classpathEntries.add(s"$flinkHome$flinkConfPathSuffix")
val envMap = builder.env
envMap.foreach { case (k, v) =>
@@ -78,9 +81,9 @@ class FlinkProcessBuilderSuite extends KyuubiFunSuite {
}
private val javaPath = s"${envDefault("JAVA_HOME")}${File.separator}bin${File.separator}java"
- private val flinkSqlClientJarPathSuffix = s"${File.separator}opt${File.separator}" +
- s"flink-sql-client_$SCALA_COMPILE_VERSION-$FLINK_COMPILE_VERSION.jar"
- private val flinkLibPathSuffix = s"${File.separator}lib${File.separator}*"
+ private val flinkSqlClientJarPathSuffixRegex = s"${File.separator}opt${File.separator}" +
+ s"flink-sql-client-.*.jar"
+ private val flinkLibPathSuffixRegex = s"${File.separator}lib${File.separator}\\*"
private val flinkConfPathSuffix = s"${File.separator}conf"
private val mainClassStr = "org.apache.kyuubi.engine.flink.FlinkSQLEngine"
@@ -88,7 +91,7 @@ class FlinkProcessBuilderSuite extends KyuubiFunSuite {
val builder = new FlinkProcessBuilder("vinoyang", conf) {
override def env: Map[String, String] = envWithAllHadoop
}
- compareActualAndExpected(builder)
+ matchActualAndExpected(builder)
}
test("all hadoop related environment variables are configured except FLINK_HADOOP_CLASSPATH") {
@@ -102,6 +105,6 @@ class FlinkProcessBuilderSuite extends KyuubiFunSuite {
override def env: Map[String, String] = envDefault +
(FLINK_HADOOP_CLASSPATH_KEY -> s"${File.separator}hadoop")
}
- compareActualAndExpected(builder)
+ matchActualAndExpected(builder)
}
}
diff --git a/pom.xml b/pom.xml
index a90d36c49..0f8639a7d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -115,8 +115,8 @@
<delta.version>2.0.0</delta.version>
<failsafe.verion>2.4.4</failsafe.verion>
<fb303.version>0.9.3</fb303.version>
- <flink.version>1.14.5</flink.version>
- <flink.module.scala.suffix>_${scala.binary.version}</flink.module.scala.suffix>
+ <flink.version>1.15.1</flink.version>
+ <flink.module.scala.suffix></flink.module.scala.suffix>
<flink.archive.name>flink-${flink.version}-bin-scala_${scala.binary.version}.tgz</flink.archive.name>
<flink.archive.mirror>${apache.archive.dist}/flink/flink-${flink.version}</flink.archive.mirror>
<flink.archive.download.skip>false</flink.archive.download.skip>