You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kyuubi.apache.org by fe...@apache.org on 2021/12/01 12:33:56 UTC
[incubator-kyuubi] branch master updated: [KYUUBI #1382] Sync the flink-provided profile in docs and testing suite comments
This is an automated email from the ASF dual-hosted git repository.
feiwang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new a6d2323 [KYUUBI #1382] Sync the flink-provided profile in docs and testing suite comments
a6d2323 is described below
commit a6d23238cbfe5da47b234be2fcf59d8ce20c1306
Author: fwang12 <fw...@ebay.com>
AuthorDate: Wed Dec 1 20:33:47 2021 +0800
[KYUUBI #1382] Sync the flink-provided profile in docs and testing suite comments
<!--
Thanks for sending a pull request!
Here are some tips for you:
1. If this is your first time, please read our contributor guidelines: https://kyuubi.readthedocs.io/en/latest/community/contributions.html
2. If the PR is related to an issue in https://github.com/apache/incubator-kyuubi/issues, add '[KYUUBI #XXXX]' in your PR title, e.g., '[KYUUBI #XXXX] Your PR title ...'.
3. If the PR is unfinished, add '[WIP]' in your PR title, e.g., '[WIP][KYUUBI #XXXX] Your PR title ...'.
-->
### _Why are the changes needed?_
<!--
Please clarify why the changes are needed. For instance,
1. If you add a feature, you can talk about the use case of it.
2. If you fix a bug, you can clarify why it is a bug.
-->
Sync the flink-provided profile in docs and testing suite comments.
### _How was this patch tested?_
- [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible
- [ ] Add screenshots for manual tests if appropriate
- [ ] [Run test](https://kyuubi.readthedocs.io/en/latest/develop_tools/testing.html#running-tests) locally before make a pull request
Closes #1382 from turboFei/flink_provided.
Closes #1382
3ae96963 [Fei Wang] address nit
ef5bba7f [fwang12] reorder
e9c9c527 [fwang12] save
Lead-authored-by: fwang12 <fw...@ebay.com>
Co-authored-by: Fei Wang <fw...@ebay.com>
Signed-off-by: Fei Wang <fw...@ebay.com>
---
.github/workflows/master.yml | 2 +-
.github/workflows/publish-snapshot.yml | 8 ++++----
build/dist | 4 +++-
build/release/release.sh | 6 +++---
docs/develop_tools/distribution.md | 7 ++++---
.../kyuubi/engine/spark/udf/KyuubiDefinedFunctionSuite.scala | 4 ++--
kyuubi-common/src/test/scala/org/apache/kyuubi/TestUtils.scala | 3 ++-
.../scala/org/apache/kyuubi/config/AllKyuubiConfiguration.scala | 4 ++--
8 files changed, 21 insertions(+), 17 deletions(-)
diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
index 0b271ba..faca022 100644
--- a/.github/workflows/master.yml
+++ b/.github/workflows/master.yml
@@ -54,7 +54,7 @@ jobs:
- java: 11
spark: '3.1'
spark-hadoop: '3.2'
- profiles: '-DskipTests -Pspark-provided'
+ profiles: '-DskipTests -Pflink-provided,spark-provided'
env:
SPARK_LOCAL_IP: localhost
diff --git a/.github/workflows/publish-snapshot.yml b/.github/workflows/publish-snapshot.yml
index 78c15e0..01c8191 100644
--- a/.github/workflows/publish-snapshot.yml
+++ b/.github/workflows/publish-snapshot.yml
@@ -37,9 +37,9 @@ jobs:
ASF_PASSWORD: ${{ secrets.NEXUS_PW }}
run: |
rm -rf ~/.m2/repository/org/apache/kyuubi
- ./build/mvn clean deploy -s ./build/release/asf-settings.xml -DskipTests -Pspark-provided
- ./build/mvn clean deploy -s ./build/release/asf-settings.xml -DskipTests -Pspark-provided,spark-3.1 -pl dev/kyuubi-extension-spark-3-1 -am
- ./build/mvn clean deploy -s ./build/release/asf-settings.xml -DskipTests -Pspark-provided,spark-3.2 -pl dev/kyuubi-extension-spark-3-2 -am
+ ./build/mvn clean deploy -s ./build/release/asf-settings.xml -DskipTests -Pflink-provided,spark-provided
+ ./build/mvn clean deploy -s ./build/release/asf-settings.xml -DskipTests -Pflink-provided,spark-provided,spark-3.1 -pl dev/kyuubi-extension-spark-3-1 -am
+ ./build/mvn clean deploy -s ./build/release/asf-settings.xml -DskipTests -Pflink-provided,spark-provided,spark-3.2 -pl dev/kyuubi-extension-spark-3-2 -am
- name: Publish snapshot - branch-1.3
if: ${{ matrix.branch == 'branch-1.3' }}
env:
@@ -47,4 +47,4 @@ jobs:
ASF_PASSWORD: ${{ secrets.NEXUS_PW }}
run: |
rm -rf ~/.m2/repository/org/apache/kyuubi
- ./build/mvn clean deploy -s ./build/release/asf-settings.xml -DskipTests -Pspark-provided,kyuubi-extension-spark-3-1
+ ./build/mvn clean deploy -s ./build/release/asf-settings.xml -DskipTests -Pflink-provided,spark-provided,kyuubi-extension-spark-3-1
diff --git a/build/dist b/build/dist
index 6e8a7dd..a4ef982 100755
--- a/build/dist
+++ b/build/dist
@@ -30,6 +30,7 @@ set -e
KYUUBI_HOME="$(cd "`dirname "$0"`/.."; pwd)"
DISTDIR="$KYUUBI_HOME/dist"
MAKE_TGZ=false
+# TODO: add FLINK_PROVIDED option
SPARK_PROVIDED=false
NAME=none
MVN="$KYUUBI_HOME/build/mvn"
@@ -40,10 +41,11 @@ function usage {
echo ""
echo "Usage:"
echo "+--------------------------------------------------------------------------------------+"
- echo "| ./build/dist [--name <custom_name>] [--tgz] [--spark-provided] <maven build options> |"
+ echo "| ./build/dist [--name <custom_name>] [--tgz] [--flink-provided] [--spark-provided] <maven build options> |"
echo "+--------------------------------------------------------------------------------------+"
echo "name: - custom binary name, using project version if undefined"
echo "tgz: - whether to make a whole bundled package"
+ echo "flink-provided: - whether to make a package without Flink binary"
echo "spark-provided: - whether to make a package without Spark binary"
echo ""
}
diff --git a/build/release/release.sh b/build/release/release.sh
index 00d7fa8..02a5d79 100755
--- a/build/release/release.sh
+++ b/build/release/release.sh
@@ -94,12 +94,12 @@ upload_svn_staging() {
}
upload_nexus_staging() {
- ${KYUUBI_DIR}/build/mvn clean deploy -DskipTests -Papache-release,spark-provided \
+ ${KYUUBI_DIR}/build/mvn clean deploy -DskipTests -Papache-release,flink-provided,spark-provided \
-s "${KYUUBI_DIR}/build/release/asf-settings.xml"
- ${KYUUBI_DIR}/build/mvn clean deploy -DskipTests -Papache-release,spark-provided,spark-3.1 \
+ ${KYUUBI_DIR}/build/mvn clean deploy -DskipTests -Papache-release,flink-provided,spark-provided,spark-3.1 \
-s "${KYUUBI_DIR}/build/release/asf-settings.xml" \
-pl dev/kyuubi-extension-spark-3-1 -am
- ${KYUUBI_DIR}/build/mvn clean deploy -DskipTests -Papache-release,spark-provided,spark-3.2 \
+ ${KYUUBI_DIR}/build/mvn clean deploy -DskipTests -Papache-release,flink-provided,spark-provided,spark-3.2 \
-s "${KYUUBI_DIR}/build/release/asf-settings.xml" \
-pl dev/kyuubi-extension-spark-3-2 -am
}
diff --git a/docs/develop_tools/distribution.md b/docs/develop_tools/distribution.md
index 1bf8fee..b6153bd 100644
--- a/docs/develop_tools/distribution.md
+++ b/docs/develop_tools/distribution.md
@@ -33,10 +33,11 @@ For more information on usage, run `./build/dist --help`
Usage:
+--------------------------------------------------------------------------------------+
-| ./build/dist [--name <custom_name>] [--tgz] [--spark-provided] <maven build options> |
+| ./build/dist [--name <custom_name>] [--tgz] [--flink-provided] [--spark-provided] <maven build options> |
+--------------------------------------------------------------------------------------+
name: - custom binary name, using project version if undefined
tgz: - whether to make a whole bundled package
+flink-provided: - whether to make a package without Flink binary
spark-provided: - whether to make a package without Spark binary
```
@@ -48,10 +49,10 @@ For instance,
This results in a Kyuubi distribution named `kyuubi-{version}-bin-custom-name.tgz` for you.
-If you are planing to deploy Kyuubi where `spark` is provided, in other word, it's not required to bundle spark binary, use
+If you are planing to deploy Kyuubi where `spark`/`flink` is provided, in other word, it's not required to bundle spark/flink binary, use
```bash
-./build/dist --tgz --spark-provided
+./build/dist --tgz --spark-provided --flink-provided
```
Then you will get a Kyuubi distribution without spark binary named `kyuubi-{version}-bin.tgz`.
diff --git a/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/udf/KyuubiDefinedFunctionSuite.scala b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/udf/KyuubiDefinedFunctionSuite.scala
index 41a5648..6cc8456 100644
--- a/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/udf/KyuubiDefinedFunctionSuite.scala
+++ b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/udf/KyuubiDefinedFunctionSuite.scala
@@ -30,12 +30,12 @@ import org.apache.kyuubi.{KyuubiFunSuite, TestUtils}
*
* To run the entire test suite:
* {{{
- * build/mvn clean install -Pspark-provided -DwildcardSuites=org.apache.kyuubi.engine.spark.udf.KyuubiDefinedFunctionSuite
+ * build/mvn clean install -Pflink-provided,spark-provided -DwildcardSuites=org.apache.kyuubi.engine.spark.udf.KyuubiDefinedFunctionSuite
* }}}
*
* To re-generate golden files for entire suite, run:
* {{{
- * KYUUBI_UPDATE=1 build/mvn clean install -Pspark-provided -DwildcardSuites=org.apache.kyuubi.engine.spark.udf.KyuubiDefinedFunctionSuite
+ * KYUUBI_UPDATE=1 build/mvn clean install -Pflink-provided,spark-provided -DwildcardSuites=org.apache.kyuubi.engine.spark.udf.KyuubiDefinedFunctionSuite
* }}}
*/
// scalastyle:on line.size.limit
diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/TestUtils.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/TestUtils.scala
index 8645fc9..107cf87 100644
--- a/kyuubi-common/src/test/scala/org/apache/kyuubi/TestUtils.scala
+++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/TestUtils.scala
@@ -52,7 +52,8 @@ object TestUtils {
}
reader.close()
val hint = s"$markdown out of date, please update doc with " +
- s"KYUUBI_UPDATE=1 build/mvn clean install -Pspark-provided -DwildcardSuites=$agent"
+ s"KYUUBI_UPDATE=1 build/mvn clean install -Pflink-provided,spark-provided " +
+ s"-DwildcardSuites=$agent"
assert(newOutput.size === expected.size, hint)
newOutput.zip(expected).foreach { case (out, in) => assert(out === in, hint) }
diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/config/AllKyuubiConfiguration.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/config/AllKyuubiConfiguration.scala
index 2151869..1f8359e 100644
--- a/kyuubi-server/src/test/scala/org/apache/kyuubi/config/AllKyuubiConfiguration.scala
+++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/config/AllKyuubiConfiguration.scala
@@ -35,12 +35,12 @@ import org.apache.kyuubi.zookeeper.ZookeeperConf
*
* To run the entire test suite:
* {{{
- * build/mvn clean install -Pspark-provided -DwildcardSuites=org.apache.kyuubi.config.AllKyuubiConfiguration
+ * build/mvn clean install -Pflink-provided,spark-provided -DwildcardSuites=org.apache.kyuubi.config.AllKyuubiConfiguration
* }}}
*
* To re-generate golden files for entire suite, run:
* {{{
- * KYUUBI_UPDATE=1 build/mvn clean install -Pspark-provided -DwildcardSuites=org.apache.kyuubi.config.AllKyuubiConfiguration
+ * KYUUBI_UPDATE=1 build/mvn clean install -Pflink-provided,spark-provided -DwildcardSuites=org.apache.kyuubi.config.AllKyuubiConfiguration
* }}}
*/
// scalastyle:on line.size.limit