You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by bl...@apache.org on 2021/10/31 19:26:45 UTC
[iceberg] branch master updated: Build: Build Flink for 1.12 and
1.13 (#3364)
This is an automated email from the ASF dual-hosted git repository.
blue pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/master by this push:
new 7f520b0 Build: Build Flink for 1.12 and 1.13 (#3364)
7f520b0 is described below
commit 7f520b0c54fbe20a8fd746697bbcba555621c242
Author: openinx <op...@gmail.com>
AuthorDate: Mon Nov 1 03:26:35 2021 +0800
Build: Build Flink for 1.12 and 1.13 (#3364)
This changes the iceberg-flink-runtime module to iceberg-flink-1.12-runtime and iceberg-flink-1.13-runtime.
---
.github/workflows/flink-ci.yml | 30 +++++-
flink-runtime/build.gradle | 77 ---------------
flink/build.gradle | 11 ++-
flink/{ => v1.12}/build.gradle | 105 ++++++++++++++++++---
.../v1.12/flink-runtime}/LICENSE | 0
.../v1.12/flink-runtime}/NOTICE | 0
flink/{ => v1.13}/build.gradle | 104 +++++++++++++++++---
.../v1.13/flink-runtime}/LICENSE | 0
.../v1.13/flink-runtime}/NOTICE | 0
gradle.properties | 4 +-
settings.gradle | 22 ++++-
11 files changed, 246 insertions(+), 107 deletions(-)
diff --git a/.github/workflows/flink-ci.yml b/.github/workflows/flink-ci.yml
index cd8a16c..35043c9 100644
--- a/.github/workflows/flink-ci.yml
+++ b/.github/workflows/flink-ci.yml
@@ -49,12 +49,38 @@ on:
- 'site/**'
jobs:
+ flink-common-tests:
+ runs-on: ubuntu-latest
+ strategy:
+ matrix:
+ jvm: [8, 11]
+ env:
+ SPARK_LOCAL_IP: localhost
+ steps:
+ - uses: actions/checkout@v2
+ - uses: actions/setup-java@v1
+ with:
+ java-version: ${{ matrix.jvm }}
+ - uses: actions/cache@v2
+ with:
+ path: ~/.gradle/caches
+ key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }}
+ restore-keys: ${{ runner.os }}-gradle
+ - run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
+ - run: ./gradlew -DsparkVersions= -DhiveVersions= :iceberg-flink:check -Pquick=true -x javadoc
+ - uses: actions/upload-artifact@v2
+ if: failure()
+ with:
+ name: test logs
+ path: |
+ **/build/testlogs
+
flink-tests:
runs-on: ubuntu-latest
strategy:
matrix:
jvm: [8, 11]
- flink: ['1.13']
+ flink: ['1.12', '1.13']
env:
SPARK_LOCAL_IP: localhost
steps:
@@ -68,7 +94,7 @@ jobs:
key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }}
restore-keys: ${{ runner.os }}-gradle
- run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
- - run: ./gradlew -DsparkVersions= -DhiveVersions= -DflinkVersions=${{ matrix.flink }} :iceberg-flink:check :iceberg-flink-runtime:check -Pquick=true -x javadoc
+ - run: ./gradlew -DsparkVersions= -DhiveVersions= -DflinkVersions=${{ matrix.flink }} :iceberg-flink:iceberg-flink-${{ matrix.flink }}:check :iceberg-flink:iceberg-flink-${{ matrix.flink }}-runtime:check -Pquick=true -x javadoc
- uses: actions/upload-artifact@v2
if: failure()
with:
diff --git a/flink-runtime/build.gradle b/flink-runtime/build.gradle
deleted file mode 100644
index 92c4350..0000000
--- a/flink-runtime/build.gradle
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-project(':iceberg-flink-runtime') {
- apply plugin: 'com.github.johnrengelman.shadow'
-
- tasks.jar.dependsOn tasks.shadowJar
-
- configurations {
- implementation {
- exclude group: 'org.apache.flink'
- // included in Flink
- exclude group: 'org.slf4j'
- exclude group: 'org.apache.commons'
- exclude group: 'commons-pool'
- exclude group: 'commons-codec'
- exclude group: 'org.xerial.snappy'
- exclude group: 'javax.xml.bind'
- exclude group: 'javax.annotation'
- }
- }
-
- dependencies {
- implementation project(':iceberg-flink')
- implementation project(':iceberg-aws')
- implementation(project(':iceberg-nessie')) {
- exclude group: 'com.google.code.findbugs', module: 'jsr305'
- }
- }
-
- shadowJar {
- configurations = [project.configurations.runtimeClasspath]
-
- zip64 true
-
- // include the LICENSE and NOTICE files for the shaded Jar
- from(projectDir) {
- include 'LICENSE'
- include 'NOTICE'
- }
-
- // Relocate dependencies to avoid conflicts
- relocate 'org.apache.avro', 'org.apache.iceberg.shaded.org.apache.avro'
- relocate 'org.apache.parquet', 'org.apache.iceberg.shaded.org.apache.parquet'
- relocate 'com.google', 'org.apache.iceberg.shaded.com.google'
- relocate 'com.fasterxml', 'org.apache.iceberg.shaded.com.fasterxml'
- relocate 'com.github.benmanes', 'org.apache.iceberg.shaded.com.github.benmanes'
- relocate 'org.checkerframework', 'org.apache.iceberg.shaded.org.checkerframework'
- relocate 'shaded.parquet', 'org.apache.iceberg.shaded.org.apache.parquet.shaded'
- relocate 'org.apache.orc', 'org.apache.iceberg.shaded.org.apache.orc'
- relocate 'io.airlift', 'org.apache.iceberg.shaded.io.airlift'
- relocate 'org.threeten.extra', 'org.apache.iceberg.shaded.org.threeten.extra'
-
- classifier null
- }
-
- jar {
- enabled = false
- }
-}
-
diff --git a/flink/build.gradle b/flink/build.gradle
index 994861a..bbf6517 100644
--- a/flink/build.gradle
+++ b/flink/build.gradle
@@ -59,7 +59,7 @@ project(':iceberg-flink') {
testImplementation "org.apache.flink:flink-core"
testImplementation "org.apache.flink:flink-runtime_2.12"
testImplementation "org.apache.flink:flink-table-planner-blink_2.12"
- testImplementation ("org.apache.flink:flink-test-utils-junit") {
+ testImplementation("org.apache.flink:flink-test-utils-junit") {
exclude group: 'junit'
}
testImplementation("org.apache.flink:flink-test-utils_2.12") {
@@ -106,3 +106,12 @@ project(':iceberg-flink') {
}
}
+def flinkVersions = (System.getProperty("flinkVersions") != null ? System.getProperty("flinkVersions") : System.getProperty("defaultFlinkVersions")).split(",")
+
+if (flinkVersions.contains("1.12")) {
+ apply from: file("$projectDir/v1.12/build.gradle")
+}
+
+if (flinkVersions.contains("1.13")) {
+ apply from: file("$projectDir/v1.13/build.gradle")
+}
\ No newline at end of file
diff --git a/flink/build.gradle b/flink/v1.12/build.gradle
similarity index 59%
copy from flink/build.gradle
copy to flink/v1.12/build.gradle
index 994861a..46cefa8 100644
--- a/flink/build.gradle
+++ b/flink/v1.12/build.gradle
@@ -17,7 +17,32 @@
* under the License.
*/
-project(':iceberg-flink') {
+def flinkProjects = [
+ project(':iceberg-flink:iceberg-flink-1.12'),
+ project(':iceberg-flink:iceberg-flink-1.12-runtime')
+]
+
+configure(flinkProjects) {
+ project.ext {
+ flinkVersion = '1.12.5'
+ }
+}
+
+project(':iceberg-flink:iceberg-flink-1.12') {
+
+ sourceSets {
+ main {
+ java.srcDirs = [
+ "${project(':iceberg-flink').projectDir}/src/main/java",
+ "src/main/java"
+ ]
+ resources.srcDirs = [
+ "${project(':iceberg-flink').projectDir}/src/main/resources",
+ "src/main/resources"
+ ]
+ }
+ }
+
dependencies {
implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
api project(':iceberg-api')
@@ -28,11 +53,11 @@ project(':iceberg-flink') {
implementation project(':iceberg-parquet')
implementation project(':iceberg-hive-metastore')
- compileOnly "org.apache.flink:flink-streaming-java_2.12"
- compileOnly "org.apache.flink:flink-streaming-java_2.12::tests"
- compileOnly "org.apache.flink:flink-table-api-java-bridge_2.12"
- compileOnly "org.apache.flink:flink-table-planner-blink_2.12"
- compileOnly "org.apache.flink:flink-table-planner_2.12"
+ compileOnly "org.apache.flink:flink-streaming-java_2.12:${flinkVersion}"
+ compileOnly "org.apache.flink:flink-streaming-java_2.12:${flinkVersion}:tests"
+ compileOnly "org.apache.flink:flink-table-api-java-bridge_2.12:${flinkVersion}"
+ compileOnly "org.apache.flink:flink-table-planner-blink_2.12:${flinkVersion}"
+ compileOnly "org.apache.flink:flink-table-planner_2.12:${flinkVersion}"
compileOnly "org.apache.hadoop:hadoop-hdfs"
compileOnly "org.apache.hadoop:hadoop-common"
compileOnly("org.apache.hadoop:hadoop-minicluster") {
@@ -56,13 +81,13 @@ project(':iceberg-flink') {
exclude group: 'org.apache.hive', module: 'hive-storage-api'
}
- testImplementation "org.apache.flink:flink-core"
- testImplementation "org.apache.flink:flink-runtime_2.12"
- testImplementation "org.apache.flink:flink-table-planner-blink_2.12"
- testImplementation ("org.apache.flink:flink-test-utils-junit") {
+ testImplementation "org.apache.flink:flink-core:${flinkVersion}"
+ testImplementation "org.apache.flink:flink-runtime_2.12:${flinkVersion}"
+ testImplementation "org.apache.flink:flink-table-planner-blink_2.12:${flinkVersion}"
+ testImplementation("org.apache.flink:flink-test-utils-junit:${flinkVersion}") {
exclude group: 'junit'
}
- testImplementation("org.apache.flink:flink-test-utils_2.12") {
+ testImplementation("org.apache.flink:flink-test-utils_2.12:${flinkVersion}") {
exclude group: "org.apache.curator", module: 'curator-test'
exclude group: 'junit'
}
@@ -106,3 +131,61 @@ project(':iceberg-flink') {
}
}
+project(':iceberg-flink:iceberg-flink-1.12-runtime') {
+ apply plugin: 'com.github.johnrengelman.shadow'
+
+ tasks.jar.dependsOn tasks.shadowJar
+
+ configurations {
+ implementation {
+ exclude group: 'org.apache.flink'
+ // included in Flink
+ exclude group: 'org.slf4j'
+ exclude group: 'org.apache.commons'
+ exclude group: 'commons-pool'
+ exclude group: 'commons-codec'
+ exclude group: 'org.xerial.snappy'
+ exclude group: 'javax.xml.bind'
+ exclude group: 'javax.annotation'
+ }
+ }
+
+ dependencies {
+ implementation project(':iceberg-flink:iceberg-flink-1.12')
+ implementation project(':iceberg-aws')
+ implementation(project(':iceberg-nessie')) {
+ exclude group: 'com.google.code.findbugs', module: 'jsr305'
+ }
+ }
+
+ shadowJar {
+ configurations = [project.configurations.runtimeClasspath]
+
+ zip64 true
+
+ // include the LICENSE and NOTICE files for the shaded Jar
+ from(projectDir) {
+ include 'LICENSE'
+ include 'NOTICE'
+ }
+
+ // Relocate dependencies to avoid conflicts
+ relocate 'org.apache.avro', 'org.apache.iceberg.shaded.org.apache.avro'
+ relocate 'org.apache.parquet', 'org.apache.iceberg.shaded.org.apache.parquet'
+ relocate 'com.google', 'org.apache.iceberg.shaded.com.google'
+ relocate 'com.fasterxml', 'org.apache.iceberg.shaded.com.fasterxml'
+ relocate 'com.github.benmanes', 'org.apache.iceberg.shaded.com.github.benmanes'
+ relocate 'org.checkerframework', 'org.apache.iceberg.shaded.org.checkerframework'
+ relocate 'shaded.parquet', 'org.apache.iceberg.shaded.org.apache.parquet.shaded'
+ relocate 'org.apache.orc', 'org.apache.iceberg.shaded.org.apache.orc'
+ relocate 'io.airlift', 'org.apache.iceberg.shaded.io.airlift'
+ relocate 'org.threeten.extra', 'org.apache.iceberg.shaded.org.threeten.extra'
+
+ classifier null
+ }
+
+ jar {
+ enabled = false
+ }
+}
+
diff --git a/flink-runtime/LICENSE b/flink/v1.12/flink-runtime/LICENSE
similarity index 100%
copy from flink-runtime/LICENSE
copy to flink/v1.12/flink-runtime/LICENSE
diff --git a/flink-runtime/NOTICE b/flink/v1.12/flink-runtime/NOTICE
similarity index 100%
copy from flink-runtime/NOTICE
copy to flink/v1.12/flink-runtime/NOTICE
diff --git a/flink/build.gradle b/flink/v1.13/build.gradle
similarity index 60%
copy from flink/build.gradle
copy to flink/v1.13/build.gradle
index 994861a..b2c24d4 100644
--- a/flink/build.gradle
+++ b/flink/v1.13/build.gradle
@@ -17,7 +17,32 @@
* under the License.
*/
-project(':iceberg-flink') {
+def flinkProjects = [
+ project(':iceberg-flink:iceberg-flink-1.13'),
+ project(':iceberg-flink:iceberg-flink-1.13-runtime')
+]
+
+configure(flinkProjects) {
+ project.ext {
+ flinkVersion = '1.13.2'
+ }
+}
+
+project(':iceberg-flink:iceberg-flink-1.13') {
+
+ sourceSets {
+ main {
+ java.srcDirs = [
+ "${project(':iceberg-flink').projectDir}/src/main/java",
+ "src/main/java"
+ ]
+ resources.srcDirs = [
+ "${project(':iceberg-flink').projectDir}/src/main/resources",
+ "src/main/resources"
+ ]
+ }
+ }
+
dependencies {
implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
api project(':iceberg-api')
@@ -28,11 +53,11 @@ project(':iceberg-flink') {
implementation project(':iceberg-parquet')
implementation project(':iceberg-hive-metastore')
- compileOnly "org.apache.flink:flink-streaming-java_2.12"
- compileOnly "org.apache.flink:flink-streaming-java_2.12::tests"
- compileOnly "org.apache.flink:flink-table-api-java-bridge_2.12"
- compileOnly "org.apache.flink:flink-table-planner-blink_2.12"
- compileOnly "org.apache.flink:flink-table-planner_2.12"
+ compileOnly "org.apache.flink:flink-streaming-java_2.12:${flinkVersion}"
+ compileOnly "org.apache.flink:flink-streaming-java_2.12:${flinkVersion}:tests"
+ compileOnly "org.apache.flink:flink-table-api-java-bridge_2.12:${flinkVersion}"
+ compileOnly "org.apache.flink:flink-table-planner-blink_2.12:${flinkVersion}"
+ compileOnly "org.apache.flink:flink-table-planner_2.12:${flinkVersion}"
compileOnly "org.apache.hadoop:hadoop-hdfs"
compileOnly "org.apache.hadoop:hadoop-common"
compileOnly("org.apache.hadoop:hadoop-minicluster") {
@@ -56,13 +81,13 @@ project(':iceberg-flink') {
exclude group: 'org.apache.hive', module: 'hive-storage-api'
}
- testImplementation "org.apache.flink:flink-core"
- testImplementation "org.apache.flink:flink-runtime_2.12"
- testImplementation "org.apache.flink:flink-table-planner-blink_2.12"
- testImplementation ("org.apache.flink:flink-test-utils-junit") {
+ testImplementation "org.apache.flink:flink-core:${flinkVersion}"
+ testImplementation "org.apache.flink:flink-runtime_2.12:${flinkVersion}"
+ testImplementation "org.apache.flink:flink-table-planner-blink_2.12:${flinkVersion}"
+ testImplementation ("org.apache.flink:flink-test-utils-junit:${flinkVersion}") {
exclude group: 'junit'
}
- testImplementation("org.apache.flink:flink-test-utils_2.12") {
+ testImplementation("org.apache.flink:flink-test-utils_2.12:${flinkVersion}") {
exclude group: "org.apache.curator", module: 'curator-test'
exclude group: 'junit'
}
@@ -106,3 +131,60 @@ project(':iceberg-flink') {
}
}
+project(':iceberg-flink:iceberg-flink-1.13-runtime') {
+ apply plugin: 'com.github.johnrengelman.shadow'
+
+ tasks.jar.dependsOn tasks.shadowJar
+
+ configurations {
+ implementation {
+ exclude group: 'org.apache.flink'
+ // included in Flink
+ exclude group: 'org.slf4j'
+ exclude group: 'org.apache.commons'
+ exclude group: 'commons-pool'
+ exclude group: 'commons-codec'
+ exclude group: 'org.xerial.snappy'
+ exclude group: 'javax.xml.bind'
+ exclude group: 'javax.annotation'
+ }
+ }
+
+ dependencies {
+ implementation project(':iceberg-flink:iceberg-flink-1.13')
+ implementation project(':iceberg-aws')
+ implementation(project(':iceberg-nessie')) {
+ exclude group: 'com.google.code.findbugs', module: 'jsr305'
+ }
+ }
+
+ shadowJar {
+ configurations = [project.configurations.runtimeClasspath]
+
+ zip64 true
+
+ // include the LICENSE and NOTICE files for the shaded Jar
+ from(projectDir) {
+ include 'LICENSE'
+ include 'NOTICE'
+ }
+
+ // Relocate dependencies to avoid conflicts
+ relocate 'org.apache.avro', 'org.apache.iceberg.shaded.org.apache.avro'
+ relocate 'org.apache.parquet', 'org.apache.iceberg.shaded.org.apache.parquet'
+ relocate 'com.google', 'org.apache.iceberg.shaded.com.google'
+ relocate 'com.fasterxml', 'org.apache.iceberg.shaded.com.fasterxml'
+ relocate 'com.github.benmanes', 'org.apache.iceberg.shaded.com.github.benmanes'
+ relocate 'org.checkerframework', 'org.apache.iceberg.shaded.org.checkerframework'
+ relocate 'shaded.parquet', 'org.apache.iceberg.shaded.org.apache.parquet.shaded'
+ relocate 'org.apache.orc', 'org.apache.iceberg.shaded.org.apache.orc'
+ relocate 'io.airlift', 'org.apache.iceberg.shaded.io.airlift'
+ relocate 'org.threeten.extra', 'org.apache.iceberg.shaded.org.threeten.extra'
+
+ classifier null
+ }
+
+ jar {
+ enabled = false
+ }
+}
diff --git a/flink-runtime/LICENSE b/flink/v1.13/flink-runtime/LICENSE
similarity index 100%
rename from flink-runtime/LICENSE
rename to flink/v1.13/flink-runtime/LICENSE
diff --git a/flink-runtime/NOTICE b/flink/v1.13/flink-runtime/NOTICE
similarity index 100%
rename from flink-runtime/NOTICE
rename to flink/v1.13/flink-runtime/NOTICE
diff --git a/gradle.properties b/gradle.properties
index 120c5fa..2f064f8 100644
--- a/gradle.properties
+++ b/gradle.properties
@@ -15,8 +15,8 @@
jmhOutputPath=build/reports/jmh/human-readable-output.txt
jmhIncludeRegex=.*
-systemProp.defaultFlinkVersions=1.13
-systemProp.knownFlinkVersions=1.13
+systemProp.defaultFlinkVersions=1.12,1.13
+systemProp.knownFlinkVersions=1.12,1.13
systemProp.defaultHiveVersions=2,3
systemProp.knownHiveVersions=2,3
systemProp.defaultSparkVersions=2.4,3.0,3.1,3.2
diff --git a/settings.gradle b/settings.gradle
index 6a65b92..6ac1f8b 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -72,11 +72,27 @@ if (!knownSparkVersions.containsAll(sparkVersions)) {
throw new GradleException("Found unsupported Spark versions: " + (sparkVersions - knownSparkVersions))
}
-if (flinkVersions.contains("1.13")) {
+if (!flinkVersions.isEmpty()) {
include 'flink'
- include 'flink-runtime'
project(':flink').name = 'iceberg-flink'
- project(':flink-runtime').name = 'iceberg-flink-runtime'
+}
+
+if (flinkVersions.contains("1.12")) {
+ include ':iceberg-flink:flink-1.12'
+ include ':iceberg-flink:flink-1.12-runtime'
+ project(':iceberg-flink:flink-1.12').projectDir = file('flink/v1.12/flink')
+ project(':iceberg-flink:flink-1.12').name = 'iceberg-flink-1.12'
+ project(':iceberg-flink:flink-1.12-runtime').projectDir = file('flink/v1.12/flink-runtime')
+ project(':iceberg-flink:flink-1.12-runtime').name = 'iceberg-flink-1.12-runtime'
+}
+
+if (flinkVersions.contains("1.13")) {
+ include ':iceberg-flink:flink-1.13'
+ include ':iceberg-flink:flink-1.13-runtime'
+ project(':iceberg-flink:flink-1.13').projectDir = file('flink/v1.13/flink')
+ project(':iceberg-flink:flink-1.13').name = 'iceberg-flink-1.13'
+ project(':iceberg-flink:flink-1.13-runtime').projectDir = file('flink/v1.13/flink-runtime')
+ project(':iceberg-flink:flink-1.13-runtime').name = 'iceberg-flink-1.13-runtime'
}
if (sparkVersions.contains("3.0")) {