You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@orc.apache.org by do...@apache.org on 2021/09/16 11:39:09 UTC
[orc] branch branch-1.7 updated (04a2a05 -> 87e9b85)
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a change to branch branch-1.7
in repository https://gitbox.apache.org/repos/asf/orc.git.
from 04a2a05 Preparing branch for post 1.7.0 development
new 2d88235 ORC-1002: Add java17 profile for Java17 unit testing (#912)
new 87e9b85 ORC-1000: Use Java 17 in GitHub Action (#911)
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/build_and_test.yml | 3 +-
java/CMakeLists.txt | 35 ++++++++++++----------
java/core/pom.xml | 19 ++++++++++++
.../org/apache/orc/impl/TestPhysicalFsWriter.java | 2 +-
java/examples/pom.xml | 4 +++
java/mapreduce/pom.xml | 21 +++++++++++++
java/pom.xml | 8 +++++
java/shims/pom.xml | 19 ++++++++++++
java/tools/pom.xml | 21 +++++++++++++
9 files changed, 113 insertions(+), 19 deletions(-)
[orc] 01/02: ORC-1002: Add java17 profile for Java17 unit testing
(#912)
Posted by do...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch branch-1.7
in repository https://gitbox.apache.org/repos/asf/orc.git
commit 2d882355b24e1cfd5f350360d25309ad9f41aa48
Author: Dongjoon Hyun <do...@apache.org>
AuthorDate: Thu Sep 16 03:11:24 2021 -0700
ORC-1002: Add java17 profile for Java17 unit testing (#912)
This PR aims to add `java17` profile for Java 17 Unit testing.
This is required because Java 17 is supported by Hadoop 3.3.
Manual. This PR didn't add `activation` intentionally.
```
$ java -version
openjdk version "17" 2021-09-14 LTS
OpenJDK Runtime Environment Zulu17.28+13-CA (build 17+35-LTS)
OpenJDK 64-Bit Server VM Zulu17.28+13-CA (build 17+35-LTS, mixed mode, sharing)
$ mvn package -Pjava17
...
[INFO] ------------------------------------------------------------------------
[INFO] BUILD SUCCESS
[INFO] ------------------------------------------------------------------------
[INFO] Total time: 01:53 min
[INFO] Finished at: 2021-09-16T02:33:07-07:00
[INFO] ------------------------------------------------------------------------
```
(cherry picked from commit f25aba1180fe0e4602e6cd31ea9d2fece980f211)
Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
java/core/pom.xml | 19 +++++++++++++++++++
.../org/apache/orc/impl/TestPhysicalFsWriter.java | 2 +-
java/examples/pom.xml | 4 ++++
java/mapreduce/pom.xml | 21 +++++++++++++++++++++
java/pom.xml | 8 ++++++++
java/shims/pom.xml | 10 ++++++++++
java/tools/pom.xml | 21 +++++++++++++++++++++
7 files changed, 84 insertions(+), 1 deletion(-)
diff --git a/java/core/pom.xml b/java/core/pom.xml
index 60508fd..4355393 100644
--- a/java/core/pom.xml
+++ b/java/core/pom.xml
@@ -153,6 +153,8 @@
</ignoredUnusedDeclaredDependencies>
<ignoredDependencies>
<ignoredDependency>org.apache.hive:hive-storage-api</ignoredDependency>
+ <ignoredDependency>org.apache.hadoop:hadoop-client-api</ignoredDependency>
+ <ignoredDependency>org.apache.hadoop:hadoop-client-runtime</ignoredDependency>
</ignoredDependencies>
</configuration>
</plugin>
@@ -166,5 +168,22 @@
<directory>${build.dir}/core</directory>
</build>
</profile>
+ <profile>
+ <id>java17</id>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-api</artifactId>
+ <version>${hadoop.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-runtime</artifactId>
+ <version>${hadoop.version}</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ </profile>
</profiles>
</project>
diff --git a/java/core/src/test/org/apache/orc/impl/TestPhysicalFsWriter.java b/java/core/src/test/org/apache/orc/impl/TestPhysicalFsWriter.java
index 23bda3e..9feac31 100644
--- a/java/core/src/test/org/apache/orc/impl/TestPhysicalFsWriter.java
+++ b/java/core/src/test/org/apache/orc/impl/TestPhysicalFsWriter.java
@@ -93,7 +93,7 @@ public class TestPhysicalFsWriter {
Progressable progress) throws IOException {
List<byte[]> contents = new ArrayList<>();
fileContents.put(f, contents);
- return new FSDataOutputStream(new MemoryOutputStream(contents));
+ return new FSDataOutputStream(new MemoryOutputStream(contents), null);
}
@Override
diff --git a/java/examples/pom.xml b/java/examples/pom.xml
index 0d6e30a..bbbdfcf 100644
--- a/java/examples/pom.xml
+++ b/java/examples/pom.xml
@@ -110,7 +110,11 @@
<ignoredUnusedDeclaredDependencies>
<ignoredUnusedDeclaredDependency>com.google.guava:guava</ignoredUnusedDeclaredDependency>
<ignoredUnusedDeclaredDependency>org.apache.hadoop:hadoop-hdfs</ignoredUnusedDeclaredDependency>
+ <ignoredUnusedDeclaredDependency>org.apache.hadoop:hadoop-common</ignoredUnusedDeclaredDependency>
</ignoredUnusedDeclaredDependencies>
+ <ignoredDependencies>
+ <ignoredDependency>org.apache.hadoop:hadoop-client-api</ignoredDependency>
+ </ignoredDependencies>
</configuration>
</plugin>
</plugins>
diff --git a/java/mapreduce/pom.xml b/java/mapreduce/pom.xml
index 7b37c46..bc3942c 100644
--- a/java/mapreduce/pom.xml
+++ b/java/mapreduce/pom.xml
@@ -129,6 +129,10 @@
org.apache.hadoop:hadoop-mapreduce-client-jobclient
</ignoredUnusedDeclaredDependency>
</ignoredUnusedDeclaredDependencies>
+ <ignoredDependencies>
+ <ignoredDependency>org.apache.hadoop:hadoop-client-api</ignoredDependency>
+ <ignoredDependency>org.apache.hadoop:hadoop-client-runtime</ignoredDependency>
+ </ignoredDependencies>
</configuration>
</plugin>
</plugins>
@@ -141,5 +145,22 @@
<directory>${build.dir}/mapreduce</directory>
</build>
</profile>
+ <profile>
+ <id>java17</id>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-api</artifactId>
+ <version>${hadoop.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-runtime</artifactId>
+ <version>${hadoop.version}</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ </profile>
</profiles>
</project>
diff --git a/java/pom.xml b/java/pom.xml
index bc156f0..db3f116 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -500,6 +500,14 @@
</os>
</activation>
</profile>
+ <profile>
+ <id>java17</id>
+ <properties>
+ <min.hadoop.version>3.3.1</min.hadoop.version>
+ <hadoop.version>3.3.1</hadoop.version>
+ <tools.hadoop.version>3.3.1</tools.hadoop.version>
+ </properties>
+ </profile>
</profiles>
<dependencyManagement>
diff --git a/java/shims/pom.xml b/java/shims/pom.xml
index 7218b76..ff55a12 100644
--- a/java/shims/pom.xml
+++ b/java/shims/pom.xml
@@ -95,5 +95,15 @@
<directory>${build.dir}/shims</directory>
</build>
</profile>
+ <profile>
+ <id>java17</id>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-api</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+ </dependencies>
+ </profile>
</profiles>
</project>
diff --git a/java/tools/pom.xml b/java/tools/pom.xml
index 0c210f2..ae6945b 100644
--- a/java/tools/pom.xml
+++ b/java/tools/pom.xml
@@ -151,6 +151,10 @@
<ignoredUnusedDeclaredDependencies>
<ignoredUnusedDeclaredDependency>com.google.guava:guava</ignoredUnusedDeclaredDependency>
</ignoredUnusedDeclaredDependencies>
+ <ignoredDependencies>
+ <ignoredDependency>org.apache.hadoop:hadoop-client-api</ignoredDependency>
+ <ignoredDependency>org.apache.hadoop:hadoop-client-runtime</ignoredDependency>
+ </ignoredDependencies>
</configuration>
</plugin>
</plugins>
@@ -163,5 +167,22 @@
<directory>${build.dir}/tools</directory>
</build>
</profile>
+ <profile>
+ <id>java17</id>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-api</artifactId>
+ <version>${hadoop.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-runtime</artifactId>
+ <version>${hadoop.version}</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ </profile>
</profiles>
</project>
[orc] 02/02: ORC-1000: Use Java 17 in GitHub Action (#911)
Posted by do...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch branch-1.7
in repository https://gitbox.apache.org/repos/asf/orc.git
commit 87e9b851cd00a88bbd4124e06cfa17c850b9bff5
Author: Dongjoon Hyun <do...@apache.org>
AuthorDate: Thu Sep 16 04:30:33 2021 -0700
ORC-1000: Use Java 17 in GitHub Action (#911)
### What changes were proposed in this pull request?
This PR aims the following.
- Use `Java 17` instead of `Java 17 EA`.
- Remove `Java 16` to save the community testing resources
- Recover GitHub Action CI by using `java17` profile.
- Disable dependency check on `hadoop-hdfs` of `shim` module
Note that there is a TODO (ORC-1003) which fails due to some `shaded` classes.
We will revisit that.
### Why are the changes needed?
Java 17 is finally available.
### How was this patch tested?
Pass the CIs with Java 17.
When we test manually, it passed like the following.
```
$ java -version
openjdk version "17" 2021-09-14 LTS
OpenJDK Runtime Environment Zulu17.28+13-CA (build 17+35-LTS)
OpenJDK 64-Bit Server VM Zulu17.28+13-CA (build 17+35-LTS, mixed mode, sharing)
...
Test project /Users/dongjoon/APACHE/orc-merge/build
Start 1: orc-test
1/7 Test #1: orc-test ......................... Passed 3.41 sec
Start 2: java-test
2/7 Test #2: java-test ........................ Passed 97.92 sec
Start 3: java-tools-test
3/7 Test #3: java-tools-test .................. Passed 0.11 sec
Start 4: java-bench-gen-test
4/7 Test #4: java-bench-gen-test .............. Passed 0.98 sec
Start 5: java-bench-scan-test
5/7 Test #5: java-bench-scan-test ............. Passed 0.67 sec
Start 6: java-bench-hive-test
6/7 Test #6: java-bench-hive-test ............. Passed 12.47 sec
Start 7: tool-test
7/7 Test #7: tool-test ........................ Passed 10.02 sec
100% tests passed, 0 tests failed out of 7
Total Test time (real) = 125.58 sec
Built target test-out
```
---
.github/workflows/build_and_test.yml | 3 +--
java/CMakeLists.txt | 35 +++++++++++++++++++----------------
java/shims/pom.xml | 9 +++++++++
3 files changed, 29 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
index 1e61eac..6eae6d6 100644
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -22,8 +22,7 @@ jobs:
java:
- 1.8
- 11
- - 16
- - 17-ea
+ - 17
env:
MAVEN_OPTS: -Xmx2g
MAVEN_SKIP_RC: true
diff --git a/java/CMakeLists.txt b/java/CMakeLists.txt
index d556bc9..e9b1b4e 100644
--- a/java/CMakeLists.txt
+++ b/java/CMakeLists.txt
@@ -38,6 +38,19 @@ else()
set(JAVA_PROFILE "-Pcmake,benchmark")
endif()
+execute_process(
+ COMMAND java --add-opens java.base/java.nio=ALL-UNNAMED -version
+ WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+ RESULT_VARIABLE RET
+ OUTPUT_QUIET
+ ERROR_QUIET)
+if(RET EQUAL 0)
+ set(JAVA_PROFILE ${JAVA_PROFILE},java17)
+ set(JAVA_TEST_PROFILE -Pcmake,java17)
+ set(ADD_OPENS --add-opens)
+ set(JAVA_NIO java.base/java.nio=ALL-UNNAMED)
+endif()
+
add_custom_command(
OUTPUT ${ORC_JARS}
COMMAND ./mvnw ${NO_DOWNLOAD_MSG} ${JAVA_PROFILE}
@@ -50,31 +63,21 @@ add_custom_target(java_build ALL DEPENDS ${ORC_JARS})
add_test(
NAME java-test
- COMMAND ./mvnw ${NO_DOWNLOAD_MSG} -Pcmake
+ COMMAND ./mvnw ${NO_DOWNLOAD_MSG} ${JAVA_TEST_PROFILE}
-Dbuild.dir=${CMAKE_CURRENT_BINARY_DIR} test
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
-add_test(
- NAME java-examples-test
- COMMAND java -jar examples/orc-examples-${ORC_VERSION}-uber.jar write
- WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
+# TOOD(ORC-1003)
+# add_test(
+# NAME java-examples-test
+# COMMAND java -jar examples/orc-examples-${ORC_VERSION}-uber.jar write
+# WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
add_test(
NAME java-tools-test
COMMAND java -jar tools/orc-tools-${ORC_VERSION}-uber.jar version
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
-execute_process(
- COMMAND java --add-opens java.base/java.nio=ALL-UNNAMED -version
- WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
- RESULT_VARIABLE RET
- OUTPUT_QUIET
- ERROR_QUIET)
-if(RET EQUAL 0)
- set(ADD_OPENS --add-opens)
- set(JAVA_NIO java.base/java.nio=ALL-UNNAMED)
-endif()
-
add_test(
NAME java-bench-gen-test
COMMAND java ${ADD_OPENS} ${JAVA_NIO} -jar bench/core/orc-benchmarks-core-${ORC_VERSION}-uber.jar generate data -d sales -s 1
diff --git a/java/shims/pom.xml b/java/shims/pom.xml
index ff55a12..6d2d155 100644
--- a/java/shims/pom.xml
+++ b/java/shims/pom.xml
@@ -85,6 +85,15 @@
<groupId>org.codehaus.mojo</groupId>
<artifactId>build-helper-maven-plugin</artifactId>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <configuration>
+ <ignoredDependencies>
+ <ignoredDependency>org.apache.hadoop:hadoop-hdfs</ignoredDependency>
+ </ignoredDependencies>
+ </configuration>
+ </plugin>
</plugins>
</build>