You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by lz...@apache.org on 2020/05/06 12:34:53 UTC
[flink] branch master updated: [FLINK-17460][orc][parquet] Create
sql-jars for parquet and orc
This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/master by this push:
new d40cf24 [FLINK-17460][orc][parquet] Create sql-jars for parquet and orc
d40cf24 is described below
commit d40cf24e32780735f1159a5a2e7383ab376f7be8
Author: Jingsong Lee <ji...@gmail.com>
AuthorDate: Wed May 6 20:34:13 2020 +0800
[FLINK-17460][orc][parquet] Create sql-jars for parquet and orc
This closes #11946
---
docs/dev/table/connect.md | 2 ++
flink-formats/flink-orc/pom.xml | 28 +++++++++++++++++++++-
.../src/main/resources/META-INF/services/NOTICE | 18 ++++++++++++++
flink-formats/flink-parquet/pom.xml | 25 +++++++++++++++++++
.../src/main/resources/META-INF/services/NOTICE | 21 ++++++++++++++++
5 files changed, 93 insertions(+), 1 deletion(-)
diff --git a/docs/dev/table/connect.md b/docs/dev/table/connect.md
index 8ffa911..6676de6 100644
--- a/docs/dev/table/connect.md
+++ b/docs/dev/table/connect.md
@@ -59,6 +59,8 @@ The following tables list all available connectors and formats. Their mutual com
| CSV (for Kafka) | `flink-csv` | [Download](https://repo.maven.apache.org/maven2/org/apache/flink/flink-csv/{{site.version}}/flink-csv-{{site.version}}-sql-jar.jar) |
| JSON | `flink-json` | [Download](https://repo.maven.apache.org/maven2/org/apache/flink/flink-json/{{site.version}}/flink-json-{{site.version}}-sql-jar.jar) |
| Apache Avro | `flink-avro` | [Download](https://repo.maven.apache.org/maven2/org/apache/flink/flink-avro/{{site.version}}/flink-avro-{{site.version}}-sql-jar.jar) |
+| Apache ORC | `flink-orc` | [Download](https://repo.maven.apache.org/maven2/org/apache/flink/flink-orc{{site.scala_version_suffix}}/{{site.version}}/flink-orc{{site.scala_version_suffix}}-{{site.version}}-jar-with-dependencies.jar) |
+| Apache Parquet | `flink-parquet` | [Download](https://repo.maven.apache.org/maven2/org/apache/flink/flink-parquet{{site.scala_version_suffix}}/{{site.version}}/flink-parquet{{site.scala_version_suffix}}-{{site.version}}-jar-with-dependencies.jar) |
{% else %}
diff --git a/flink-formats/flink-orc/pom.xml b/flink-formats/flink-orc/pom.xml
index 4e6b329..3aa36cb 100644
--- a/flink-formats/flink-orc/pom.xml
+++ b/flink-formats/flink-orc/pom.xml
@@ -84,6 +84,14 @@ under the License.
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>javax.activation</groupId>
+ <artifactId>javax.activation-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.xml.bind</groupId>
+ <artifactId>jaxb-api</artifactId>
+ </exclusion>
</exclusions>
</dependency>
@@ -169,7 +177,25 @@ under the License.
</execution>
</executions>
</plugin>
+
+ <!-- build a jar-with-dependencies SQL Client uber jars -->
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <configuration>
+ <descriptorRefs>
+ <descriptorRef>jar-with-dependencies</descriptorRef>
+ </descriptorRefs>
+ </configuration>
+ <executions>
+ <execution>
+ <id>make-assembly</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
-
</project>
diff --git a/flink-formats/flink-orc/src/main/resources/META-INF/services/NOTICE b/flink-formats/flink-orc/src/main/resources/META-INF/services/NOTICE
new file mode 100644
index 0000000..a1d6ff3
--- /dev/null
+++ b/flink-formats/flink-orc/src/main/resources/META-INF/services/NOTICE
@@ -0,0 +1,18 @@
+flink-orc
+Copyright 2014-2020 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
+
+This project bundles the following dependencies under the Apache Software License 2.0. (http://www.apache.org/licenses/LICENSE-2.0.txt)
+
+- org.apache.orc:orc-core:1.5.6
+- org.apache.orc:orc-shims:1.5.6
+- org.apache.hive:hive-storage-api:2.6.0
+- io.airlift:aircompressor:0.10
+- commons-lang:commons-lang:2.6
+
+This project bundles the following dependencies under the BSD license.
+See bundled license files for details.
+
+- com.google.protobuf:protobuf-java:2.5.0
diff --git a/flink-formats/flink-parquet/pom.xml b/flink-formats/flink-parquet/pom.xml
index 901fa66..00b1d3e 100644
--- a/flink-formats/flink-parquet/pom.xml
+++ b/flink-formats/flink-parquet/pom.xml
@@ -87,6 +87,12 @@ under the License.
<groupId>org.apache.parquet</groupId>
<artifactId>parquet-hadoop</artifactId>
<version>${flink.format.parquet.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.xerial.snappy</groupId>
+ <artifactId>snappy-java</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<!-- Hadoop is needed by Parquet -->
@@ -200,6 +206,25 @@ under the License.
</execution>
</executions>
</plugin>
+
+ <!-- build a jar-with-dependencies SQL Client uber jars -->
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <configuration>
+ <descriptorRefs>
+ <descriptorRef>jar-with-dependencies</descriptorRef>
+ </descriptorRefs>
+ </configuration>
+ <executions>
+ <execution>
+ <id>make-assembly</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
</project>
diff --git a/flink-formats/flink-parquet/src/main/resources/META-INF/services/NOTICE b/flink-formats/flink-parquet/src/main/resources/META-INF/services/NOTICE
new file mode 100644
index 0000000..6a2ea27
--- /dev/null
+++ b/flink-formats/flink-parquet/src/main/resources/META-INF/services/NOTICE
@@ -0,0 +1,21 @@
+flink-orc
+Copyright 2014-2020 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
+
+This project bundles the following dependencies under the Apache Software License 2.0. (http://www.apache.org/licenses/LICENSE-2.0.txt)
+
+- org.apache.parquet:parquet-hadoop:1.10.0
+- org.apache.parquet:parquet-column:1.10.0
+- org.apache.parquet:parquet-common:1.10.0
+- org.apache.parquet:parquet-encoding:1.10.0
+- org.apache.parquet:parquet-format:2.4.0
+- org.apache.parquet:parquet-jackson:1.10.0
+- org.apache.parquet:parquet-avro:1.10.0
+- org.codehaus.jackson:jackson-mapper-asl:1.9.13
+- org.codehaus.jackson:jackson-core-asl:1.9.13
+- org.apache.commons:commons-compress:1.20
+- org.apache.avro:avro:1.8.2
+- commons-pool:commons-pool:1.6
+- commons-codec:commons-codec:1.10