You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by lz...@apache.org on 2023/12/01 05:06:04 UTC
(incubator-paimon) branch master updated: [oss] Use old oss sdk because jindo sdk contains closed-source resources (#2435)
This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-paimon.git
The following commit(s) were added to refs/heads/master by this push:
new e93099dea [oss] Use old oss sdk because jindo sdk contains closed-source resources (#2435)
e93099dea is described below
commit e93099dea341c44788f0e585a26660fc4b7f3529
Author: tsreaper <ts...@gmail.com>
AuthorDate: Fri Dec 1 13:06:00 2023 +0800
[oss] Use old oss sdk because jindo sdk contains closed-source resources (#2435)
---
paimon-filesystems/paimon-oss-impl/pom.xml | 48 ++++++++++++++--------
.../main/java/org/apache/paimon/oss/OSSFileIO.java | 10 ++---
.../src/main/resources/META-INF/NOTICE | 8 ++--
3 files changed, 38 insertions(+), 28 deletions(-)
diff --git a/paimon-filesystems/paimon-oss-impl/pom.xml b/paimon-filesystems/paimon-oss-impl/pom.xml
index 457755e8c..6ad2ce032 100644
--- a/paimon-filesystems/paimon-oss-impl/pom.xml
+++ b/paimon-filesystems/paimon-oss-impl/pom.xml
@@ -32,16 +32,9 @@
<packaging>jar</packaging>
<properties>
- <jindodata.version>6.1.1</jindodata.version>
+ <fs.oss.sdk.version>3.13.2</fs.oss.sdk.version>
</properties>
- <repositories>
- <repository>
- <id>jindodata</id>
- <url>https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/mvn-repo/</url>
- </repository>
- </repositories>
-
<dependencies>
<dependency>
<groupId>org.apache.paimon</groupId>
@@ -57,15 +50,40 @@
</dependency>
<dependency>
- <groupId>com.aliyun.jindodata</groupId>
- <artifactId>jindo-core</artifactId>
- <version>${jindodata.version}</version>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-aliyun</artifactId>
+ <version>${fs.hadoopshaded.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.aliyun.oss</groupId>
+ <artifactId>aliyun-oss-sdk</artifactId>
+ </exclusion>
+ <exclusion>
+ <!-- provided by paimon-hadoop-shaded -->
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>ch.qos.reload4j</groupId>
+ <artifactId>reload4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-reload4j</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
- <groupId>com.aliyun.jindodata</groupId>
- <artifactId>jindo-sdk</artifactId>
- <version>${jindodata.version}</version>
+ <groupId>com.aliyun.oss</groupId>
+ <artifactId>aliyun-sdk-oss</artifactId>
+ <version>${fs.oss.sdk.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>javax.xml.bind</groupId>
+ <artifactId>jaxb-api</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
@@ -105,8 +123,6 @@
<exclude>.gitkeep</exclude>
<exclude>mime.types</exclude>
<exclude>mozilla/**</exclude>
- <exclude>META-INF/maven/**</exclude>
- <exclude>META-INF/LICENSE.txt</exclude>
</excludes>
</filter>
</filters>
diff --git a/paimon-filesystems/paimon-oss-impl/src/main/java/org/apache/paimon/oss/OSSFileIO.java b/paimon-filesystems/paimon-oss-impl/src/main/java/org/apache/paimon/oss/OSSFileIO.java
index eb3b11cdb..6ecc76da3 100644
--- a/paimon-filesystems/paimon-oss-impl/src/main/java/org/apache/paimon/oss/OSSFileIO.java
+++ b/paimon-filesystems/paimon-oss-impl/src/main/java/org/apache/paimon/oss/OSSFileIO.java
@@ -22,9 +22,9 @@ import org.apache.paimon.catalog.CatalogContext;
import org.apache.paimon.fs.FileIO;
import org.apache.paimon.options.Options;
-import com.aliyun.jindodata.oss.JindoOssFileSystem;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -66,7 +66,7 @@ public class OSSFileIO extends HadoopCompliantFileIO {
* Cache AliyunOSSFileSystem, at present, there is no good mechanism to ensure that the file
* system will be shut down, so here the fs cache is used to avoid resource leakage.
*/
- private static final Map<CacheKey, JindoOssFileSystem> CACHE = new ConcurrentHashMap<>();
+ private static final Map<CacheKey, AliyunOSSFileSystem> CACHE = new ConcurrentHashMap<>();
private Options hadoopOptions;
@@ -78,10 +78,6 @@ public class OSSFileIO extends HadoopCompliantFileIO {
@Override
public void configure(CatalogContext context) {
hadoopOptions = new Options();
- // https://github.com/aliyun/alibabacloud-jindodata/blob/master/docs/user/4.x/4.6.x/4.6.1/oss/hadoop/jindosdk_ide_hadoop.md
- hadoopOptions.set("fs.oss.impl", "com.aliyun.jindodata.oss.JindoOssFileSystem");
- hadoopOptions.set("fs.AbstractFileSystem.oss.impl", "com.aliyun.jindodata.oss.OSS");
-
// read all configuration with prefix 'CONFIG_PREFIXES'
for (String key : context.options().keySet()) {
for (String prefix : CONFIG_PREFIXES) {
@@ -121,7 +117,7 @@ public class OSSFileIO extends HadoopCompliantFileIO {
}
}
- JindoOssFileSystem fs = new JindoOssFileSystem();
+ AliyunOSSFileSystem fs = new AliyunOSSFileSystem();
try {
fs.initialize(fsUri, hadoopConf);
} catch (IOException e) {
diff --git a/paimon-filesystems/paimon-oss-impl/src/main/resources/META-INF/NOTICE b/paimon-filesystems/paimon-oss-impl/src/main/resources/META-INF/NOTICE
index f698fdffe..44d2a1d2f 100644
--- a/paimon-filesystems/paimon-oss-impl/src/main/resources/META-INF/NOTICE
+++ b/paimon-filesystems/paimon-oss-impl/src/main/resources/META-INF/NOTICE
@@ -13,14 +13,14 @@ This project bundles the following dependencies under the Apache Software Licens
- com.aliyun:aliyun-java-sdk-kms:2.11.0
- com.aliyun:aliyun-java-sdk-ram:3.1.0
- com.google.code.gson:gson:2.8.9
-- commons-codec:commons-codec:1.15
+- commons-codec:commons-codec:1.9
- commons-logging:commons-logging:1.1.3
- io.opentracing:opentracing-api:0.33.0
- io.opentracing:opentracing-noop:0.33.0
- io.opentracing:opentracing-util:0.33.0
- org.apache.hadoop:hadoop-aliyun:3.3.4
-- org.apache.httpcomponents:httpclient:4.5.13
-- org.apache.httpcomponents:httpcore:4.4.14
+- org.apache.httpcomponents:httpclient:4.4.1
+- org.apache.httpcomponents:httpcore:4.4.1
- org.codehaus.jettison:jettison:1.1
- org.ini4j:ini4j:0.5.4
- stax:stax-api:1.0.1
@@ -47,8 +47,6 @@ This project bundles the following dependencies under the Apache Software Licens
- org.xerial.snappy:snappy-java:1.1.8.2
- com.fasterxml.jackson.core:jackson-core:2.14.2
- com.fasterxml.jackson.core:jackson-annotations:2.14.2
-- com.aliyun.jindodata:jindo-core:6.1.1
-- com.aliyun.jindodata:jindo-sdk:6.1.1
- com.google.code.findbugs:jsr305:1.3.9
The binary distribution of this product bundles these dependencies under the Eclipse Public License - v 2.0