You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by se...@apache.org on 2018/09/18 17:54:53 UTC
[flink] branch master updated (8d842e3 -> 0794fa8)
This is an automated email from the ASF dual-hosted git repository.
sewen pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git.
from 8d842e3 [hotfix] Fix checkstyle violations in SlotManagerTest
new 679d887 [FLINk-10362] [s3] S3 config loading does not search Hadoop classpath
new 84d23f9 [FLINK-10363] [s3] Only log config keys in config loader, to avoid exposing secret values
new 48931c1 [hotfix] [hdfs] Suppress some deprecation warnings
new 0794fa8 [hotfix] Move common dependencies into 'flink-filesystems'
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../flink/runtime/fs/hdfs/HadoopConfigLoader.java | 5 ++--
.../org/apache/flink/runtime/util/HadoopUtils.java | 1 +
.../runtime/fs/hdfs/HadoopConfigLoadingTest.java | 1 +
flink-filesystems/flink-s3-fs-hadoop/pom.xml | 14 -----------
flink-filesystems/flink-s3-fs-presto/pom.xml | 14 -----------
flink-filesystems/flink-swift-fs-hadoop/pom.xml | 21 ----------------
flink-filesystems/pom.xml | 29 ++++++++++++++++------
7 files changed, 26 insertions(+), 59 deletions(-)
[flink] 01/04: [FLINk-10362] [s3] S3 config loading does not search
Hadoop classpath
Posted by se...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sewen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 679d887d7bfb4a6510735b39f3afd7599f219b8b
Author: Stephan Ewen <se...@apache.org>
AuthorDate: Thu Sep 13 12:39:21 2018 +0200
[FLINk-10362] [s3] S3 config loading does not search Hadoop classpath
---
.../main/java/org/apache/flink/runtime/fs/hdfs/HadoopConfigLoader.java | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/flink-filesystems/flink-hadoop-fs/src/main/java/org/apache/flink/runtime/fs/hdfs/HadoopConfigLoader.java b/flink-filesystems/flink-hadoop-fs/src/main/java/org/apache/flink/runtime/fs/hdfs/HadoopConfigLoader.java
index a40e24f..499bb74 100644
--- a/flink-filesystems/flink-hadoop-fs/src/main/java/org/apache/flink/runtime/fs/hdfs/HadoopConfigLoader.java
+++ b/flink-filesystems/flink-hadoop-fs/src/main/java/org/apache/flink/runtime/fs/hdfs/HadoopConfigLoader.java
@@ -19,7 +19,6 @@
package org.apache.flink.runtime.fs.hdfs;
import org.apache.flink.configuration.Configuration;
-import org.apache.flink.runtime.util.HadoopUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -91,7 +90,7 @@ public class HadoopConfigLoader {
// add additional config entries from the Flink config to the Hadoop config
private org.apache.hadoop.conf.Configuration loadHadoopConfigFromFlink() {
- org.apache.hadoop.conf.Configuration hadoopConfig = HadoopUtils.getHadoopConfiguration(flinkConfig);
+ org.apache.hadoop.conf.Configuration hadoopConfig = new org.apache.hadoop.conf.Configuration();
for (String key : flinkConfig.keySet()) {
for (String prefix : flinkConfigPrefixes) {
if (key.startsWith(prefix)) {
[flink] 03/04: [hotfix] [hdfs] Suppress some deprecation warnings
Posted by se...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sewen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 48931c1e74ff1078e1f9062fab4b2a6085253b7b
Author: Stephan Ewen <se...@apache.org>
AuthorDate: Thu Sep 13 20:46:16 2018 +0200
[hotfix] [hdfs] Suppress some deprecation warnings
---
.../src/main/java/org/apache/flink/runtime/util/HadoopUtils.java | 1 +
.../java/org/apache/flink/runtime/fs/hdfs/HadoopConfigLoadingTest.java | 1 +
2 files changed, 2 insertions(+)
diff --git a/flink-filesystems/flink-hadoop-fs/src/main/java/org/apache/flink/runtime/util/HadoopUtils.java b/flink-filesystems/flink-hadoop-fs/src/main/java/org/apache/flink/runtime/util/HadoopUtils.java
index d0b3a7a..1e774fe 100644
--- a/flink-filesystems/flink-hadoop-fs/src/main/java/org/apache/flink/runtime/util/HadoopUtils.java
+++ b/flink-filesystems/flink-hadoop-fs/src/main/java/org/apache/flink/runtime/util/HadoopUtils.java
@@ -44,6 +44,7 @@ public class HadoopUtils {
private static final Text HDFS_DELEGATION_TOKEN_KIND = new Text("HDFS_DELEGATION_TOKEN");
+ @SuppressWarnings("deprecation")
public static Configuration getHadoopConfiguration(org.apache.flink.configuration.Configuration flinkConfiguration) {
// Instantiate a HdfsConfiguration to load the hdfs-site.xml and hdfs-default.xml
diff --git a/flink-filesystems/flink-hadoop-fs/src/test/java/org/apache/flink/runtime/fs/hdfs/HadoopConfigLoadingTest.java b/flink-filesystems/flink-hadoop-fs/src/test/java/org/apache/flink/runtime/fs/hdfs/HadoopConfigLoadingTest.java
index bb2c492..bb3d088 100644
--- a/flink-filesystems/flink-hadoop-fs/src/test/java/org/apache/flink/runtime/fs/hdfs/HadoopConfigLoadingTest.java
+++ b/flink-filesystems/flink-hadoop-fs/src/test/java/org/apache/flink/runtime/fs/hdfs/HadoopConfigLoadingTest.java
@@ -41,6 +41,7 @@ import static org.junit.Assert.assertTrue;
* Tests that validate the loading of the Hadoop configuration, relative to
* entries in the Flink configuration and the environment variables.
*/
+@SuppressWarnings("deprecation")
public class HadoopConfigLoadingTest {
private static final String IN_CP_CONFIG_KEY = "cp_conf_key";
[flink] 02/04: [FLINK-10363] [s3] Only log config keys in config
loader, to avoid exposing secret values
Posted by se...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sewen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 84d23f954ca58e7184f46e6a2b4f3473b49c8084
Author: Stephan Ewen <se...@apache.org>
AuthorDate: Thu Sep 13 20:44:39 2018 +0200
[FLINK-10363] [s3] Only log config keys in config loader, to avoid exposing secret values
---
.../main/java/org/apache/flink/runtime/fs/hdfs/HadoopConfigLoader.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/flink-filesystems/flink-hadoop-fs/src/main/java/org/apache/flink/runtime/fs/hdfs/HadoopConfigLoader.java b/flink-filesystems/flink-hadoop-fs/src/main/java/org/apache/flink/runtime/fs/hdfs/HadoopConfigLoader.java
index 499bb74..82916f3 100644
--- a/flink-filesystems/flink-hadoop-fs/src/main/java/org/apache/flink/runtime/fs/hdfs/HadoopConfigLoader.java
+++ b/flink-filesystems/flink-hadoop-fs/src/main/java/org/apache/flink/runtime/fs/hdfs/HadoopConfigLoader.java
@@ -99,7 +99,7 @@ public class HadoopConfigLoader {
String newValue = fixHadoopConfig(key, flinkConfig.getString(key, null));
hadoopConfig.set(newKey, newValue);
- LOG.debug("Adding Flink config entry for {} as {}={} to Hadoop config", key, newKey, value);
+ LOG.debug("Adding Flink config entry for {} as {} to Hadoop config", key, newKey);
}
}
}
[flink] 04/04: [hotfix] Move common dependencies into
'flink-filesystems'
Posted by se...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sewen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 0794fa862134fbd669c4e15a410adc2f877cc393
Author: Stephan Ewen <se...@apache.org>
AuthorDate: Wed Sep 12 12:54:16 2018 +0200
[hotfix] Move common dependencies into 'flink-filesystems'
---
flink-filesystems/flink-s3-fs-hadoop/pom.xml | 14 ------------
flink-filesystems/flink-s3-fs-presto/pom.xml | 14 ------------
flink-filesystems/flink-swift-fs-hadoop/pom.xml | 21 ------------------
flink-filesystems/pom.xml | 29 +++++++++++++++++++------
4 files changed, 22 insertions(+), 56 deletions(-)
diff --git a/flink-filesystems/flink-s3-fs-hadoop/pom.xml b/flink-filesystems/flink-s3-fs-hadoop/pom.xml
index 1598518..fb6bda1 100644
--- a/flink-filesystems/flink-s3-fs-hadoop/pom.xml
+++ b/flink-filesystems/flink-s3-fs-hadoop/pom.xml
@@ -161,20 +161,6 @@ under the License.
<artifactId>aws-java-sdk-s3</artifactId>
</dependency>
- <!-- make sure that also logger and JSR is provided -->
-
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
- <groupId>com.google.code.findbugs</groupId>
- <artifactId>jsr305</artifactId>
- <scope>provided</scope>
- </dependency>
-
<!-- for the behavior test suite -->
<dependency>
<groupId>org.apache.flink</groupId>
diff --git a/flink-filesystems/flink-s3-fs-presto/pom.xml b/flink-filesystems/flink-s3-fs-presto/pom.xml
index 67ba69c..8720217 100644
--- a/flink-filesystems/flink-s3-fs-presto/pom.xml
+++ b/flink-filesystems/flink-s3-fs-presto/pom.xml
@@ -184,20 +184,6 @@ under the License.
<version>2.7.3-1</version>
</dependency>
- <!-- make sure that also logger and JSR is provided -->
-
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
- <groupId>com.google.code.findbugs</groupId>
- <artifactId>jsr305</artifactId>
- <scope>provided</scope>
- </dependency>
-
<!-- for the behavior test suite -->
<dependency>
<groupId>org.apache.flink</groupId>
diff --git a/flink-filesystems/flink-swift-fs-hadoop/pom.xml b/flink-filesystems/flink-swift-fs-hadoop/pom.xml
index e2a3bab..d0445d6 100644
--- a/flink-filesystems/flink-swift-fs-hadoop/pom.xml
+++ b/flink-filesystems/flink-swift-fs-hadoop/pom.xml
@@ -130,27 +130,6 @@ under the License.
</exclusion>
</exclusions>
</dependency>
-
- <!-- make sure that also logger and JSR is provided -->
-
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
- <groupId>com.google.code.findbugs</groupId>
- <artifactId>jsr305</artifactId>
- <scope>provided</scope>
- </dependency>
-
- <!-- test dependencies -->
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-test-utils-junit</artifactId>
- </dependency>
</dependencies>
<build>
diff --git a/flink-filesystems/pom.xml b/flink-filesystems/pom.xml
index f0a4ab3..148eed6 100644
--- a/flink-filesystems/pom.xml
+++ b/flink-filesystems/pom.xml
@@ -43,6 +43,28 @@ under the License.
<module>flink-swift-fs-hadoop</module>
</modules>
+ <!-- Common dependency setup for all filesystems -->
+ <dependencies>
+
+ <!-- The filesystems are extensions that can assume logger and JSR to be already provided -->
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.google.code.findbugs</groupId>
+ <artifactId>jsr305</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <!-- Base testing tools -->
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-test-utils-junit</artifactId>
+ </dependency>
+ </dependencies>
+
<build>
<plugins>
<plugin>
@@ -63,11 +85,4 @@ under the License.
</plugins>
</build>
- <dependencies>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-test-utils-junit</artifactId>
- </dependency>
- </dependencies>
-
</project>