You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by se...@apache.org on 2020/09/15 20:13:50 UTC
[flink] 02/11: [FLINK-19218][core] Remove inconsistent/misleading
locality information from Local File Splits.
This is an automated email from the ASF dual-hosted git repository.
sewen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 5bcdd7de67bf21bc19c8b5b84f617dacaf7f0c99
Author: Stephan Ewen <se...@apache.org>
AuthorDate: Mon Sep 14 16:28:53 2020 +0200
[FLINK-19218][core] Remove inconsistent/misleading locality information from Local File Splits.
---
.../flink/core/fs/local/LocalBlockLocation.java | 16 ++++++++--------
.../apache/flink/core/fs/local/LocalFileSystem.java | 20 +-------------------
.../main/java/org/apache/flink/util/StringUtils.java | 6 ++++++
3 files changed, 15 insertions(+), 27 deletions(-)
diff --git a/flink-core/src/main/java/org/apache/flink/core/fs/local/LocalBlockLocation.java b/flink-core/src/main/java/org/apache/flink/core/fs/local/LocalBlockLocation.java
index 25dd92d..512bc5b 100644
--- a/flink-core/src/main/java/org/apache/flink/core/fs/local/LocalBlockLocation.java
+++ b/flink-core/src/main/java/org/apache/flink/core/fs/local/LocalBlockLocation.java
@@ -20,27 +20,27 @@ package org.apache.flink.core.fs.local;
import org.apache.flink.annotation.Internal;
import org.apache.flink.core.fs.BlockLocation;
-
-import java.io.IOException;
+import org.apache.flink.util.StringUtils;
/**
* Implementation of the {@link BlockLocation} interface for a local file system.
+ *
+ * <p>Local files have only one block that represents the entire file.
+ * The block has no location information, because it is not accessible where the files (or their block)
+ * actually reside, especially in cases where the files are on a mounted file system.
*/
@Internal
public class LocalBlockLocation implements BlockLocation {
private final long length;
- private final String[] hosts;
-
- public LocalBlockLocation(final String host, final long length) {
- this.hosts = new String[] { host };
+ public LocalBlockLocation(final long length) {
this.length = length;
}
@Override
- public String[] getHosts() throws IOException {
- return this.hosts;
+ public String[] getHosts() {
+ return StringUtils.EMPTY_STRING_ARRAY;
}
@Override
diff --git a/flink-core/src/main/java/org/apache/flink/core/fs/local/LocalFileSystem.java b/flink-core/src/main/java/org/apache/flink/core/fs/local/LocalFileSystem.java
index 32ae50b..154fc2c 100644
--- a/flink-core/src/main/java/org/apache/flink/core/fs/local/LocalFileSystem.java
+++ b/flink-core/src/main/java/org/apache/flink/core/fs/local/LocalFileSystem.java
@@ -35,15 +35,10 @@ import org.apache.flink.core.fs.FileSystemKind;
import org.apache.flink.core.fs.Path;
import org.apache.flink.util.OperatingSystem;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
-import java.net.InetAddress;
import java.net.URI;
-import java.net.UnknownHostException;
import java.nio.file.AccessDeniedException;
import java.nio.file.DirectoryNotEmptyException;
import java.nio.file.FileAlreadyExistsException;
@@ -61,8 +56,6 @@ import static org.apache.flink.util.Preconditions.checkState;
@Internal
public class LocalFileSystem extends FileSystem {
- private static final Logger LOG = LoggerFactory.getLogger(LocalFileSystem.class);
-
/** The URI representing the local file system. */
private static final URI LOCAL_URI = OperatingSystem.isWindows() ? URI.create("file:/") : URI.create("file:///");
@@ -77,23 +70,12 @@ public class LocalFileSystem extends FileSystem {
* Because Paths are not immutable, we cannot cache the proper path here. */
private final URI homeDir;
- /** The host name of this machine. */
- private final String hostName;
-
/**
* Constructs a new <code>LocalFileSystem</code> object.
*/
public LocalFileSystem() {
this.workingDir = new File(System.getProperty("user.dir")).toURI();
this.homeDir = new File(System.getProperty("user.home")).toURI();
-
- String tmp = "unknownHost";
- try {
- tmp = InetAddress.getLocalHost().getHostName();
- } catch (UnknownHostException e) {
- LOG.error("Could not resolve local host", e);
- }
- this.hostName = tmp;
}
// ------------------------------------------------------------------------
@@ -101,7 +83,7 @@ public class LocalFileSystem extends FileSystem {
@Override
public BlockLocation[] getFileBlockLocations(FileStatus file, long start, long len) throws IOException {
return new BlockLocation[] {
- new LocalBlockLocation(hostName, file.getLen())
+ new LocalBlockLocation(file.getLen())
};
}
diff --git a/flink-core/src/main/java/org/apache/flink/util/StringUtils.java b/flink-core/src/main/java/org/apache/flink/util/StringUtils.java
index fa591d3..a23bc14 100644
--- a/flink-core/src/main/java/org/apache/flink/util/StringUtils.java
+++ b/flink-core/src/main/java/org/apache/flink/util/StringUtils.java
@@ -41,6 +41,12 @@ import static org.apache.flink.util.Preconditions.checkNotNull;
@PublicEvolving
public final class StringUtils {
+ /**
+ * An empty string array. There are just too many places where one needs an empty string array
+ * and wants to save some object allocation.
+ */
+ public static final String[] EMPTY_STRING_ARRAY = new String[0];
+
private static final char[] HEX_CHARS = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f' };
/**