You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by gy...@apache.org on 2015/01/25 22:11:11 UTC
[1/3] flink git commit: [FLINK-1081] Add getPos() method into
FSDataInputStream class
Repository: flink
Updated Branches:
refs/heads/master 7ce9a8ff9 -> 9b63f269e
[FLINK-1081] Add getPos() method into FSDataInputStream class
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/7aa9a503
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/7aa9a503
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/7aa9a503
Branch: refs/heads/master
Commit: 7aa9a5035c3679ba80ca7d6b29ced5f51d92265e
Parents: 7ce9a8f
Author: Chiwan Park <ch...@icloud.com>
Authored: Thu Nov 27 20:01:25 2014 +0900
Committer: Gyula Fora <gy...@apache.org>
Committed: Sun Jan 25 18:56:55 2015 +0100
----------------------------------------------------------------------
.../io/InflaterInputStreamFSInputWrapper.java | 5 ++
.../apache/flink/core/fs/FSDataInputStream.java | 6 +++
.../core/fs/local/LocalDataInputStream.java | 5 ++
.../runtime/fs/hdfs/HadoopDataInputStream.java | 6 +++
.../flink/runtime/fs/s3/S3DataInputStream.java | 55 +++++++++++++++++---
5 files changed, 71 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/7aa9a503/flink-core/src/main/java/org/apache/flink/api/common/io/InflaterInputStreamFSInputWrapper.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/io/InflaterInputStreamFSInputWrapper.java b/flink-core/src/main/java/org/apache/flink/api/common/io/InflaterInputStreamFSInputWrapper.java
index 250b5f5..c940cbc 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/io/InflaterInputStreamFSInputWrapper.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/io/InflaterInputStreamFSInputWrapper.java
@@ -38,6 +38,11 @@ public class InflaterInputStreamFSInputWrapper extends FSDataInputStream {
}
@Override
+ public long getPos() throws IOException {
+ throw new UnsupportedOperationException("Compressed streams do not support the getPos operation");
+ }
+
+ @Override
public int read() throws IOException {
return inStream.read();
}
http://git-wip-us.apache.org/repos/asf/flink/blob/7aa9a503/flink-core/src/main/java/org/apache/flink/core/fs/FSDataInputStream.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/core/fs/FSDataInputStream.java b/flink-core/src/main/java/org/apache/flink/core/fs/FSDataInputStream.java
index 19a34dd..9ebc7f7 100644
--- a/flink-core/src/main/java/org/apache/flink/core/fs/FSDataInputStream.java
+++ b/flink-core/src/main/java/org/apache/flink/core/fs/FSDataInputStream.java
@@ -45,4 +45,10 @@ public abstract class FSDataInputStream extends InputStream {
*/
public abstract void seek(long desired) throws IOException;
+ /**
+ * Get the current position in the input stream.
+ *
+ * @return current position in the input stream
+ */
+ public abstract long getPos() throws IOException;
}
http://git-wip-us.apache.org/repos/asf/flink/blob/7aa9a503/flink-core/src/main/java/org/apache/flink/core/fs/local/LocalDataInputStream.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/core/fs/local/LocalDataInputStream.java b/flink-core/src/main/java/org/apache/flink/core/fs/local/LocalDataInputStream.java
index 1421ab5..0d5fcf2 100644
--- a/flink-core/src/main/java/org/apache/flink/core/fs/local/LocalDataInputStream.java
+++ b/flink-core/src/main/java/org/apache/flink/core/fs/local/LocalDataInputStream.java
@@ -57,6 +57,11 @@ public class LocalDataInputStream extends FSDataInputStream {
this.fis.getChannel().position(desired);
}
+ @Override
+ public long getPos() throws IOException {
+ return this.fis.getChannel().position();
+ }
+
@Override
public int read() throws IOException {
http://git-wip-us.apache.org/repos/asf/flink/blob/7aa9a503/flink-runtime/src/main/java/org/apache/flink/runtime/fs/hdfs/HadoopDataInputStream.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/fs/hdfs/HadoopDataInputStream.java b/flink-runtime/src/main/java/org/apache/flink/runtime/fs/hdfs/HadoopDataInputStream.java
index 9a606fd..8aa3c57 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/fs/hdfs/HadoopDataInputStream.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/fs/hdfs/HadoopDataInputStream.java
@@ -49,6 +49,12 @@ public final class HadoopDataInputStream extends FSDataInputStream {
}
@Override
+ public long getPos() throws IOException {
+
+ return fsDataInputStream.getPos();
+ }
+
+ @Override
public int read() throws IOException {
return fsDataInputStream.read();
http://git-wip-us.apache.org/repos/asf/flink/blob/7aa9a503/flink-runtime/src/main/java/org/apache/flink/runtime/fs/s3/S3DataInputStream.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/fs/s3/S3DataInputStream.java b/flink-runtime/src/main/java/org/apache/flink/runtime/fs/s3/S3DataInputStream.java
index 6dbae7d..715f1a1 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/fs/s3/S3DataInputStream.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/fs/s3/S3DataInputStream.java
@@ -32,7 +32,6 @@ import com.amazonaws.services.s3.model.S3Object;
/**
* This class implements an {@link FSDataInputStream} that downloads its data from Amazon S3 in the background.
* Essentially, this class is just a wrapper to the Amazon AWS SDK.
- *
*/
public class S3DataInputStream extends FSDataInputStream {
@@ -42,8 +41,18 @@ public class S3DataInputStream extends FSDataInputStream {
private final InputStream inputStream;
/**
+ * The current position of input stream.
+ */
+ private long position;
+
+ /**
+ * The marked position.
+ */
+ private long marked;
+
+ /**
* Constructs a new input stream which reads its data from the specified S3 object.
- *
+ *
* @param s3Client
* the S3 client to connect to Amazon S3.
* @param bucket
@@ -63,6 +72,8 @@ public class S3DataInputStream extends FSDataInputStream {
}
this.inputStream = s3o.getObjectContent();
+ this.position = 0;
+ this.marked = 0;
}
@@ -84,6 +95,7 @@ public class S3DataInputStream extends FSDataInputStream {
public void mark(final int readlimit) {
this.inputStream.mark(readlimit);
+ marked = readlimit;
}
@@ -97,21 +109,36 @@ public class S3DataInputStream extends FSDataInputStream {
@Override
public int read() throws IOException {
- return this.inputStream.read();
+ int read = this.inputStream.read();
+ if (read != -1) {
+ ++position;
+ }
+
+ return read;
}
@Override
public int read(final byte[] b) throws IOException {
- return this.inputStream.read(b);
+ int read = this.inputStream.read(b);
+ if (read > 0) {
+ position += read;
+ }
+
+ return read;
}
@Override
public int read(final byte[] b, final int off, final int len) throws IOException {
- return this.inputStream.read(b, off, len);
+ int read = this.inputStream.read(b, off, len);
+ if (read > 0) {
+ position += read;
+ }
+
+ return read;
}
@@ -119,12 +146,28 @@ public class S3DataInputStream extends FSDataInputStream {
public void reset() throws IOException {
this.inputStream.reset();
+ position = marked;
}
@Override
public void seek(final long desired) throws IOException {
- this.inputStream.skip(desired);
+ skip(desired);
+ }
+
+ @Override
+ public long skip(long n) throws IOException {
+ long skipped = this.inputStream.skip(n);
+ if (skipped > 0) {
+ position += skipped;
+ }
+
+ return skipped;
+ }
+
+ @Override
+ public long getPos() throws IOException {
+ return position;
}
}
[3/3] flink git commit: [FLINK-1081] FileMonitoringFunction fix for
proper handling of modification times
Posted by gy...@apache.org.
[FLINK-1081] FileMonitoringFunction fix for proper handling of modification times
Closes #226
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/9b63f269
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/9b63f269
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/9b63f269
Branch: refs/heads/master
Commit: 9b63f269e7ced1c9cb58bffe0091eb3d86ac6624
Parents: bf5e39a
Author: Gyula Fora <gy...@apache.org>
Authored: Sun Jan 25 20:20:40 2015 +0100
Committer: Gyula Fora <gy...@apache.org>
Committed: Sun Jan 25 21:26:54 2015 +0100
----------------------------------------------------------------------
.../environment/StreamExecutionEnvironment.java | 9 ++---
.../function/source/FileMonitoringFunction.java | 36 ++++++++++++--------
.../api/scala/StreamExecutionEnvironment.scala | 15 ++++----
3 files changed, 35 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/9b63f269/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
index b7eeed2..e0cdd02 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
@@ -224,8 +224,8 @@ public abstract class StreamExecutionEnvironment {
* @param filePath
* The path of the file, as a URI (e.g.,
* "file:///some/local/file" or "hdfs://host:port/file/path/").
- * @param interval
- * The interval of file watching.
+ * @param intervalMillis
+ * The interval of file watching in milliseconds.
* @param watchType
* The watch type of file stream. When watchType is
* {@link WatchType.ONLY_NEW_FILES}, the system processes only
@@ -236,9 +236,10 @@ public abstract class StreamExecutionEnvironment {
*
* @return The DataStream containing the given directory.
*/
- public DataStream<String> readFileStream(String filePath, long interval, WatchType watchType) {
+ public DataStream<String> readFileStream(String filePath, long intervalMillis,
+ WatchType watchType) {
DataStream<Tuple3<String, Long, Long>> source = addSource(new FileMonitoringFunction(
- filePath, interval, watchType));
+ filePath, intervalMillis, watchType), null, "File Stream");
return source.flatMap(new FileReadFunction());
}
http://git-wip-us.apache.org/repos/asf/flink/blob/9b63f269/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/FileMonitoringFunction.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/FileMonitoringFunction.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/FileMonitoringFunction.java
index c223c53..05a2489 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/FileMonitoringFunction.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/FileMonitoringFunction.java
@@ -48,15 +48,14 @@ public class FileMonitoringFunction implements SourceFunction<Tuple3<String, Lon
private WatchType watchType;
private FileSystem fileSystem;
- private long lastModificationTime;
private Map<String, Long> offsetOfFiles;
+ private Map<String, Long> modificationTimes;
public FileMonitoringFunction(String path, long interval, WatchType watchType) {
this.path = path;
this.interval = interval;
this.watchType = watchType;
-
- this.lastModificationTime = System.currentTimeMillis();
+ this.modificationTimes = new HashMap<String, Long>();
this.offsetOfFiles = new HashMap<String, Long>();
}
@@ -67,7 +66,8 @@ public class FileMonitoringFunction implements SourceFunction<Tuple3<String, Lon
while (true) {
List<String> files = listNewFiles();
for (String filePath : files) {
- if (watchType == WatchType.ONLY_NEW_FILES || watchType == WatchType.REPROCESS_WITH_APPENDED) {
+ if (watchType == WatchType.ONLY_NEW_FILES
+ || watchType == WatchType.REPROCESS_WITH_APPENDED) {
collector.collect(new Tuple3<String, Long, Long>(filePath, 0L, -1L));
offsetOfFiles.put(filePath, -1L);
} else if (watchType == WatchType.PROCESS_ONLY_APPENDED) {
@@ -90,28 +90,34 @@ public class FileMonitoringFunction implements SourceFunction<Tuple3<String, Lon
private List<String> listNewFiles() throws IOException {
List<String> files = new ArrayList<String>();
+
FileStatus[] statuses = fileSystem.listStatus(new Path(path));
for (FileStatus status : statuses) {
Path filePath = status.getPath();
+ String fileName = filePath.getName();
long modificationTime = status.getModificationTime();
- if (!isFiltered(filePath, modificationTime)) {
+ if (!isFiltered(fileName, modificationTime)) {
files.add(filePath.toString());
+ modificationTimes.put(fileName, modificationTime);
}
}
-
- lastModificationTime = System.currentTimeMillis();
-
return files;
}
- private boolean isFiltered(Path path, long modificationTime) {
- String filename = path.getName();
-
- return lastModificationTime > modificationTime // not modified file
- || (watchType == WatchType.ONLY_NEW_FILES && offsetOfFiles.containsKey(path.toString())) // modified file but already processed
- || filename.startsWith(".") // hidden file
- || filename.contains("_COPYING_"); // currently copying file
+ private boolean isFiltered(String fileName, long modificationTime) {
+
+ if ((watchType == WatchType.ONLY_NEW_FILES && modificationTimes.containsKey(fileName))
+ || fileName.startsWith(".") || fileName.contains("_COPYING_")) {
+ return true;
+ } else {
+ Long lastModification = modificationTimes.get(fileName);
+ if (lastModification == null) {
+ return false;
+ } else {
+ return lastModification >= modificationTime;
+ }
+ }
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/9b63f269/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamExecutionEnvironment.scala
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamExecutionEnvironment.scala b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamExecutionEnvironment.scala
index b4565c7..e0f50f8 100644
--- a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamExecutionEnvironment.scala
+++ b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamExecutionEnvironment.scala
@@ -25,6 +25,7 @@ import org.apache.flink.streaming.api.environment.{StreamExecutionEnvironment =>
import org.apache.flink.streaming.api.function.source.{ FromElementsFunction, SourceFunction }
import org.apache.flink.util.Collector
import org.apache.flink.api.scala.ClosureCleaner
+import org.apache.flink.streaming.api.function.source.FileMonitoringFunction.WatchType
class StreamExecutionEnvironment(javaEnv: JavaEnv) {
@@ -81,14 +82,16 @@ class StreamExecutionEnvironment(javaEnv: JavaEnv) {
javaEnv.readTextFile(filePath)
/**
- * Creates a DataStream that represents the Strings produced by reading the
- * given file line wise multiple times(infinite). The file will be read with
- * the system's default character set. This functionality can be used for
- * testing a topology.
+ * Creates a DataStream that contains the contents of file created while
+ * system watches the given path. The file will be read with the system's
+ * default character set. The user can check the monitoring interval in milliseconds,
+ * and the the way file modifications are handled. By default it checks for only new files
+ * every 100 milliseconds.
*
*/
- def readTextStream(StreamPath: String): DataStream[String] =
- javaEnv.readTextStream(StreamPath)
+ def readFileStream(StreamPath: String, intervalMillis: Long = 100, watchType: WatchType =
+ WatchType.ONLY_NEW_FILES): DataStream[String] =
+ javaEnv.readFileStream(StreamPath, intervalMillis, watchType)
/**
* Creates a new DataStream that contains the strings received infinitely
[2/3] flink git commit: [FLINK-1081] Add HDFS file-stream source for
streaming
Posted by gy...@apache.org.
[FLINK-1081] Add HDFS file-stream source for streaming
Conflicts:
flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/bf5e39a8
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/bf5e39a8
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/bf5e39a8
Branch: refs/heads/master
Commit: bf5e39a8dea1e52de8f8eb92ded3d128ce66d572
Parents: 7aa9a50
Author: Chiwan Park <ch...@icloud.com>
Authored: Fri Dec 12 02:49:01 2014 +0900
Committer: Gyula Fora <gy...@apache.org>
Committed: Sun Jan 25 20:21:22 2015 +0100
----------------------------------------------------------------------
.../environment/StreamExecutionEnvironment.java | 47 ++++----
.../function/source/FileMonitoringFunction.java | 117 +++++++++++++++++++
.../api/function/source/FileReadFunction.java | 51 ++++++++
.../api/function/source/FileStreamFunction.java | 49 --------
4 files changed, 191 insertions(+), 73 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/bf5e39a8/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
index 8a9be93..b7eeed2 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
@@ -29,6 +29,7 @@ import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.java.ExecutionEnvironment;
import org.apache.flink.api.java.io.TextInputFormat;
+import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.api.java.typeutils.TypeExtractor;
import org.apache.flink.client.program.Client;
import org.apache.flink.client.program.ContextEnvironment;
@@ -36,8 +37,10 @@ import org.apache.flink.core.fs.Path;
import org.apache.flink.streaming.api.StreamGraph;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
+import org.apache.flink.streaming.api.function.source.FileMonitoringFunction;
+import org.apache.flink.streaming.api.function.source.FileMonitoringFunction.WatchType;
+import org.apache.flink.streaming.api.function.source.FileReadFunction;
import org.apache.flink.streaming.api.function.source.FileSourceFunction;
-import org.apache.flink.streaming.api.function.source.FileStreamFunction;
import org.apache.flink.streaming.api.function.source.FromElementsFunction;
import org.apache.flink.streaming.api.function.source.GenSequenceFunction;
import org.apache.flink.streaming.api.function.source.GenericSourceFunction;
@@ -214,34 +217,30 @@ public abstract class StreamExecutionEnvironment {
}
/**
- * Creates a DataStream that represents the Strings produced by reading the
- * given file line wise multiple times(infinite). The file will be read with
- * the system's default character set. This functionality can be used for
- * testing a topology.
+ * Creates a DataStream that contains the contents of file created while
+ * system watches the given path. The file will be read with the system's
+ * default character set.
*
* @param filePath
* The path of the file, as a URI (e.g.,
- * "file:///some/local/file" or "hdfs://host:port/file/path").
- * @return The DataStream representing the text file.
+ * "file:///some/local/file" or "hdfs://host:port/file/path/").
+ * @param interval
+ * The interval of file watching.
+ * @param watchType
+ * The watch type of file stream. When watchType is
+ * {@link WatchType.ONLY_NEW_FILES}, the system processes only
+ * new files. {@link WatchType.REPROCESS_WITH_APPENDED} means
+ * that the system re-processes all contents of appended file.
+ * {@link WatchType.PROCESS_ONLY_APPENDED} means that the system
+ * processes only appended contents of files.
+ *
+ * @return The DataStream containing the given directory.
*/
- public DataStreamSource<String> readTextStream(String filePath) {
- checkIfFileExists(filePath);
- return addSource(new FileStreamFunction(filePath), null, "textStream");
- }
+ public DataStream<String> readFileStream(String filePath, long interval, WatchType watchType) {
+ DataStream<Tuple3<String, Long, Long>> source = addSource(new FileMonitoringFunction(
+ filePath, interval, watchType));
- private static void checkIfFileExists(String filePath) {
- File file = new File(filePath);
- if (!file.exists()) {
- throw new IllegalArgumentException("File not found: " + filePath);
- }
-
- if (!file.canRead()) {
- throw new IllegalArgumentException("Cannot read file: " + filePath);
- }
-
- if (file.isDirectory()) {
- throw new IllegalArgumentException("Given path is a directory: " + filePath);
- }
+ return source.flatMap(new FileReadFunction());
}
/**
http://git-wip-us.apache.org/repos/asf/flink/blob/bf5e39a8/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/FileMonitoringFunction.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/FileMonitoringFunction.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/FileMonitoringFunction.java
new file mode 100644
index 0000000..c223c53
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/FileMonitoringFunction.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.function.source;
+
+import java.io.IOException;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.flink.api.java.tuple.Tuple3;
+import org.apache.flink.core.fs.FileStatus;
+import org.apache.flink.core.fs.FileSystem;
+import org.apache.flink.core.fs.Path;
+import org.apache.flink.util.Collector;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class FileMonitoringFunction implements SourceFunction<Tuple3<String, Long, Long>> {
+ private static final long serialVersionUID = 1L;
+
+ private static final Logger LOG = LoggerFactory.getLogger(FileMonitoringFunction.class);
+
+ public enum WatchType {
+ ONLY_NEW_FILES, // Only new files will be processed.
+ REPROCESS_WITH_APPENDED, // When some files are appended, all contents of the files will be processed.
+ PROCESS_ONLY_APPENDED // When some files are appended, only appended contents will be processed.
+ }
+
+ private String path;
+ private long interval;
+ private WatchType watchType;
+
+ private FileSystem fileSystem;
+ private long lastModificationTime;
+ private Map<String, Long> offsetOfFiles;
+
+ public FileMonitoringFunction(String path, long interval, WatchType watchType) {
+ this.path = path;
+ this.interval = interval;
+ this.watchType = watchType;
+
+ this.lastModificationTime = System.currentTimeMillis();
+ this.offsetOfFiles = new HashMap<String, Long>();
+ }
+
+ @Override
+ public void invoke(Collector<Tuple3<String, Long, Long>> collector) throws Exception {
+ fileSystem = FileSystem.get(new URI(path));
+
+ while (true) {
+ List<String> files = listNewFiles();
+ for (String filePath : files) {
+ if (watchType == WatchType.ONLY_NEW_FILES || watchType == WatchType.REPROCESS_WITH_APPENDED) {
+ collector.collect(new Tuple3<String, Long, Long>(filePath, 0L, -1L));
+ offsetOfFiles.put(filePath, -1L);
+ } else if (watchType == WatchType.PROCESS_ONLY_APPENDED) {
+ long offset = 0;
+ long fileSize = fileSystem.getFileStatus(new Path(filePath)).getLen();
+ if (offsetOfFiles.containsKey(filePath)) {
+ offset = offsetOfFiles.get(filePath);
+ }
+
+ collector.collect(new Tuple3<String, Long, Long>(filePath, offset, fileSize));
+ offsetOfFiles.put(filePath, fileSize);
+
+ LOG.info("File processed: {}, {}, {}", filePath, offset, fileSize);
+ }
+ }
+
+ Thread.sleep(interval);
+ }
+ }
+
+ private List<String> listNewFiles() throws IOException {
+ List<String> files = new ArrayList<String>();
+ FileStatus[] statuses = fileSystem.listStatus(new Path(path));
+
+ for (FileStatus status : statuses) {
+ Path filePath = status.getPath();
+ long modificationTime = status.getModificationTime();
+
+ if (!isFiltered(filePath, modificationTime)) {
+ files.add(filePath.toString());
+ }
+ }
+
+ lastModificationTime = System.currentTimeMillis();
+
+ return files;
+ }
+
+ private boolean isFiltered(Path path, long modificationTime) {
+ String filename = path.getName();
+
+ return lastModificationTime > modificationTime // not modified file
+ || (watchType == WatchType.ONLY_NEW_FILES && offsetOfFiles.containsKey(path.toString())) // modified file but already processed
+ || filename.startsWith(".") // hidden file
+ || filename.contains("_COPYING_"); // currently copying file
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/bf5e39a8/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/FileReadFunction.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/FileReadFunction.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/FileReadFunction.java
new file mode 100644
index 0000000..0882d9e
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/FileReadFunction.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.function.source;
+
+import org.apache.flink.api.common.functions.FlatMapFunction;
+import org.apache.flink.api.java.tuple.Tuple3;
+import org.apache.flink.core.fs.FSDataInputStream;
+import org.apache.flink.core.fs.FileSystem;
+import org.apache.flink.core.fs.Path;
+import org.apache.flink.util.Collector;
+
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.net.URI;
+
+public class FileReadFunction implements FlatMapFunction<Tuple3<String, Long, Long>, String> {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public void flatMap(Tuple3<String, Long, Long> value, Collector<String> out) throws Exception {
+ FSDataInputStream stream = FileSystem.get(new URI(value.f0)).open(new Path(value.f0));
+ stream.seek(value.f1);
+
+ BufferedReader reader = new BufferedReader(new InputStreamReader(stream));
+ String line;
+
+ try {
+ while ((line = reader.readLine()) != null && (value.f2 == -1L || stream.getPos() <= value.f2)) {
+ out.collect(line);
+ }
+ } finally {
+ reader.close();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/bf5e39a8/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/FileStreamFunction.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/FileStreamFunction.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/FileStreamFunction.java
deleted file mode 100644
index 7371ac9..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/FileStreamFunction.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.function.source;
-
-import java.io.BufferedReader;
-import java.io.FileReader;
-import java.io.IOException;
-
-import org.apache.flink.util.Collector;
-
-public class FileStreamFunction implements SourceFunction<String> {
- private static final long serialVersionUID = 1L;
-
- private final String path;
-
- public FileStreamFunction(String path) {
- this.path = path;
- }
-
- @Override
- public void invoke(Collector<String> collector) throws IOException {
- while (true) {
- BufferedReader br = new BufferedReader(new FileReader(path));
- String line = br.readLine();
- while (line != null) {
- if (!line.equals("")) {
- collector.collect(line);
- }
- line = br.readLine();
- }
- br.close();
- }
- }
-}