You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by pn...@apache.org on 2019/06/05 08:03:06 UTC

[flink] 02/09: [FLINK-12556][e2e] Add read-only test FileSystem for end-to-end tests

This is an automated email from the ASF dual-hosted git repository.

pnowojski pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git

commit 1c916a47322e04b42f8521cbdb833f8cc2147ee2
Author: Aleksey Pak <al...@ververica.com>
AuthorDate: Mon May 20 11:32:42 2019 +0200

    [FLINK-12556][e2e] Add read-only test FileSystem for end-to-end tests
---
 flink-end-to-end-tests/flink-plugins-test/pom.xml  |  74 ++++++++++
 .../org/apache/flink/fs/dummy/DummyFSFactory.java  |  51 +++++++
 .../apache/flink/fs/dummy/DummyFSFileStatus.java   |  67 +++++++++
 .../apache/flink/fs/dummy/DummyFSFileSystem.java   | 162 +++++++++++++++++++++
 .../apache/flink/fs/dummy/DummyFSInputStream.java  |  51 +++++++
 .../org.apache.flink.core.fs.FileSystemFactory     |  16 ++
 flink-end-to-end-tests/pom.xml                     |   1 +
 7 files changed, 422 insertions(+)

diff --git a/flink-end-to-end-tests/flink-plugins-test/pom.xml b/flink-end-to-end-tests/flink-plugins-test/pom.xml
new file mode 100644
index 0000000..1c68fbb
--- /dev/null
+++ b/flink-end-to-end-tests/flink-plugins-test/pom.xml
@@ -0,0 +1,74 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+		 xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+		 xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+	<parent>
+		<artifactId>flink-end-to-end-tests</artifactId>
+		<groupId>org.apache.flink</groupId>
+		<version>1.9-SNAPSHOT</version>
+	</parent>
+	<modelVersion>4.0.0</modelVersion>
+
+	<artifactId>flink-plugins-test</artifactId>
+
+	<packaging>jar</packaging>
+
+	<dependencies>
+		<dependency>
+			<groupId>org.apache.flink</groupId>
+			<artifactId>flink-core</artifactId>
+			<version>${project.version}</version>
+			<scope>provided</scope>
+		</dependency>
+	</dependencies>
+
+	<build>
+		<plugins>
+
+			<!-- this is merely an intermediate build artifact and should not be -->
+			<!-- deployed to maven central                                       -->
+			<plugin>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-deploy-plugin</artifactId>
+				<configuration>
+					<skip>true</skip>
+				</configuration>
+			</plugin>
+
+			<plugin>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-shade-plugin</artifactId>
+				<executions>
+					<execution>
+						<id>shade-flink</id>
+						<phase>package</phase>
+						<goals>
+							<goal>shade</goal>
+						</goals>
+						<configuration>
+							<finalName>flink-dummy-fs</finalName>
+						</configuration>
+					</execution>
+				</executions>
+			</plugin>
+		</plugins>
+	</build>
+</project>
diff --git a/flink-end-to-end-tests/flink-plugins-test/src/main/java/org/apache/flink/fs/dummy/DummyFSFactory.java b/flink-end-to-end-tests/flink-plugins-test/src/main/java/org/apache/flink/fs/dummy/DummyFSFactory.java
new file mode 100644
index 0000000..35187966
--- /dev/null
+++ b/flink-end-to-end-tests/flink-plugins-test/src/main/java/org/apache/flink/fs/dummy/DummyFSFactory.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.fs.dummy;
+
+import org.apache.flink.core.fs.FileSystem;
+import org.apache.flink.core.fs.FileSystemFactory;
+
+import java.io.IOException;
+import java.net.URI;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Factory of dummy FileSystem. See documentation of {@link DummyFSFileSystem}.
+ */
+public class DummyFSFactory implements FileSystemFactory {
+
+	private final FileSystem fileSystem = new DummyFSFileSystem(getData());
+
+	@Override
+	public String getScheme() {
+		return DummyFSFileSystem.FS_URI.getScheme();
+	}
+
+	@Override
+	public FileSystem create(URI fsUri) throws IOException {
+		return fileSystem;
+	}
+
+	private static Map<String, String> getData() {
+		Map<String, String> data = new HashMap<>();
+		data.put("/words", "Hello World how are you, my dear dear world\n");
+		return data;
+	}
+}
diff --git a/flink-end-to-end-tests/flink-plugins-test/src/main/java/org/apache/flink/fs/dummy/DummyFSFileStatus.java b/flink-end-to-end-tests/flink-plugins-test/src/main/java/org/apache/flink/fs/dummy/DummyFSFileStatus.java
new file mode 100644
index 0000000..3c6037d
--- /dev/null
+++ b/flink-end-to-end-tests/flink-plugins-test/src/main/java/org/apache/flink/fs/dummy/DummyFSFileStatus.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.fs.dummy;
+
+import org.apache.flink.core.fs.FileStatus;
+import org.apache.flink.core.fs.Path;
+
+class DummyFSFileStatus implements FileStatus {
+	private final Path path;
+	private final int length;
+
+	DummyFSFileStatus(Path path, int length) {
+		this.path = path;
+		this.length = length;
+	}
+
+	@Override
+	public long getLen() {
+		return length;
+	}
+
+	@Override
+	public long getBlockSize() {
+		return length;
+	}
+
+	@Override
+	public short getReplication() {
+		return 0;
+	}
+
+	@Override
+	public long getModificationTime() {
+		return 0;
+	}
+
+	@Override
+	public long getAccessTime() {
+		return 0;
+	}
+
+	@Override
+	public boolean isDir() {
+		return false;
+	}
+
+	@Override
+	public Path getPath() {
+		return path;
+	}
+}
diff --git a/flink-end-to-end-tests/flink-plugins-test/src/main/java/org/apache/flink/fs/dummy/DummyFSFileSystem.java b/flink-end-to-end-tests/flink-plugins-test/src/main/java/org/apache/flink/fs/dummy/DummyFSFileSystem.java
new file mode 100644
index 0000000..ac11807
--- /dev/null
+++ b/flink-end-to-end-tests/flink-plugins-test/src/main/java/org/apache/flink/fs/dummy/DummyFSFileSystem.java
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.fs.dummy;
+
+import org.apache.flink.core.fs.BlockLocation;
+import org.apache.flink.core.fs.FSDataInputStream;
+import org.apache.flink.core.fs.FSDataOutputStream;
+import org.apache.flink.core.fs.FileStatus;
+import org.apache.flink.core.fs.FileSystem;
+import org.apache.flink.core.fs.FileSystemKind;
+import org.apache.flink.core.fs.Path;
+import org.apache.flink.core.fs.local.LocalBlockLocation;
+
+import javax.annotation.Nullable;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.URI;
+import java.nio.charset.Charset;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * A FileSystem implementation for integration testing purposes. Supports and serves read-only content from static
+ * key value map.
+ */
+class DummyFSFileSystem extends FileSystem {
+
+	static final URI FS_URI = URI.create("dummy:///");
+
+	private static final String HOSTNAME = "localhost";
+
+	private final URI workingDir;
+
+	private final URI homeDir;
+
+	private final Map<String, byte[]> contents;
+
+	DummyFSFileSystem(Map<String, String> contents) {
+		this.workingDir = new File(System.getProperty("user.dir")).toURI();
+		this.homeDir = new File(System.getProperty("user.home")).toURI();
+		this.contents = convertToByteArrayMap(contents);
+	}
+
+	// ------------------------------------------------------------------------
+
+	@Override
+	public URI getUri() {
+		return FS_URI;
+	}
+
+	@Override
+	public Path getWorkingDirectory() {
+		return new Path(workingDir);
+	}
+
+	@Override
+	public Path getHomeDirectory() {
+		return new Path(homeDir);
+	}
+
+	@Override
+	public boolean exists(Path f) throws IOException {
+		return getDataByPath(f) != null;
+	}
+
+	@Override
+	public FileStatus[] listStatus(final Path f) throws IOException {
+		byte[] data = getDataByPath(f);
+		if (data == null) {
+			return null;
+		}
+		return new FileStatus[] { new DummyFSFileStatus(f, data.length) };
+	}
+
+	@Override
+	public BlockLocation[] getFileBlockLocations(FileStatus file, long start, long len) throws IOException {
+		return new BlockLocation[] {
+			new LocalBlockLocation(HOSTNAME, file.getLen())
+		};
+	}
+
+	@Override
+	public FileStatus getFileStatus(Path f) throws IOException {
+		byte[] data = getDataByPath(f);
+		if (data == null) {
+			throw new FileNotFoundException("File " + f + " does not exist.");
+		}
+		return new DummyFSFileStatus(f, data.length);
+	}
+
+	@Override
+	public FSDataInputStream open(final Path f, final int bufferSize) throws IOException {
+		return open(f);
+	}
+
+	@Override
+	public FSDataInputStream open(final Path f) throws IOException {
+		return DummyFSInputStream.create(getDataByPath(f));
+	}
+
+	@Override
+	public boolean delete(final Path path, final boolean recursive) throws IOException {
+		throw new UnsupportedOperationException("Dummy FS doesn't support delete operation");
+	}
+
+	@Override
+	public boolean mkdirs(final Path path) throws IOException {
+		throw new UnsupportedOperationException("Dummy FS doesn't support mkdirs operation");
+	}
+
+	@Override
+	public FSDataOutputStream create(final Path path, final WriteMode overwrite) throws IOException {
+		throw new UnsupportedOperationException("Dummy FS doesn't support create operation");
+	}
+
+	@Override
+	public boolean rename(final Path src, final Path dst) throws IOException {
+		throw new UnsupportedOperationException("Dummy FS doesn't support rename operation");
+	}
+
+	@Override
+	public boolean isDistributedFS() {
+		return true;
+	}
+
+	@Override
+	public FileSystemKind getKind() {
+		return FileSystemKind.OBJECT_STORE;
+	}
+
+	@Nullable
+	private byte[] getDataByPath(Path path) {
+		return contents.get(path.toUri().getPath());
+	}
+
+	private static Map<String, byte[]> convertToByteArrayMap(Map<String, String> content) {
+		Map<String, byte[]> data = new HashMap<>();
+		Charset utf8 = Charset.forName("UTF-8");
+		content.entrySet().forEach(
+			entry -> data.put(entry.getKey(), entry.getValue().getBytes(utf8))
+		);
+		return data;
+	}
+}
diff --git a/flink-end-to-end-tests/flink-plugins-test/src/main/java/org/apache/flink/fs/dummy/DummyFSInputStream.java b/flink-end-to-end-tests/flink-plugins-test/src/main/java/org/apache/flink/fs/dummy/DummyFSInputStream.java
new file mode 100644
index 0000000..77b798d
--- /dev/null
+++ b/flink-end-to-end-tests/flink-plugins-test/src/main/java/org/apache/flink/fs/dummy/DummyFSInputStream.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.fs.dummy;
+
+import org.apache.flink.core.fs.FSDataInputStream;
+import org.apache.flink.core.memory.ByteArrayInputStreamWithPos;
+
+import java.io.IOException;
+
+class DummyFSInputStream extends FSDataInputStream {
+	private final ByteArrayInputStreamWithPos stream;
+
+	private DummyFSInputStream(ByteArrayInputStreamWithPos stream) {
+		this.stream = stream;
+	}
+
+	static DummyFSInputStream create(byte[] buffer) {
+		return new DummyFSInputStream(new ByteArrayInputStreamWithPos(buffer));
+	}
+
+	@Override
+	public void seek(long desired) throws IOException {
+		stream.setPosition((int) desired);
+	}
+
+	@Override
+	public long getPos() throws IOException {
+		return stream.getPosition();
+	}
+
+	@Override
+	public int read() throws IOException {
+		return stream.read();
+	}
+}
diff --git a/flink-end-to-end-tests/flink-plugins-test/src/main/resources/META-INF/services/org.apache.flink.core.fs.FileSystemFactory b/flink-end-to-end-tests/flink-plugins-test/src/main/resources/META-INF/services/org.apache.flink.core.fs.FileSystemFactory
new file mode 100644
index 0000000..1c5ec8c
--- /dev/null
+++ b/flink-end-to-end-tests/flink-plugins-test/src/main/resources/META-INF/services/org.apache.flink.core.fs.FileSystemFactory
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+org.apache.flink.fs.dummy.DummyFSFactory
diff --git a/flink-end-to-end-tests/pom.xml b/flink-end-to-end-tests/pom.xml
index 73ea7ef..0950e2f 100644
--- a/flink-end-to-end-tests/pom.xml
+++ b/flink-end-to-end-tests/pom.xml
@@ -65,6 +65,7 @@ under the License.
 		<module>flink-streaming-kafka-test</module>
 		<module>flink-streaming-kafka011-test</module>
 		<module>flink-streaming-kafka010-test</module>
+		<module>flink-plugins-test</module>
 	</modules>
 
 	<!-- See main pom.xml for explanation of profiles -->