You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ch...@apache.org on 2021/09/09 05:25:22 UTC
[flink] branch master updated (16ab8b4 -> 69fefdd)
This is an automated email from the ASF dual-hosted git repository.
chesnay pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git.
from 16ab8b4 Revert "[FLINK-24041][connectors] First draft of API, Add DynamoDB sink to verify API, Add Firehose sink, Refactor to use FLIP-143: Unified Sink API, Increase robustness for flush, Add license header, Update JavaDoc comments"
new a5e83af [FLINK-24193][tests] Add ClassLoaderExtension
new 8081dfb [FLINK-24193][rest][tests] Migrate tests to ClassLoaderExtension
new 69fefdd [hotfix][rest][tests] Cleanup
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../webmonitor/utils/WebFrontendBootstrapTest.java | 47 +++++---
.../runtime/rest/RestExternalHandlersITCase.java | 40 +++++--
...e.io.network.netty.InboundChannelHandlerFactory | 17 ---
....io.network.netty.OutboundChannelHandlerFactory | 17 ---
.../extensions/ContextClassLoaderExtension.java | 129 +++++++++++++++++++++
5 files changed, 188 insertions(+), 62 deletions(-)
delete mode 100644 flink-runtime/src/test/resources/META-INF/services/org.apache.flink.runtime.io.network.netty.InboundChannelHandlerFactory
delete mode 100644 flink-runtime/src/test/resources/META-INF/services/org.apache.flink.runtime.io.network.netty.OutboundChannelHandlerFactory
create mode 100644 flink-test-utils-parent/flink-test-utils-junit/src/main/java/org/apache/flink/testutils/junit/extensions/ContextClassLoaderExtension.java
[flink] 01/03: [FLINK-24193][tests] Add ClassLoaderExtension
Posted by ch...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
chesnay pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit a5e83afb510aafa05aac31413e2b59704c374aa3
Author: Chesnay Schepler <ch...@apache.org>
AuthorDate: Tue Sep 7 15:16:48 2021 +0200
[FLINK-24193][tests] Add ClassLoaderExtension
---
.../extensions/ContextClassLoaderExtension.java | 129 +++++++++++++++++++++
1 file changed, 129 insertions(+)
diff --git a/flink-test-utils-parent/flink-test-utils-junit/src/main/java/org/apache/flink/testutils/junit/extensions/ContextClassLoaderExtension.java b/flink-test-utils-parent/flink-test-utils-junit/src/main/java/org/apache/flink/testutils/junit/extensions/ContextClassLoaderExtension.java
new file mode 100644
index 0000000..168cae6
--- /dev/null
+++ b/flink-test-utils-parent/flink-test-utils-junit/src/main/java/org/apache/flink/testutils/junit/extensions/ContextClassLoaderExtension.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.testutils.junit.extensions;
+
+import org.junit.jupiter.api.extension.AfterAllCallback;
+import org.junit.jupiter.api.extension.BeforeAllCallback;
+import org.junit.jupiter.api.extension.ExtensionContext;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.IOException;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+import java.util.function.Function;
+
+/** JUnit extension to customize the classloader that a test is run with. */
+public class ContextClassLoaderExtension implements BeforeAllCallback, AfterAllCallback {
+
+ private final TemporaryFolder temporaryFolder = new TemporaryFolder();
+ private final Function<TemporaryFolder, URLClassLoader> temporaryClassLoaderFactory;
+
+ private ClassLoader originalClassLoader;
+ private URLClassLoader temporaryClassLoader;
+
+ private ContextClassLoaderExtension(
+ Function<TemporaryFolder, URLClassLoader> temporaryClassLoaderFactory) {
+ this.temporaryClassLoaderFactory = temporaryClassLoaderFactory;
+ }
+
+ @Override
+ public void beforeAll(ExtensionContext context) throws Exception {
+ temporaryFolder.create();
+ originalClassLoader = Thread.currentThread().getContextClassLoader();
+ temporaryClassLoader = temporaryClassLoaderFactory.apply(temporaryFolder);
+ Thread.currentThread().setContextClassLoader(temporaryClassLoader);
+ }
+
+ @Override
+ public void afterAll(ExtensionContext context) throws Exception {
+ Thread.currentThread().setContextClassLoader(originalClassLoader);
+ temporaryClassLoader.close();
+ temporaryFolder.delete();
+ }
+
+ public static ContextClassLoaderExtensionBuilder builder() {
+ return new ContextClassLoaderExtensionBuilder();
+ }
+
+ /** Builder for {@link ContextClassLoaderExtension}. */
+ public static class ContextClassLoaderExtensionBuilder {
+
+ private final Collection<ServiceEntry> serviceEntries = new ArrayList<>();
+
+ public ContextClassLoaderExtensionBuilder withServiceEntry(
+ Class<?> serviceClass, String... serviceImplementations) {
+ serviceEntries.add(new ServiceEntry(serviceClass, serviceImplementations));
+ return this;
+ }
+
+ public ContextClassLoaderExtension build() {
+ final Function<TemporaryFolder, URLClassLoader> factory =
+ temporaryFolder -> setupClassLoader(temporaryFolder, serviceEntries);
+
+ return new ContextClassLoaderExtension(factory);
+ }
+
+ private ContextClassLoaderExtensionBuilder() {}
+ }
+
+ private static URLClassLoader setupClassLoader(
+ TemporaryFolder temporaryFolder, Collection<ServiceEntry> serviceEntries) {
+ final Path root = temporaryFolder.getRoot().toPath();
+ try {
+ writeServiceEntries(root, serviceEntries);
+ final URL url = temporaryFolder.getRoot().toURI().toURL();
+ return new URLClassLoader(
+ new URL[] {url}, ContextClassLoaderExtension.class.getClassLoader());
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private static void writeServiceEntries(Path tmpDir, Collection<ServiceEntry> serviceEntries)
+ throws IOException {
+ for (ServiceEntry serviceEntry : serviceEntries) {
+ final Path path =
+ tmpDir.resolve(
+ Paths.get(
+ "META-INF",
+ "services",
+ serviceEntry.serviceClass.getCanonicalName()));
+
+ Files.createDirectories(path.getParent());
+ Files.write(path, serviceEntry.serviceImplementations, StandardOpenOption.CREATE);
+ }
+ }
+
+ private static class ServiceEntry {
+ private final Class<?> serviceClass;
+ private final List<String> serviceImplementations;
+
+ public ServiceEntry(Class<?> serviceClass, String... serviceImplementations) {
+ this.serviceClass = serviceClass;
+ this.serviceImplementations = Arrays.asList(serviceImplementations);
+ }
+ }
+}
[flink] 02/03: [FLINK-24193][rest][tests] Migrate tests to
ClassLoaderExtension
Posted by ch...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
chesnay pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 8081dfbcc2c63dfda385a68f4615ddf5d51ccc26
Author: Chesnay Schepler <ch...@apache.org>
AuthorDate: Tue Sep 7 15:18:51 2021 +0200
[FLINK-24193][rest][tests] Migrate tests to ClassLoaderExtension
---
.../webmonitor/utils/WebFrontendBootstrapTest.java | 42 ++++++++++++++--------
.../runtime/rest/RestExternalHandlersITCase.java | 40 +++++++++++++++------
...e.io.network.netty.InboundChannelHandlerFactory | 17 ---------
....io.network.netty.OutboundChannelHandlerFactory | 17 ---------
4 files changed, 56 insertions(+), 60 deletions(-)
diff --git a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/utils/WebFrontendBootstrapTest.java b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/utils/WebFrontendBootstrapTest.java
index f58093f..42c2491 100644
--- a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/utils/WebFrontendBootstrapTest.java
+++ b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/utils/WebFrontendBootstrapTest.java
@@ -20,31 +20,43 @@ package org.apache.flink.runtime.webmonitor.utils;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.configuration.Configuration;
+import org.apache.flink.runtime.io.network.netty.InboundChannelHandlerFactory;
import org.apache.flink.runtime.io.network.netty.Prio0InboundChannelHandlerFactory;
import org.apache.flink.runtime.io.network.netty.Prio1InboundChannelHandlerFactory;
import org.apache.flink.runtime.rest.handler.router.Router;
import org.apache.flink.runtime.webmonitor.history.HistoryServerStaticFileServerHandler;
import org.apache.flink.runtime.webmonitor.history.HistoryServerTest;
+import org.apache.flink.testutils.junit.extensions.ContextClassLoaderExtension;
-import org.junit.Assert;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.Extension;
+import org.junit.jupiter.api.extension.RegisterExtension;
+import org.junit.jupiter.api.io.TempDir;
import org.slf4j.LoggerFactory;
-import java.io.File;
+import java.nio.file.Files;
+import java.nio.file.Path;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
/** Tests for the WebFrontendBootstrap. */
public class WebFrontendBootstrapTest {
- @Rule public TemporaryFolder tmp = new TemporaryFolder();
+ @RegisterExtension
+ static final Extension CONTEXT_CLASS_LOADER_EXTENSION =
+ ContextClassLoaderExtension.builder()
+ .withServiceEntry(
+ InboundChannelHandlerFactory.class,
+ Prio0InboundChannelHandlerFactory.class.getCanonicalName(),
+ Prio1InboundChannelHandlerFactory.class.getCanonicalName())
+ .build();
+
+ @TempDir Path tmp;
@Test
- public void testHandlersMustBeLoaded() throws Exception {
- File webDir = tmp.newFolder("webDir");
+ void testHandlersMustBeLoaded() throws Exception {
+ Path webDir = Files.createDirectories(tmp.resolve("webDir"));
Configuration configuration = new Configuration();
configuration.setString(
Prio0InboundChannelHandlerFactory.REDIRECT_FROM_URL, "/nonExisting");
@@ -55,7 +67,7 @@ public class WebFrontendBootstrapTest {
new WebFrontendBootstrap(
router,
LoggerFactory.getLogger(WebFrontendBootstrapTest.class),
- tmp.newFolder("uploadDir"),
+ Files.createDirectories(webDir.resolve("uploadDir")).toFile(),
null,
"localhost",
0,
@@ -73,13 +85,13 @@ public class WebFrontendBootstrapTest {
try {
Tuple2<Integer, String> index =
HistoryServerTest.getFromHTTP("http://localhost:" + port + "/index.html");
- Assert.assertEquals(index.f0.intValue(), 200);
- Assert.assertTrue(index.f1.contains("Apache Flink Web Dashboard"));
+ assertEquals(index.f0.intValue(), 200);
+ assertTrue(index.f1.contains("Apache Flink Web Dashboard"));
Tuple2<Integer, String> index2 =
HistoryServerTest.getFromHTTP("http://localhost:" + port + "/nonExisting");
- Assert.assertEquals(index2.f0.intValue(), 200);
- Assert.assertEquals(index, index2);
+ assertEquals(index2.f0.intValue(), 200);
+ assertEquals(index, index2);
} finally {
webUI.shutdown();
}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/RestExternalHandlersITCase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/RestExternalHandlersITCase.java
index bc8dd31..9fd6dcc 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/RestExternalHandlersITCase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/RestExternalHandlersITCase.java
@@ -21,6 +21,8 @@ package org.apache.flink.runtime.rest;
import org.apache.flink.api.common.time.Time;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.configuration.RestOptions;
+import org.apache.flink.runtime.io.network.netty.InboundChannelHandlerFactory;
+import org.apache.flink.runtime.io.network.netty.OutboundChannelHandlerFactory;
import org.apache.flink.runtime.io.network.netty.Prio0InboundChannelHandlerFactory;
import org.apache.flink.runtime.io.network.netty.Prio1InboundChannelHandlerFactory;
import org.apache.flink.runtime.rest.messages.EmptyMessageParameters;
@@ -29,23 +31,26 @@ import org.apache.flink.runtime.rest.messages.RequestBody;
import org.apache.flink.runtime.rest.messages.ResponseBody;
import org.apache.flink.runtime.rest.util.TestRestServerEndpoint;
import org.apache.flink.runtime.testutils.TestingUtils;
+import org.apache.flink.testutils.junit.extensions.ContextClassLoaderExtension;
import org.apache.flink.util.ConfigurationException;
import org.apache.flink.util.TestLogger;
import org.apache.flink.shaded.netty4.io.netty.handler.codec.http.HttpResponseStatus;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.Extension;
+import org.junit.jupiter.api.extension.RegisterExtension;
import java.io.IOException;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.util.concurrent.CompletableFuture;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
/** IT cases for {@link RestClient} and {@link RestServerEndpoint}. */
public class RestExternalHandlersITCase extends TestLogger {
@@ -59,6 +64,19 @@ public class RestExternalHandlersITCase extends TestLogger {
private RestClient restClient;
private InetSocketAddress serverAddress;
+ @RegisterExtension
+ static final Extension CONTEXT_CLASS_LOADER_EXTENSION =
+ ContextClassLoaderExtension.builder()
+ .withServiceEntry(
+ InboundChannelHandlerFactory.class,
+ Prio0InboundChannelHandlerFactory.class.getCanonicalName(),
+ Prio1InboundChannelHandlerFactory.class.getCanonicalName())
+ .withServiceEntry(
+ OutboundChannelHandlerFactory.class,
+ Prio0OutboundChannelHandlerFactory.class.getCanonicalName(),
+ Prio1OutboundChannelHandlerFactory.class.getCanonicalName())
+ .build();
+
private final Configuration config;
public RestExternalHandlersITCase() {
@@ -78,15 +96,15 @@ public class RestExternalHandlersITCase extends TestLogger {
return config;
}
- @Before
- public void setup() throws Exception {
+ @BeforeEach
+ private void setup() throws Exception {
serverEndpoint = TestRestServerEndpoint.builder(config).buildAndStart();
restClient = new TestRestClient(config);
serverAddress = serverEndpoint.getServerAddress();
}
- @After
- public void teardown() throws Exception {
+ @AfterEach
+ private void teardown() throws Exception {
if (restClient != null) {
restClient.shutdown(timeout);
restClient = null;
@@ -99,7 +117,7 @@ public class RestExternalHandlersITCase extends TestLogger {
}
@Test
- public void testHandlersMustBeLoaded() throws Exception {
+ void testHandlersMustBeLoaded() throws Exception {
assertEquals(serverEndpoint.inboundChannelHandlerFactories.size(), 2);
assertTrue(
serverEndpoint.inboundChannelHandlerFactories.get(0)
diff --git a/flink-runtime/src/test/resources/META-INF/services/org.apache.flink.runtime.io.network.netty.InboundChannelHandlerFactory b/flink-runtime/src/test/resources/META-INF/services/org.apache.flink.runtime.io.network.netty.InboundChannelHandlerFactory
deleted file mode 100644
index 09b801c..0000000
--- a/flink-runtime/src/test/resources/META-INF/services/org.apache.flink.runtime.io.network.netty.InboundChannelHandlerFactory
+++ /dev/null
@@ -1,17 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-org.apache.flink.runtime.io.network.netty.Prio0InboundChannelHandlerFactory
-org.apache.flink.runtime.io.network.netty.Prio1InboundChannelHandlerFactory
diff --git a/flink-runtime/src/test/resources/META-INF/services/org.apache.flink.runtime.io.network.netty.OutboundChannelHandlerFactory b/flink-runtime/src/test/resources/META-INF/services/org.apache.flink.runtime.io.network.netty.OutboundChannelHandlerFactory
deleted file mode 100644
index ea7be93..0000000
--- a/flink-runtime/src/test/resources/META-INF/services/org.apache.flink.runtime.io.network.netty.OutboundChannelHandlerFactory
+++ /dev/null
@@ -1,17 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-org.apache.flink.runtime.rest.Prio0OutboundChannelHandlerFactory
-org.apache.flink.runtime.rest.Prio1OutboundChannelHandlerFactory
[flink] 03/03: [hotfix][rest][tests] Cleanup
Posted by ch...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
chesnay pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 69fefdd49e725dd102f6a24774686bc058ff74e0
Author: Chesnay Schepler <ch...@apache.org>
AuthorDate: Wed Sep 8 10:12:53 2021 +0200
[hotfix][rest][tests] Cleanup
This reverts commit 69de4b66b27f4b83e3b5fff59dfc793209c72640.
---
.../flink/runtime/webmonitor/utils/WebFrontendBootstrapTest.java | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/utils/WebFrontendBootstrapTest.java b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/utils/WebFrontendBootstrapTest.java
index 42c2491..5deba65 100644
--- a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/utils/WebFrontendBootstrapTest.java
+++ b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/utils/WebFrontendBootstrapTest.java
@@ -61,8 +61,9 @@ public class WebFrontendBootstrapTest {
configuration.setString(
Prio0InboundChannelHandlerFactory.REDIRECT_FROM_URL, "/nonExisting");
configuration.setString(Prio0InboundChannelHandlerFactory.REDIRECT_TO_URL, "/index.html");
- Router router =
- new Router().addGet("/:*", new HistoryServerStaticFileServerHandler(webDir));
+ Router<?> router =
+ new Router<>()
+ .addGet("/:*", new HistoryServerStaticFileServerHandler(webDir.toFile()));
WebFrontendBootstrap webUI =
new WebFrontendBootstrap(
router,