You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@flink.apache.org by GitBox <gi...@apache.org> on 2021/06/30 09:43:11 UTC
[GitHub] [flink] zentol commented on a change in pull request #16286: [FLINK-21445] Refactors the PackagedProgramRetriever implementation and adds configuration to PackagedProgram
zentol commented on a change in pull request #16286:
URL: https://github.com/apache/flink/pull/16286#discussion_r661232763
##########
File path: flink-clients/src/test/java/org/apache/flink/client/deployment/application/JarManifestParserTest.java
##########
@@ -157,6 +158,24 @@ public void testFindOnlyEntryClassMultipleJarsWithSingleManifestEntry() throws I
is(equalTo(TestJob.class.getCanonicalName())));
}
+ @Test
+ public void testFindFirstManifestAttributeWithNoAttribute() throws IOException {
+ assertThat(
+ JarManifestParser.findFirstManifestAttribute(TestJob.getTestJobJar()).isPresent(),
+ is(false));
+ }
+
+ @Test
+ public void testFindFirstManifestAttributeWithAttributes() throws IOException {
+ Optional<String> optionalValue =
+ JarManifestParser.findFirstManifestAttribute(
+ TestJob.getTestJobJar(),
+ "some-attr",
Review comment:
what's this for?
##########
File path: flink-clients/src/main/java/org/apache/flink/client/deployment/application/FromClasspathEntryClassInformationProvider.java
##########
@@ -0,0 +1,207 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.client.deployment.application;
+
+import org.apache.flink.annotation.VisibleForTesting;
+import org.apache.flink.util.ExceptionUtils;
+import org.apache.flink.util.FileUtils;
+import org.apache.flink.util.FlinkException;
+import org.apache.flink.util.Preconditions;
+import org.apache.flink.util.StringUtils;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URL;
+import java.util.Arrays;
+import java.util.List;
+import java.util.NoSuchElementException;
+import java.util.Optional;
+import java.util.jar.JarEntry;
+import java.util.jar.JarFile;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
+
+/**
+ * {@code FromClasspathEntryClassInformationProvider} assumes the passed job class being available
+ * on some classpath.
+ */
+public class FromClasspathEntryClassInformationProvider implements EntryClassInformationProvider {
+
+ private final String jobClassName;
+
+ /**
+ * Creates a {@code FromClasspathEntryClassInformationProvider} based on the passed job class
+ * and classpath.
+ *
+ * @param jobClassName The job's class name.
+ * @param classpath The classpath the job class should be part of.
+ * @return The {@code FromClasspathEntryClassInformationProvider} instances collecting the
+ * necessary information.
+ * @throws FlinkException If the passed job class is not present on the passed classpath.
+ */
+ public static FromClasspathEntryClassInformationProvider create(
+ String jobClassName, Iterable<URL> classpath) throws FlinkException {
+ Preconditions.checkNotNull(jobClassName, "No job class name passed.");
+ Preconditions.checkNotNull(classpath, "No classpath passed.");
+ if (!userClasspathContainsJobClass(jobClassName, classpath)) {
+ throw new FlinkException(
+ String.format(
+ "Could not find the provided job class (%s) in the user lib directory.",
+ jobClassName));
+ }
+
+ return new FromClasspathEntryClassInformationProvider(jobClassName);
+ }
+
+ /**
+ * Creates a {@code FromClasspathEntryClassInformationProvider} looking for the job class on the
+ * passed classpath.
Review comment:
Would be good to clarify _which_ job class it is looking for.
##########
File path: flink-clients/src/test/java/org/apache/flink/client/testjar/ClasspathProvider.java
##########
@@ -0,0 +1,230 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.client.testjar;
+
+import org.apache.flink.client.deployment.application.JarManifestParser;
+import org.apache.flink.client.program.PackagedProgram;
+
+import org.junit.rules.ExternalResource;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
+
+/** {@code ClasspathProvider} offers methods to generate classpaths based on actual jars. */
+public class ClasspathProvider extends ExternalResource {
+
+ private static final String CLASSPATH_PROPERTY_NAME = "java.class.path";
+
+ private static final Path TEST_JOB_JAR_PATH = Paths.get("target", "maven-test-jar.jar");
+
+ private static final Path JOB_JAR_PATH =
+ Paths.get("target", "maven-test-user-classloader-job-jar.jar");
+ private static final Path JOB_LIB_JAR_PATH =
+ Paths.get("target", "maven-test-user-classloader-job-lib-jar.jar");
+
+ private final TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+ private File userDirHasTwoEntryClasses;
+
+ private File userDirHasEntryClass;
+
+ private File userDirHasNotEntryClass;
+
+ // this was added to support the legacy tests of ClassPathPackagedProgramRetrieverTest in the
+ // refactored PackagedProgramRetrieverImplTest
+ private File userDirWithTestJob;
+
+ private File userDirWithNoJarButTextFile;
+
+ private String originalSystemClasspath;
+
+ @Override
+ public void before() throws IOException {
+ temporaryFolder.create();
+
+ userDirHasTwoEntryClasses =
+ temporaryFolder.newFolder("_test_user_dir_has_two_entry_classes");
+ copyJar(JOB_JAR_PATH, userDirHasTwoEntryClasses);
Review comment:
Why do we need these copy operations? Couldn't we construct the classpath from the original locations?
##########
File path: flink-clients/src/main/java/org/apache/flink/client/program/PackagedProgramRetrieverImpl.java
##########
@@ -0,0 +1,214 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.client.program;
+
+import org.apache.flink.annotation.VisibleForTesting;
+import org.apache.flink.client.deployment.application.EntryClassInformationProvider;
+import org.apache.flink.client.deployment.application.FromClasspathEntryClassInformationProvider;
+import org.apache.flink.client.deployment.application.FromJarEntryClassInformationProvider;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.util.FileUtils;
+import org.apache.flink.util.FlinkException;
+import org.apache.flink.util.Preconditions;
+import org.apache.flink.util.function.FunctionUtils;
+
+import javax.annotation.Nullable;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URL;
+import java.nio.file.Path;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/** {@code PackageProgramRetrieverImpl} implements {@link PackagedProgramRetriever}. */
+public class PackagedProgramRetrieverImpl implements PackagedProgramRetriever {
+
+ private final EntryClassInformationProvider entryClassInformationProvider;
+ private final String[] programArguments;
+ private final List<URL> userClasspath;
+ private final Configuration configuration;
+
+ /**
+ * Creates a {@code PackageProgramRetrieverImpl} with the given parameters.
+ *
+ * @param userLibDir The user library directory that is used for generating the user classpath
+ * if specified. The system classpath is used if not specified.
+ * @param jobClassName The job class that will be used if specified. The classpath is used to
+ * detect any main class if not specified.
+ * @param programArgs The program arguments.
+ * @param configuration The Flink configuration for the given job.
+ * @return The {@code PackageProgramRetrieverImpl} that can be used to create a {@link
+ * PackagedProgram} instance.
+ * @throws FlinkException If something goes wrong during instantiation.
+ */
+ public static PackagedProgramRetrieverImpl create(
+ @Nullable File userLibDir,
+ @Nullable String jobClassName,
+ String[] programArgs,
+ Configuration configuration)
+ throws FlinkException {
+ return create(userLibDir, null, jobClassName, programArgs, configuration);
+ }
+
+ /**
+ * Creates a {@code PackageProgramRetrieverImpl} with the given parameters.
+ *
+ * @param userLibDir The user library directory that is used for generating the user classpath
+ * if specified. The system classpath is used if not specified.
+ * @param jarFile The jar archive used expecting to have the job class included.
+ * @param jobClassName The job class that will be used if specified. The classpath is used to
+ * detect any main class if not specified.
+ * @param programArgs The program arguments.
+ * @param configuration The Flink configuration for the given job.
+ * @return The {@code PackageProgramRetrieverImpl} that can be used to create a {@link
+ * PackagedProgram} instance.
+ * @throws FlinkException If something goes wrong during instantiation.
+ */
+ public static PackagedProgramRetrieverImpl create(
+ @Nullable File userLibDir,
+ @Nullable File jarFile,
+ @Nullable String jobClassName,
+ String[] programArgs,
+ Configuration configuration)
+ throws FlinkException {
+ List<URL> userClasspath;
+ try {
+ userClasspath = extractUserClasspath(userLibDir);
+ } catch (IOException e) {
+ throw new FlinkException("An error occurred while extracting the user classpath.", e);
+ }
+
+ final EntryClassInformationProvider entryClassInformationProvider =
+ createEntryClassInformationProvider(
+ userLibDir, userClasspath, jarFile, jobClassName, programArgs);
+ return new PackagedProgramRetrieverImpl(
+ entryClassInformationProvider, programArgs, userClasspath, configuration);
+ }
+
+ @VisibleForTesting
+ static EntryClassInformationProvider createEntryClassInformationProvider(
+ @Nullable File userLibDir,
+ Iterable<URL> userClasspath,
+ @Nullable File jarFile,
+ @Nullable String jobClassName,
+ String[] programArgs)
+ throws FlinkException {
+ if (PackagedProgramUtils.isPython(jobClassName)
+ || PackagedProgramUtils.isPython(programArgs)) {
+ return FromJarEntryClassInformationProvider.createFromPythonJar();
+ }
+
+ if (jarFile != null) {
+ return FromJarEntryClassInformationProvider.createFromCustomJar(jarFile, jobClassName);
+ }
+
+ if (userLibDir != null) {
+ return fromUserClasspath(jobClassName, userClasspath);
+ }
+
+ return fromSystemClasspath(jobClassName);
+ }
+
+ static EntryClassInformationProvider fromSystemClasspath(@Nullable String jobClassName)
+ throws FlinkException {
+ if (jobClassName != null) {
+ return FromClasspathEntryClassInformationProvider
+ .createWithJobClassAssumingOnSystemClasspath(jobClassName);
+ }
+
+ try {
+ return FromClasspathEntryClassInformationProvider.createFromSystemClasspath();
+ } catch (Throwable t) {
Review comment:
Why do you catch Throwables? Why is catching FlinkExceptions not sufficient?
##########
File path: flink-clients/src/test/java/org/apache/flink/client/program/PackagedProgramRetrieverImplTest.java
##########
@@ -0,0 +1,573 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.client.program;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.api.dag.Pipeline;
+import org.apache.flink.client.deployment.application.EntryClassInformationProvider;
+import org.apache.flink.client.deployment.executors.PipelineExecutorUtils;
+import org.apache.flink.client.testjar.ClasspathProvider;
+import org.apache.flink.configuration.ConfigConstants;
+import org.apache.flink.configuration.ConfigUtils;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.CoreOptions;
+import org.apache.flink.configuration.PipelineOptions;
+import org.apache.flink.configuration.PipelineOptionsInternal;
+import org.apache.flink.runtime.execution.librarycache.FlinkUserCodeClassLoaders;
+import org.apache.flink.runtime.jobgraph.JobGraph;
+import org.apache.flink.runtime.jobgraph.SavepointRestoreSettings;
+import org.apache.flink.util.ChildFirstClassLoader;
+import org.apache.flink.util.ExceptionUtils;
+import org.apache.flink.util.FileUtils;
+import org.apache.flink.util.FlinkException;
+import org.apache.flink.util.TestLogger;
+
+import org.hamcrest.collection.IsIterableContainingInAnyOrder;
+import org.hamcrest.core.IsInstanceOf;
+import org.junit.Assume;
+import org.junit.Ignore;
+import org.junit.Rule;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.core.Is.is;
+import static org.hamcrest.core.IsNot.not;
+import static org.junit.Assert.fail;
+
+/** {@code PackagedProgramRetrieverImplTest} tests {@link PackagedProgramRetrieverImpl}. */
+public class PackagedProgramRetrieverImplTest extends TestLogger {
+
+ @Rule public ClasspathProvider classpathProvider = new ClasspathProvider();
+
+ @Test
+ public void testDeriveEntryClassInformationForPythonBasedOnJobName() throws FlinkException {
+ // TODO: see FLINK-23154
+ Assume.assumeTrue(
+ "This tests only succeeds if " + ConfigConstants.ENV_FLINK_OPT_DIR + " is set.",
+ System.getProperty(ConfigConstants.ENV_FLINK_OPT_DIR) != null);
+ final EntryClassInformationProvider informationProvider =
+ PackagedProgramRetrieverImpl.createEntryClassInformationProvider(
+ null,
+ Collections.emptyList(),
+ null,
+ PackagedProgramUtils.getPythonDriverClassName(),
+ new String[0]);
+ assertThat(informationProvider.getJobClassName().isPresent(), is(true));
+ assertThat(
+ informationProvider.getJobClassName().get(),
+ is(PackagedProgramUtils.getPythonDriverClassName()));
+ assertThat(informationProvider.getJarFile().isPresent(), is(true));
+ assertThat(
+ informationProvider.getJarFile().get(),
+ is(PackagedProgramUtils.getPythonJar().getFile()));
+ }
+
+ @Test
+ public void testDeriveEntryClassInformationForPythonBasedOnParameter() throws FlinkException {
+ // TODO: see FLINK-23154
+ Assume.assumeTrue(
+ "This tests only succeeds if " + ConfigConstants.ENV_FLINK_OPT_DIR + " is set.",
+ System.getProperty(ConfigConstants.ENV_FLINK_OPT_DIR) != null);
+
+ final EntryClassInformationProvider informationProvider =
+ PackagedProgramRetrieverImpl.createEntryClassInformationProvider(
+ null, Collections.emptyList(), null, null, new String[] {"--python"});
+
+ assertThat(informationProvider.getJobClassName().isPresent(), is(true));
+ assertThat(
+ informationProvider.getJobClassName().get(),
+ is(PackagedProgramUtils.getPythonDriverClassName()));
+ assertThat(informationProvider.getJarFile().isPresent(), is(true));
+ assertThat(
+ informationProvider.getJarFile().get(),
+ is(PackagedProgramUtils.getPythonJar().getFile()));
+ }
+
+ @Test
+ public void testDeriveEntryClassInformationForCustomJar()
+ throws FlinkException, MalformedURLException {
+ // make loading from system classpath fail to make sure that it's not triggered
+ classpathProvider.setSystemClasspathWithTwoEntryClasses();
+
+ final String jobClassName = "SomeJobClassName";
+ final File jarFile = new File("some/jar/file.jar");
+ final EntryClassInformationProvider informationProvider =
+ PackagedProgramRetrieverImpl.createEntryClassInformationProvider(
+ null, null, jarFile, jobClassName, new String[0]);
+ assertThat(informationProvider.getJobClassName().isPresent(), is(true));
+ assertThat(informationProvider.getJobClassName().get(), is(jobClassName));
+ assertThat(informationProvider.getJarFile().isPresent(), is(true));
+ assertThat(informationProvider.getJarFile().get(), is(jarFile));
+ }
+
+ @Test
+ // TODO: we might want to change this behavior triggering a failure here
+ public void testDeriveEntryClassInformationFromSystemClasspathWithNonExistingJobClassName()
+ throws IOException, FlinkException {
+ classpathProvider.setSystemClasspathWithEntryClass();
+
+ final String jobClassName = "SomeJobClassNotBeingOnTheSystemClasspath";
+ final EntryClassInformationProvider informationProvider =
+ PackagedProgramRetrieverImpl.createEntryClassInformationProvider(
+ null, null, null, jobClassName, new String[0]);
+ assertThat(informationProvider.getJobClassName().isPresent(), is(true));
+ assertThat(informationProvider.getJobClassName().get(), is(jobClassName));
+ assertThat(informationProvider.getJarFile().isPresent(), is(false));
+ }
+
+ @Test
+ public void testDeriveEntryClassInformationFromSystemClasspathWithExistingJobClassName()
+ throws IOException, FlinkException {
+ classpathProvider.setSystemClasspathWithEntryClass();
+
+ final EntryClassInformationProvider informationProvider =
+ PackagedProgramRetrieverImpl.createEntryClassInformationProvider(
+ null, null, null, classpathProvider.getJobClassName(), new String[0]);
+ assertThat(informationProvider.getJobClassName().isPresent(), is(true));
+ assertThat(
+ informationProvider.getJobClassName().get(),
+ is(classpathProvider.getJobClassName()));
+ assertThat(informationProvider.getJarFile().isPresent(), is(false));
+ }
+
+ @Test
+ public void testDeriveEntryClassInformationFromSystemClasspathExtractingTheJobClassFromThere()
+ throws IOException, FlinkException {
+ classpathProvider.setSystemClasspathWithEntryClass();
+
+ final EntryClassInformationProvider informationProvider =
+ PackagedProgramRetrieverImpl.createEntryClassInformationProvider(
+ null, null, null, null, new String[0]);
+ assertThat(informationProvider.getJobClassName().isPresent(), is(true));
+ assertThat(
+ informationProvider.getJobClassName().get(),
+ is(classpathProvider.getJobClassName()));
+ assertThat(informationProvider.getJarFile().isPresent(), is(false));
+ }
+
+ @Test
+ public void testDeriveEntryClassInformationFromClasspathWithJobClass()
+ throws IOException, FlinkException {
+ final EntryClassInformationProvider informationProvider =
+ PackagedProgramRetrieverImpl.createEntryClassInformationProvider(
+ // the user directory must be specified
+ classpathProvider.getUserDirectoryWithTwoEntryClasses(),
+ // the user classpath is derived from the user directory outside of the
+ // method
+ classpathProvider.getURLUserClasspathWithTwoEntryClasses(),
+ null,
+ // we have to specify the job class - otherwise the call would fail due to
+ // two main method being present
+ classpathProvider.getJobClassName(),
+ new String[0]);
+ assertThat(informationProvider.getJobClassName().isPresent(), is(true));
+ assertThat(
+ informationProvider.getJobClassName().get(),
+ is(classpathProvider.getJobClassName()));
+ assertThat(informationProvider.getJarFile().isPresent(), is(false));
+ }
+
+ @Test
+ public void testDeriveEntryClassInformationFromClasspathWithNoJobClass()
+ throws IOException, FlinkException {
+ final EntryClassInformationProvider informationProvider =
+ PackagedProgramRetrieverImpl.createEntryClassInformationProvider(
+ // the user directory must be specified
+ classpathProvider.getUserDirectoryWithOneEntryClass(),
+ // the user classpath is derived from the user directory outside of the
+ // method
+ classpathProvider.getURLUserClasspathWithEntryClass(),
+ null,
+ // no job class name is specified which enables looking for the entry class
+ // on the user classpath
+ null,
+ new String[0]);
+ assertThat(informationProvider.getJobClassName().isPresent(), is(true));
+ assertThat(
+ informationProvider.getJobClassName().get(),
+ is(classpathProvider.getJobClassName()));
+ assertThat(informationProvider.getJarFile().isPresent(), is(false));
+ }
+
+ @Test
+ public void testCreateWithUserLibDir() throws FlinkException {
+ final PackagedProgramRetriever retriever =
+ PackagedProgramRetrieverImpl.create(
+ classpathProvider.getUserDirectoryWithOneEntryClass(),
+ null,
+ classpathProvider.getJobClassName(),
+ new String[0],
+ new Configuration());
+
+ // the right information is picked up without any error
+ assertThat(
+ retriever.getPackagedProgram().getMainClassName(),
+ is(classpathProvider.getJobClassName()));
+ }
+
+ @Test
+ public void testJobGraphRetrieval()
+ throws IOException, FlinkException, ProgramInvocationException {
+ final int parallelism = 42;
+ final JobID jobId = new JobID();
+
+ final Configuration configuration = new Configuration();
+ configuration.setInteger(CoreOptions.DEFAULT_PARALLELISM, parallelism);
+ configuration.set(PipelineOptionsInternal.PIPELINE_FIXED_JOB_ID, jobId.toHexString());
+
+ final String expectedSuffix = "suffix";
+ final PackagedProgramRetriever retriever =
+ PackagedProgramRetrieverImpl.create(
+ null,
+ classpathProvider.getTestJobClassName(),
+ classpathProvider.getTestJobArgs(expectedSuffix),
+ new Configuration());
+
+ final JobGraph jobGraph = retrieveJobGraph(retriever, configuration);
+
+ assertThat(
+ jobGraph.getName(),
+ is(classpathProvider.getTestJobClassName() + "-" + expectedSuffix));
+ assertThat(jobGraph.getSavepointRestoreSettings(), is(SavepointRestoreSettings.none()));
+ assertThat(jobGraph.getMaximumParallelism(), is(parallelism));
+ assertThat(jobGraph.getJobID(), is(jobId));
+ }
+
+ @Test
+ public void testJobGraphRetrievalFromJar()
+ throws IOException, FlinkException, ProgramInvocationException {
+ final String expectedSuffix = "suffix";
+ final PackagedProgramRetriever retrieverUnderTest =
+ PackagedProgramRetrieverImpl.create(
+ classpathProvider.getUserDirectoryWithTestJob(),
+ null,
+ null,
+ classpathProvider.getTestJobArgs(expectedSuffix),
+ new Configuration());
+
+ final JobGraph jobGraph = retrieveJobGraph(retrieverUnderTest, new Configuration());
+
+ assertThat(
+ jobGraph.getName(),
+ is(classpathProvider.getTestJobClassName() + "-" + expectedSuffix));
+ }
+
+ @Test
+ public void testJobGraphRetrievalJobClassNameHasPrecedenceOverClasspath()
+ throws IOException, FlinkException, ProgramInvocationException {
+ // TODO: I don't get this test: It feels like it does not add any extra value - it does the
+ // same like testTooManyEntryClassesOnSystemClasspath
+ final String expectedSuffix = "suffix";
+ final PackagedProgramRetriever retrieverUnderTest =
+ // Both a class name is specified and a JAR "is" on the class path
+ // The class name should have precedence.
+ PackagedProgramRetrieverImpl.create(
+ null,
+ classpathProvider.getTestJobClassName(),
+ classpathProvider.getTestJobArgs(expectedSuffix),
+ new Configuration());
+
+ final JobGraph jobGraph = retrieveJobGraph(retrieverUnderTest, new Configuration());
+
+ assertThat(jobGraph.getName(), is(classpathProvider.getTestJobClassName() + "-suffix"));
+ }
+
+ @Test
+ public void testSavepointRestoreSettings()
+ throws FlinkException, IOException, ProgramInvocationException {
+ final Configuration configuration = new Configuration();
+ final SavepointRestoreSettings savepointRestoreSettings =
+ SavepointRestoreSettings.forPath("foobar", true);
+ final JobID jobId = new JobID();
+
+ configuration.setString(PipelineOptionsInternal.PIPELINE_FIXED_JOB_ID, jobId.toHexString());
+ SavepointRestoreSettings.toConfiguration(savepointRestoreSettings, configuration);
+
+ final String expectedSuffix = "suffix";
+ final PackagedProgramRetriever retrieverUnderTest =
+ PackagedProgramRetrieverImpl.create(
+ null,
+ classpathProvider.getTestJobClassName(),
+ classpathProvider.getTestJobArgs(expectedSuffix),
+ new Configuration());
+
+ final JobGraph jobGraph = retrieveJobGraph(retrieverUnderTest, configuration);
+
+ assertThat(jobGraph.getSavepointRestoreSettings(), is(savepointRestoreSettings));
+ assertThat(jobGraph.getJobID(), is(jobId));
+ }
+
+ @Test
+ public void testFailIfJobDirDoesNotHaveEntryClass() {
+ try {
+ PackagedProgramRetrieverImpl.create(
+ classpathProvider.getUserDirectoryWithNoEntryClass(),
+ classpathProvider.getTestJobClassName(),
+ classpathProvider.getTestJobArgs("suffix"),
+ new Configuration());
+ fail("This case should throw exception !");
+ } catch (FlinkException e) {
+ assertThat(
+ ExceptionUtils.findThrowableWithMessage(
+ e,
+ String.format(
+ "Could not find the provided job class (%s) in the user lib directory.",
+ classpathProvider.getTestJobClassName()))
+ .isPresent(),
+ is(true));
+ }
+ }
+
+ // TODO: this test checks the same code path as
+ // testDeriveEntryClassInformationFromSystemClasspathWithNonExistingJobClassName
+ // We should make it fail early if the class is not present on the system classpath
+ // Right now, the test is failing because no error is thrown
+ @Ignore
+ @Test(expected = FlinkException.class)
+ public void testEntryClassNotFoundOnSystemClasspath() throws FlinkException {
+ PackagedProgramRetrieverImpl.create(
+ null, "NotExistingClass", new String[0], new Configuration());
+ }
+
+ @Test(expected = FlinkException.class)
+ public void testEntryClassNotFoundOnUserClasspath() throws FlinkException {
+ PackagedProgramRetrieverImpl.create(
+ classpathProvider.getUserDirectoryWithNoEntryClass(),
+ classpathProvider.getTestJobClassName(),
+ new String[0],
+ new Configuration());
+ }
+
+ @Test(expected = FlinkException.class)
+ public void testTooManyEntryClassesOnSystemClasspath() throws FlinkException {
+ PackagedProgramRetrieverImpl.create(null, null, new String[0], new Configuration());
+ }
+
+ @Test
+ public void testTooManyEntryClassesOnUserClasspath() throws FlinkException, IOException {
+ classpathProvider.setSystemClasspathWithTwoEntryClasses();
+ final PackagedProgramRetrieverImpl retriever =
+ PackagedProgramRetrieverImpl.create(
+ classpathProvider.getUserDirectoryWithTwoEntryClasses(),
+ classpathProvider.getTestJobClassName(),
+ new String[0],
+ new Configuration());
+ assertThat(
+ retriever.getPackagedProgram().getMainClassName(),
+ is(classpathProvider.getTestJobClassName()));
+ }
+
+ @Test
+ public void testRetrieveCorrectUserClasspathsWithoutSpecifiedEntryClass()
+ throws IOException, FlinkException, ProgramInvocationException {
+ final PackagedProgramRetriever retrieverUnderTest =
+ PackagedProgramRetrieverImpl.create(
+ classpathProvider.getUserDirectoryWithOneEntryClass(),
+ null,
+ classpathProvider.getTestJobArgs("suffix"),
+ new Configuration());
+ final JobGraph jobGraph = retrieveJobGraph(retrieverUnderTest, new Configuration());
+ final List<String> actualClasspath =
+ jobGraph.getClasspaths().stream().map(URL::toString).collect(Collectors.toList());
+
+ final Path workingDirectory = FileUtils.getCurrentWorkingDirectory();
+ final List<String> expectedClasspath = new ArrayList<>();
+ for (File file : classpathProvider.getUserDirectoryWithOneEntryClass().listFiles()) {
+ if (!file.getName().endsWith("jar")) {
+ // only jars are expected
+ continue;
+ }
+
+ Path relativePath = FileUtils.relativizePath(workingDirectory, file.toPath());
+ expectedClasspath.add(FileUtils.toURL(relativePath).toString());
+ }
+
+ assertThat(
+ actualClasspath,
+ IsIterableContainingInAnyOrder.containsInAnyOrder(expectedClasspath.toArray()));
+ }
+
+ @Test
+ public void testRetrieveCorrectUserClasspathsWithSpecifiedEntryClass()
+ throws IOException, FlinkException, ProgramInvocationException {
+ final PackagedProgramRetriever retrieverUnderTest =
+ PackagedProgramRetrieverImpl.create(
+ classpathProvider.getUserDirectoryWithOneEntryClass(),
+ classpathProvider.getJobClassName(),
+ classpathProvider.getTestJobArgs("suffix"),
+ new Configuration());
+ final JobGraph jobGraph = retrieveJobGraph(retrieverUnderTest, new Configuration());
+ final List<String> actualClasspath =
+ jobGraph.getClasspaths().stream().map(URL::toString).collect(Collectors.toList());
+
+ final Path workingDirectory = FileUtils.getCurrentWorkingDirectory();
+ final List<String> expectedClasspath = new ArrayList<>();
+ for (File file : classpathProvider.getUserDirectoryWithOneEntryClass().listFiles()) {
+ if (!file.getName().endsWith("jar")) {
+ // only jars are expected
+ continue;
+ }
+
+ Path relativePath = FileUtils.relativizePath(workingDirectory, file.toPath());
+ expectedClasspath.add(FileUtils.toURL(relativePath).toString());
+ }
+
+ assertThat(
+ actualClasspath,
+ IsIterableContainingInAnyOrder.containsInAnyOrder(expectedClasspath.toArray()));
+ }
+
+ @Test
+ public void testRetrieveFromJarFileWithoutUserLib()
+ throws IOException, FlinkException, ProgramInvocationException {
+ final PackagedProgramRetriever retrieverUnderTest =
+ PackagedProgramRetrieverImpl.create(
+ null,
+ classpathProvider.getTestJobJar(),
+ null,
+ classpathProvider.getTestJobArgs("suffix"),
+ new Configuration());
+ final JobGraph jobGraph = retrieveJobGraph(retrieverUnderTest, new Configuration());
+
+ assertThat(
+ jobGraph.getUserJars(),
+ containsInAnyOrder(
+ new org.apache.flink.core.fs.Path(
+ classpathProvider.getTestJobJar().toURI())));
+ assertThat(jobGraph.getClasspaths().isEmpty(), is(true));
+ }
+
+ @Test
+ public void testRetrieveFromJarFileWithUserLib()
+ throws IOException, FlinkException, ProgramInvocationException {
+ final PackagedProgramRetriever retrieverUnderTest =
+ PackagedProgramRetrieverImpl.create(
+ classpathProvider.getUserDirectoryWithOneEntryClass(),
+ classpathProvider.getTestJobJar(),
+ null,
+ classpathProvider.getTestJobArgs("suffix"),
+ new Configuration());
+ final JobGraph jobGraph = retrieveJobGraph(retrieverUnderTest, new Configuration());
+
+ assertThat(
+ jobGraph.getUserJars(),
+ containsInAnyOrder(
+ new org.apache.flink.core.fs.Path(
+ classpathProvider.getTestJobJar().toURI())));
+ final List<String> actualClasspath =
+ jobGraph.getClasspaths().stream().map(URL::toString).collect(Collectors.toList());
+ final Path workingDirectory = FileUtils.getCurrentWorkingDirectory();
+ final List<String> expectedClasspath = new ArrayList<>();
+ for (File file : classpathProvider.getUserDirectoryWithOneEntryClass().listFiles()) {
+ if (!file.getName().endsWith("jar")) {
+ // only jars are expected
+ continue;
+ }
+
+ Path relativePath = FileUtils.relativizePath(workingDirectory, file.toPath());
+ expectedClasspath.add(FileUtils.toURL(relativePath).toString());
+ }
+
+ assertThat(
+ actualClasspath,
+ IsIterableContainingInAnyOrder.containsInAnyOrder(expectedClasspath.toArray()));
+ }
+
+ @Test
+ public void testChildFirstDefaultConfiguration() throws FlinkException {
Review comment:
both this test and the one below should be renamed to clarify that they are about the class loaders
##########
File path: flink-clients/src/test/java/org/apache/flink/client/program/PackagedProgramRetrieverImplTest.java
##########
@@ -0,0 +1,573 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.client.program;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.api.dag.Pipeline;
+import org.apache.flink.client.deployment.application.EntryClassInformationProvider;
+import org.apache.flink.client.deployment.executors.PipelineExecutorUtils;
+import org.apache.flink.client.testjar.ClasspathProvider;
+import org.apache.flink.configuration.ConfigConstants;
+import org.apache.flink.configuration.ConfigUtils;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.CoreOptions;
+import org.apache.flink.configuration.PipelineOptions;
+import org.apache.flink.configuration.PipelineOptionsInternal;
+import org.apache.flink.runtime.execution.librarycache.FlinkUserCodeClassLoaders;
+import org.apache.flink.runtime.jobgraph.JobGraph;
+import org.apache.flink.runtime.jobgraph.SavepointRestoreSettings;
+import org.apache.flink.util.ChildFirstClassLoader;
+import org.apache.flink.util.ExceptionUtils;
+import org.apache.flink.util.FileUtils;
+import org.apache.flink.util.FlinkException;
+import org.apache.flink.util.TestLogger;
+
+import org.hamcrest.collection.IsIterableContainingInAnyOrder;
+import org.hamcrest.core.IsInstanceOf;
+import org.junit.Assume;
+import org.junit.Ignore;
+import org.junit.Rule;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.core.Is.is;
+import static org.hamcrest.core.IsNot.not;
+import static org.junit.Assert.fail;
+
+/** {@code PackagedProgramRetrieverImplTest} tests {@link PackagedProgramRetrieverImpl}. */
+public class PackagedProgramRetrieverImplTest extends TestLogger {
+
+ @Rule public ClasspathProvider classpathProvider = new ClasspathProvider();
+
+ @Test
+ public void testDeriveEntryClassInformationForPythonBasedOnJobName() throws FlinkException {
+ // TODO: see FLINK-23154
+ Assume.assumeTrue(
+ "This tests only succeeds if " + ConfigConstants.ENV_FLINK_OPT_DIR + " is set.",
+ System.getProperty(ConfigConstants.ENV_FLINK_OPT_DIR) != null);
+ final EntryClassInformationProvider informationProvider =
+ PackagedProgramRetrieverImpl.createEntryClassInformationProvider(
+ null,
+ Collections.emptyList(),
+ null,
+ PackagedProgramUtils.getPythonDriverClassName(),
+ new String[0]);
+ assertThat(informationProvider.getJobClassName().isPresent(), is(true));
+ assertThat(
+ informationProvider.getJobClassName().get(),
+ is(PackagedProgramUtils.getPythonDriverClassName()));
+ assertThat(informationProvider.getJarFile().isPresent(), is(true));
+ assertThat(
+ informationProvider.getJarFile().get(),
+ is(PackagedProgramUtils.getPythonJar().getFile()));
+ }
+
+ @Test
+ public void testDeriveEntryClassInformationForPythonBasedOnParameter() throws FlinkException {
+ // TODO: see FLINK-23154
+ Assume.assumeTrue(
+ "This tests only succeeds if " + ConfigConstants.ENV_FLINK_OPT_DIR + " is set.",
+ System.getProperty(ConfigConstants.ENV_FLINK_OPT_DIR) != null);
+
+ final EntryClassInformationProvider informationProvider =
+ PackagedProgramRetrieverImpl.createEntryClassInformationProvider(
+ null, Collections.emptyList(), null, null, new String[] {"--python"});
+
+ assertThat(informationProvider.getJobClassName().isPresent(), is(true));
+ assertThat(
+ informationProvider.getJobClassName().get(),
+ is(PackagedProgramUtils.getPythonDriverClassName()));
+ assertThat(informationProvider.getJarFile().isPresent(), is(true));
+ assertThat(
+ informationProvider.getJarFile().get(),
+ is(PackagedProgramUtils.getPythonJar().getFile()));
+ }
+
+ @Test
+ public void testDeriveEntryClassInformationForCustomJar()
+ throws FlinkException, MalformedURLException {
+ // make loading from system classpath fail to make sure that it's not triggered
+ classpathProvider.setSystemClasspathWithTwoEntryClasses();
+
+ final String jobClassName = "SomeJobClassName";
+ final File jarFile = new File("some/jar/file.jar");
+ final EntryClassInformationProvider informationProvider =
+ PackagedProgramRetrieverImpl.createEntryClassInformationProvider(
+ null, null, jarFile, jobClassName, new String[0]);
+ assertThat(informationProvider.getJobClassName().isPresent(), is(true));
+ assertThat(informationProvider.getJobClassName().get(), is(jobClassName));
+ assertThat(informationProvider.getJarFile().isPresent(), is(true));
+ assertThat(informationProvider.getJarFile().get(), is(jarFile));
+ }
+
+ @Test
+ // TODO: we might want to change this behavior triggering a failure here
+ public void testDeriveEntryClassInformationFromSystemClasspathWithNonExistingJobClassName()
+ throws IOException, FlinkException {
+ classpathProvider.setSystemClasspathWithEntryClass();
+
+ final String jobClassName = "SomeJobClassNotBeingOnTheSystemClasspath";
+ final EntryClassInformationProvider informationProvider =
+ PackagedProgramRetrieverImpl.createEntryClassInformationProvider(
+ null, null, null, jobClassName, new String[0]);
+ assertThat(informationProvider.getJobClassName().isPresent(), is(true));
+ assertThat(informationProvider.getJobClassName().get(), is(jobClassName));
+ assertThat(informationProvider.getJarFile().isPresent(), is(false));
+ }
+
+ @Test
+ public void testDeriveEntryClassInformationFromSystemClasspathWithExistingJobClassName()
+ throws IOException, FlinkException {
+ classpathProvider.setSystemClasspathWithEntryClass();
+
+ final EntryClassInformationProvider informationProvider =
+ PackagedProgramRetrieverImpl.createEntryClassInformationProvider(
+ null, null, null, classpathProvider.getJobClassName(), new String[0]);
+ assertThat(informationProvider.getJobClassName().isPresent(), is(true));
+ assertThat(
+ informationProvider.getJobClassName().get(),
+ is(classpathProvider.getJobClassName()));
+ assertThat(informationProvider.getJarFile().isPresent(), is(false));
+ }
+
+ @Test
+ public void testDeriveEntryClassInformationFromSystemClasspathExtractingTheJobClassFromThere()
+ throws IOException, FlinkException {
+ classpathProvider.setSystemClasspathWithEntryClass();
+
+ final EntryClassInformationProvider informationProvider =
+ PackagedProgramRetrieverImpl.createEntryClassInformationProvider(
+ null, null, null, null, new String[0]);
+ assertThat(informationProvider.getJobClassName().isPresent(), is(true));
+ assertThat(
+ informationProvider.getJobClassName().get(),
+ is(classpathProvider.getJobClassName()));
+ assertThat(informationProvider.getJarFile().isPresent(), is(false));
+ }
+
+ @Test
+ public void testDeriveEntryClassInformationFromClasspathWithJobClass()
+ throws IOException, FlinkException {
+ final EntryClassInformationProvider informationProvider =
+ PackagedProgramRetrieverImpl.createEntryClassInformationProvider(
+ // the user directory must be specified
+ classpathProvider.getUserDirectoryWithTwoEntryClasses(),
+ // the user classpath is derived from the user directory outside of the
+ // method
+ classpathProvider.getURLUserClasspathWithTwoEntryClasses(),
Review comment:
the comments are also weird; they claim the user directory is required, but testDeriveEntryClassInformationFromSystemClasspathWithNonExistingJobClassName uses null for it and does the same assertions.
##########
File path: flink-clients/src/test/java/org/apache/flink/client/testjar/ClasspathProvider.java
##########
@@ -0,0 +1,226 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.client.testjar;
+
+import org.apache.flink.client.deployment.application.JarManifestParser;
+import org.apache.flink.client.program.PackagedProgram;
+
+import org.junit.rules.ExternalResource;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
+
+/** {@code ClasspathProvider} offers methods to generate classpaths based on actual jars. */
+public class ClasspathProvider extends ExternalResource {
+
+ private static final String CLASSPATH_PROPERTY_NAME = "java.class.path";
+
+ private static final Path TEST_JOB_JAR_PATH = Paths.get("target", "maven-test-jar.jar");
+
+ private static final Path JOB_JAR_PATH =
+ Paths.get("target", "maven-test-user-classloader-job-jar.jar");
+ private static final Path JOB_LIB_JAR_PATH =
+ Paths.get("target", "maven-test-user-classloader-job-lib-jar.jar");
+
+ private final TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+ private File userDirHasTwoEntryClasses;
+
+ private File userDirHasEntryClass;
+
+ private File userDirHasNotEntryClass;
+
+ // this was added to support the legacy tests of ClassPathPackagedProgramRetrieverTest in the
Review comment:
Yeah its a bit weird; why are we referencing a deleted class?
##########
File path: flink-clients/src/test/java/org/apache/flink/client/program/PackagedProgramRetrieverImplTest.java
##########
@@ -0,0 +1,573 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.client.program;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.api.dag.Pipeline;
+import org.apache.flink.client.deployment.application.EntryClassInformationProvider;
+import org.apache.flink.client.deployment.executors.PipelineExecutorUtils;
+import org.apache.flink.client.testjar.ClasspathProvider;
+import org.apache.flink.configuration.ConfigConstants;
+import org.apache.flink.configuration.ConfigUtils;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.CoreOptions;
+import org.apache.flink.configuration.PipelineOptions;
+import org.apache.flink.configuration.PipelineOptionsInternal;
+import org.apache.flink.runtime.execution.librarycache.FlinkUserCodeClassLoaders;
+import org.apache.flink.runtime.jobgraph.JobGraph;
+import org.apache.flink.runtime.jobgraph.SavepointRestoreSettings;
+import org.apache.flink.util.ChildFirstClassLoader;
+import org.apache.flink.util.ExceptionUtils;
+import org.apache.flink.util.FileUtils;
+import org.apache.flink.util.FlinkException;
+import org.apache.flink.util.TestLogger;
+
+import org.hamcrest.collection.IsIterableContainingInAnyOrder;
+import org.hamcrest.core.IsInstanceOf;
+import org.junit.Assume;
+import org.junit.Ignore;
+import org.junit.Rule;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.core.Is.is;
+import static org.hamcrest.core.IsNot.not;
+import static org.junit.Assert.fail;
+
+/** {@code PackagedProgramRetrieverImplTest} tests {@link PackagedProgramRetrieverImpl}. */
+public class PackagedProgramRetrieverImplTest extends TestLogger {
+
+ @Rule public ClasspathProvider classpathProvider = new ClasspathProvider();
+
+ @Test
+ public void testDeriveEntryClassInformationForPythonBasedOnJobName() throws FlinkException {
+ // TODO: see FLINK-23154
+ Assume.assumeTrue(
+ "This tests only succeeds if " + ConfigConstants.ENV_FLINK_OPT_DIR + " is set.",
+ System.getProperty(ConfigConstants.ENV_FLINK_OPT_DIR) != null);
+ final EntryClassInformationProvider informationProvider =
+ PackagedProgramRetrieverImpl.createEntryClassInformationProvider(
+ null,
+ Collections.emptyList(),
+ null,
+ PackagedProgramUtils.getPythonDriverClassName(),
+ new String[0]);
+ assertThat(informationProvider.getJobClassName().isPresent(), is(true));
+ assertThat(
+ informationProvider.getJobClassName().get(),
+ is(PackagedProgramUtils.getPythonDriverClassName()));
+ assertThat(informationProvider.getJarFile().isPresent(), is(true));
+ assertThat(
+ informationProvider.getJarFile().get(),
+ is(PackagedProgramUtils.getPythonJar().getFile()));
+ }
+
+ @Test
+ public void testDeriveEntryClassInformationForPythonBasedOnParameter() throws FlinkException {
+ // TODO: see FLINK-23154
+ Assume.assumeTrue(
+ "This tests only succeeds if " + ConfigConstants.ENV_FLINK_OPT_DIR + " is set.",
+ System.getProperty(ConfigConstants.ENV_FLINK_OPT_DIR) != null);
+
+ final EntryClassInformationProvider informationProvider =
+ PackagedProgramRetrieverImpl.createEntryClassInformationProvider(
+ null, Collections.emptyList(), null, null, new String[] {"--python"});
+
+ assertThat(informationProvider.getJobClassName().isPresent(), is(true));
+ assertThat(
+ informationProvider.getJobClassName().get(),
+ is(PackagedProgramUtils.getPythonDriverClassName()));
+ assertThat(informationProvider.getJarFile().isPresent(), is(true));
+ assertThat(
+ informationProvider.getJarFile().get(),
+ is(PackagedProgramUtils.getPythonJar().getFile()));
+ }
+
+ @Test
+ public void testDeriveEntryClassInformationForCustomJar()
+ throws FlinkException, MalformedURLException {
+ // make loading from system classpath fail to make sure that it's not triggered
+ classpathProvider.setSystemClasspathWithTwoEntryClasses();
Review comment:
I don't understand why this would make it fail, nor what "it" that is not triggered actually is. Wouldn't it be more obvious if you did something like clearing the classpath?
##########
File path: flink-clients/src/main/java/org/apache/flink/client/program/PackagedProgramRetrieverImpl.java
##########
@@ -0,0 +1,214 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.client.program;
+
+import org.apache.flink.annotation.VisibleForTesting;
+import org.apache.flink.client.deployment.application.EntryClassInformationProvider;
+import org.apache.flink.client.deployment.application.FromClasspathEntryClassInformationProvider;
+import org.apache.flink.client.deployment.application.FromJarEntryClassInformationProvider;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.util.FileUtils;
+import org.apache.flink.util.FlinkException;
+import org.apache.flink.util.Preconditions;
+import org.apache.flink.util.function.FunctionUtils;
+
+import javax.annotation.Nullable;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URL;
+import java.nio.file.Path;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/** {@code PackageProgramRetrieverImpl} implements {@link PackagedProgramRetriever}. */
+public class PackagedProgramRetrieverImpl implements PackagedProgramRetriever {
+
+ private final EntryClassInformationProvider entryClassInformationProvider;
+ private final String[] programArguments;
+ private final List<URL> userClasspath;
+ private final Configuration configuration;
+
+ /**
+ * Creates a {@code PackageProgramRetrieverImpl} with the given parameters.
+ *
+ * @param userLibDir The user library directory that is used for generating the user classpath
+ * if specified. The system classpath is used if not specified.
+ * @param jobClassName The job class that will be used if specified. The classpath is used to
+ * detect any main class if not specified.
+ * @param programArgs The program arguments.
+ * @param configuration The Flink configuration for the given job.
+ * @return The {@code PackageProgramRetrieverImpl} that can be used to create a {@link
+ * PackagedProgram} instance.
+ * @throws FlinkException If something goes wrong during instantiation.
+ */
+ public static PackagedProgramRetrieverImpl create(
+ @Nullable File userLibDir,
+ @Nullable String jobClassName,
+ String[] programArgs,
+ Configuration configuration)
+ throws FlinkException {
+ return create(userLibDir, null, jobClassName, programArgs, configuration);
+ }
+
+ /**
+ * Creates a {@code PackageProgramRetrieverImpl} with the given parameters.
+ *
+ * @param userLibDir The user library directory that is used for generating the user classpath
+ * if specified. The system classpath is used if not specified.
+ * @param jarFile The jar archive used expecting to have the job class included.
+ * @param jobClassName The job class that will be used if specified. The classpath is used to
+ * detect any main class if not specified.
+ * @param programArgs The program arguments.
+ * @param configuration The Flink configuration for the given job.
+ * @return The {@code PackageProgramRetrieverImpl} that can be used to create a {@link
+ * PackagedProgram} instance.
+ * @throws FlinkException If something goes wrong during instantiation.
+ */
+ public static PackagedProgramRetrieverImpl create(
+ @Nullable File userLibDir,
+ @Nullable File jarFile,
Review comment:
Why do we allow this to be nullable if there is a dedicated factory method where this argument does not exist?
##########
File path: flink-yarn/src/main/java/org/apache/flink/yarn/entrypoint/YarnApplicationClusterEntryPoint.java
##########
@@ -129,16 +129,12 @@ private static PackagedProgramRetriever getPackagedProgramRetriever(
final Configuration configuration,
final String[] programArguments,
@Nullable final String jobClassName)
- throws IOException {
+ throws FlinkException {
final File userLibDir = YarnEntrypointUtils.getUsrLibDir(configuration).orElse(null);
final File userApplicationJar = getUserApplicationJar(userLibDir, configuration);
- final ClassPathPackagedProgramRetriever.Builder retrieverBuilder =
- ClassPathPackagedProgramRetriever.newBuilder(programArguments)
- .setUserLibDirectory(userLibDir)
- .setJarFile(userApplicationJar)
- .setJobClassName(jobClassName);
- return retrieverBuilder.build();
+ return PackagedProgramRetrieverImpl.create(
+ userLibDir, userApplicationJar, jobClassName, programArguments, configuration);
Review comment:
Why is this not using the builder approach any longer? Wasn't that way more flexible?
##########
File path: flink-clients/src/test/java/org/apache/flink/client/testjar/ClasspathProvider.java
##########
@@ -0,0 +1,230 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.client.testjar;
+
+import org.apache.flink.client.deployment.application.JarManifestParser;
+import org.apache.flink.client.program.PackagedProgram;
+
+import org.junit.rules.ExternalResource;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
+
+/** {@code ClasspathProvider} offers methods to generate classpaths based on actual jars. */
+public class ClasspathProvider extends ExternalResource {
+
+ private static final String CLASSPATH_PROPERTY_NAME = "java.class.path";
+
+ private static final Path TEST_JOB_JAR_PATH = Paths.get("target", "maven-test-jar.jar");
+
+ private static final Path JOB_JAR_PATH =
+ Paths.get("target", "maven-test-user-classloader-job-jar.jar");
+ private static final Path JOB_LIB_JAR_PATH =
+ Paths.get("target", "maven-test-user-classloader-job-lib-jar.jar");
+
+ private final TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+ private File userDirHasTwoEntryClasses;
+
+ private File userDirHasEntryClass;
+
+ private File userDirHasNotEntryClass;
+
+ // this was added to support the legacy tests of ClassPathPackagedProgramRetrieverTest in the
+ // refactored PackagedProgramRetrieverImplTest
+ private File userDirWithTestJob;
+
+ private File userDirWithNoJarButTextFile;
+
+ private String originalSystemClasspath;
+
+ @Override
+ public void before() throws IOException {
+ temporaryFolder.create();
+
+ userDirHasTwoEntryClasses =
+ temporaryFolder.newFolder("_test_user_dir_has_two_entry_classes");
+ copyJar(JOB_JAR_PATH, userDirHasTwoEntryClasses);
+ copyJar(TEST_JOB_JAR_PATH, userDirHasTwoEntryClasses);
+ copyJar(JOB_LIB_JAR_PATH, userDirHasTwoEntryClasses);
+ createTestFile(userDirHasTwoEntryClasses);
+
+ userDirHasEntryClass = temporaryFolder.newFolder("_test_user_dir_has_entry_class");
+ copyJar(JOB_JAR_PATH, userDirHasEntryClass);
+ copyJar(JOB_LIB_JAR_PATH, userDirHasEntryClass);
+ createTestFile(userDirHasEntryClass);
+
+ userDirHasNotEntryClass = temporaryFolder.newFolder("_test_user_dir_has_not_entry_class");
+ copyJar(JOB_LIB_JAR_PATH, userDirHasNotEntryClass);
+ createTestFile(userDirHasNotEntryClass);
+
+ userDirWithTestJob = temporaryFolder.newFolder("_with_test_job");
+ copyJar(TEST_JOB_JAR_PATH, userDirWithTestJob);
+
+ userDirWithNoJarButTextFile = temporaryFolder.newFolder("_with_no_jar");
+ createTestFile(userDirWithNoJarButTextFile);
+ }
+
+ private static void copyJar(Path sourcePath, File targetDir) throws IOException {
+ Files.copy(sourcePath, targetDir.toPath().resolve(sourcePath.toFile().getName()));
+ }
+
+ private static void createTestFile(File targetDir) throws IOException {
+ Files.createFile(targetDir.toPath().resolve("test.txt"));
+ }
+
+ @Override
+ protected void after() {
+ temporaryFolder.delete();
+ resetSystemClasspath();
+ }
+
+ public String getJobClassName() {
+ return extractEntryClassNameFromJar(JOB_JAR_PATH.toFile());
+ }
+
+ public String getTestJobClassName() {
+ return extractEntryClassNameFromJar(getTestJobJar());
+ }
Review comment:
these really need better names.
##########
File path: flink-clients/src/main/java/org/apache/flink/client/deployment/application/EntryClassInformationProvider.java
##########
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.client.deployment.application;
+
+import org.apache.flink.client.program.PackagedProgramRetrieverImpl;
+
+import java.io.File;
+import java.util.Optional;
+
+/**
+ * {@code EntryClassInformationProvider} provides information about the entry class that shall be
+ * used by the {@link PackagedProgramRetrieverImpl}.
+ */
+public interface EntryClassInformationProvider {
+
+ /**
+ * Returns the {@link File} referring to the Jar file that contains the job class or no {@code
+ * File} if the job class is located on some classpath.
+ *
+ * @return The {@code File} referring to the job's Jar archive or an empty {@code Optional} in
+ * case the job shall be extracted from the some classpath.
Review comment:
```suggestion
* case the job shall be extracted from the classpath.
```
##########
File path: flink-clients/src/test/java/org/apache/flink/client/deployment/application/FromClasspathEntryClassInformationProviderTest.java
##########
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.client.deployment.application;
+
+import org.apache.flink.client.testjar.ClasspathProvider;
+import org.apache.flink.util.FlinkException;
+import org.apache.flink.util.TestLogger;
+
+import org.apache.commons.io.FilenameUtils;
+import org.hamcrest.CoreMatchers;
+import org.hamcrest.collection.IsIterableContainingInAnyOrder;
+import org.hamcrest.core.IsCollectionContaining;
+import org.junit.Rule;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.Collection;
+import java.util.NoSuchElementException;
+import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.core.Is.is;
+
+/**
+ * {@code FromClasspathEntryClassInformationProviderTest} tests {@link
+ * FromClasspathEntryClassInformationProvider}.
+ */
+public class FromClasspathEntryClassInformationProviderTest extends TestLogger {
+
+ @Rule public final ClasspathProvider classpathProvider = new ClasspathProvider();
+
+ @Test
+ public void testJobClassOnUserClasspathWithExplicitJobClassName()
+ throws MalformedURLException, FlinkException {
+ FromClasspathEntryClassInformationProvider testInstance =
+ FromClasspathEntryClassInformationProvider.create(
+ classpathProvider.getJobClassName(),
+ classpathProvider.getURLUserClasspathWithEntryClass());
+
+ assertThat(testInstance.getJobClassName().isPresent(), is(true));
+ assertThat(testInstance.getJobClassName().get(), is(classpathProvider.getJobClassName()));
+ assertThat(testInstance.getJarFile().isPresent(), is(false));
+ }
+
+ @Test(expected = FlinkException.class)
+ public void testJobClassOnUserClasspathWithOnlyTestFileOnClasspath()
+ throws MalformedURLException, FlinkException {
+ // we want to check that the right exception is thrown if the user classpath is empty
+ FromClasspathEntryClassInformationProvider.create(
+ "SomeJobClassName", classpathProvider.getURLUserClasspathWithOnlyTextFile());
+ }
+
+ @Test(expected = NullPointerException.class)
+ public void testJobClassOnUserClasspathWithMissingJobClassName()
+ throws MalformedURLException, FlinkException {
+ FromClasspathEntryClassInformationProvider.create(
+ null, classpathProvider.getURLUserClasspathWithEntryClass());
+ }
+
+ @Test(expected = NullPointerException.class)
+ public void testJobClassOnUserClasspathWithMissingUserClasspath() throws FlinkException {
+ FromClasspathEntryClassInformationProvider.create("jobClassName", null);
+ }
+
+ @Test
+ public void testJobClassOnUserClasspathWithoutExplicitJobClassName() throws IOException {
+ FromClasspathEntryClassInformationProvider testInstance =
+ FromClasspathEntryClassInformationProvider.createFromClasspath(
+ classpathProvider.getURLUserClasspathWithEntryClass());
+
+ assertThat(testInstance.getJobClassName().isPresent(), is(true));
+ assertThat(testInstance.getJobClassName().get(), is(classpathProvider.getJobClassName()));
+ assertThat(testInstance.getJarFile().isPresent(), is(false));
+ }
+
+ @Test(expected = NoSuchElementException.class)
+ public void testMissingJobClassOnUserClasspathWithoutExplicitJobClassName() throws IOException {
+ FromClasspathEntryClassInformationProvider.createFromClasspath(
+ classpathProvider.getURLUserClasspathWithoutEntryClass());
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testTooManyMainMethodsOnUserClasspath() throws IOException {
+ FromClasspathEntryClassInformationProvider.createFromClasspath(
+ classpathProvider.getURLUserClasspathWithTwoEntryClasses());
+ }
+
+ @Test(expected = NullPointerException.class)
+ public void testJobClassOnUserClasspathWithoutExplicitJobClassNameAndMissingUserClasspath()
+ throws IOException {
+ FromClasspathEntryClassInformationProvider.createFromClasspath(null);
+ }
+
+ @Test
+ public void testJobClassNameFromSystemClasspath() throws IOException {
+ classpathProvider.setSystemClasspathWithEntryClass();
+ FromClasspathEntryClassInformationProvider testInstance =
+ FromClasspathEntryClassInformationProvider.createFromSystemClasspath();
+ assertThat(testInstance.getJobClassName().isPresent(), is(true));
+ assertThat(testInstance.getJobClassName().get(), is(classpathProvider.getJobClassName()));
+ assertThat(testInstance.getJarFile().isPresent(), is(false));
+ }
+
+ @Test(expected = NoSuchElementException.class)
+ public void testMissingJobClassNameFromSystemClasspath() throws IOException {
+ classpathProvider.setSystemClasspathWithoutEntryClass();
+ FromClasspathEntryClassInformationProvider.createFromSystemClasspath();
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testTooManyMainMethodsOnSystemClasspath() throws IOException {
+ classpathProvider.setSystemClasspathWithTwoEntryClasses();
+ FromClasspathEntryClassInformationProvider.createFromSystemClasspath();
+ }
+
+ @Test
+ public void testJarFromSystemClasspathSanityCheck() {
Review comment:
what do we need this test for? If you want to check that we can properly load something from the system CP without any modifications to it, then why not put a jar on the test classpath and try loading a job from that.
##########
File path: flink-clients/src/test/java/org/apache/flink/client/testjar/ClasspathProvider.java
##########
@@ -0,0 +1,230 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.client.testjar;
+
+import org.apache.flink.client.deployment.application.JarManifestParser;
+import org.apache.flink.client.program.PackagedProgram;
+
+import org.junit.rules.ExternalResource;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
+
+/** {@code ClasspathProvider} offers methods to generate classpaths based on actual jars. */
+public class ClasspathProvider extends ExternalResource {
+
+ private static final String CLASSPATH_PROPERTY_NAME = "java.class.path";
+
+ private static final Path TEST_JOB_JAR_PATH = Paths.get("target", "maven-test-jar.jar");
+
+ private static final Path JOB_JAR_PATH =
+ Paths.get("target", "maven-test-user-classloader-job-jar.jar");
+ private static final Path JOB_LIB_JAR_PATH =
+ Paths.get("target", "maven-test-user-classloader-job-lib-jar.jar");
+
+ private final TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+ private File userDirHasTwoEntryClasses;
+
+ private File userDirHasEntryClass;
+
+ private File userDirHasNotEntryClass;
+
+ // this was added to support the legacy tests of ClassPathPackagedProgramRetrieverTest in the
+ // refactored PackagedProgramRetrieverImplTest
+ private File userDirWithTestJob;
+
+ private File userDirWithNoJarButTextFile;
+
+ private String originalSystemClasspath;
+
+ @Override
+ public void before() throws IOException {
+ temporaryFolder.create();
+
+ userDirHasTwoEntryClasses =
Review comment:
nit: seems weird that a more complex case is handled before the plain case of 1 entry class
##########
File path: flink-clients/src/main/java/org/apache/flink/client/deployment/application/EntryClassInformationProvider.java
##########
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.client.deployment.application;
+
+import org.apache.flink.client.program.PackagedProgramRetrieverImpl;
+
+import java.io.File;
+import java.util.Optional;
+
+/**
+ * {@code EntryClassInformationProvider} provides information about the entry class that shall be
+ * used by the {@link PackagedProgramRetrieverImpl}.
Review comment:
Seems weird that this interface has a reference to an implementation of another interface.
##########
File path: flink-clients/src/test/java/org/apache/flink/client/deployment/application/FromJarEntryClassInformationProviderTest.java
##########
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.client.deployment.application;
+
+import org.apache.flink.client.program.PackagedProgramUtils;
+import org.apache.flink.configuration.ConfigConstants;
+import org.apache.flink.util.TestLogger;
+
+import org.junit.Assume;
+import org.junit.Test;
+
+import java.io.File;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.MatcherAssert.assertThat;
+
+/**
+ * {@code FromJarEntryClassInformationProviderTest} tests {@link
+ * FromJarEntryClassInformationProvider}.
+ */
+public class FromJarEntryClassInformationProviderTest extends TestLogger {
+
+ @Test
+ public void testCustomJarFile() {
+ final File jarFile = new File("some/path/to/jar");
+ final String jobClassName = "JobClassName";
+ final FromJarEntryClassInformationProvider testInstance =
+ FromJarEntryClassInformationProvider.createFromCustomJar(jarFile, jobClassName);
+
+ assertThat(testInstance.getJarFile().isPresent(), is(true));
+ assertThat(testInstance.getJarFile().get(), is(jarFile));
+ assertThat(testInstance.getJobClassName().isPresent(), is(true));
+ assertThat(testInstance.getJobClassName().get(), is(jobClassName));
+ }
+
+ @Test
+ public void testMissingJar() {
+ final String jobClassName = "JobClassName";
+ final EntryClassInformationProvider testInstance =
+ FromJarEntryClassInformationProvider.createFromCustomJar(null, jobClassName);
+ assertThat(testInstance.getJarFile().isPresent(), is(false));
+ assertThat(testInstance.getJobClassName().isPresent(), is(true));
+ assertThat(testInstance.getJobClassName().get(), is(jobClassName));
+ }
+
+ @Test
+ public void testMissingJobClassName() {
+ final File jarFile = new File("some/path/to/jar");
+ final EntryClassInformationProvider testInstance =
+ FromJarEntryClassInformationProvider.createFromCustomJar(jarFile, null);
+ assertThat(testInstance.getJarFile().isPresent(), is(true));
+ assertThat(testInstance.getJarFile().get(), is(jarFile));
+ assertThat(testInstance.getJobClassName().isPresent(), is(false));
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testEitherJobClassNameOrJarHasToBeSet() {
+ FromJarEntryClassInformationProvider.createFromCustomJar(null, null);
+ }
+
+ @Test
+ public void testPythonJarFile() {
Review comment:
I think it's a highly questionable to add a test, then effectively disable it, and then file a ticket for fixing it before this PR is even merged.
Instead, I would not have added this test, and opened a ticket _afterwards_ for adding a test for the python side of things.
##########
File path: flink-clients/src/test/java/org/apache/flink/client/testjar/ClasspathProvider.java
##########
@@ -0,0 +1,230 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.client.testjar;
+
+import org.apache.flink.client.deployment.application.JarManifestParser;
+import org.apache.flink.client.program.PackagedProgram;
+
+import org.junit.rules.ExternalResource;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
+
+/** {@code ClasspathProvider} offers methods to generate classpaths based on actual jars. */
+public class ClasspathProvider extends ExternalResource {
+
+ private static final String CLASSPATH_PROPERTY_NAME = "java.class.path";
+
+ private static final Path TEST_JOB_JAR_PATH = Paths.get("target", "maven-test-jar.jar");
+
+ private static final Path JOB_JAR_PATH =
+ Paths.get("target", "maven-test-user-classloader-job-jar.jar");
+ private static final Path JOB_LIB_JAR_PATH =
+ Paths.get("target", "maven-test-user-classloader-job-lib-jar.jar");
+
+ private final TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+ private File userDirHasTwoEntryClasses;
+
+ private File userDirHasEntryClass;
+
+ private File userDirHasNotEntryClass;
+
+ // this was added to support the legacy tests of ClassPathPackagedProgramRetrieverTest in the
+ // refactored PackagedProgramRetrieverImplTest
+ private File userDirWithTestJob;
+
+ private File userDirWithNoJarButTextFile;
+
+ private String originalSystemClasspath;
+
+ @Override
+ public void before() throws IOException {
+ temporaryFolder.create();
+
+ userDirHasTwoEntryClasses =
+ temporaryFolder.newFolder("_test_user_dir_has_two_entry_classes");
+ copyJar(JOB_JAR_PATH, userDirHasTwoEntryClasses);
+ copyJar(TEST_JOB_JAR_PATH, userDirHasTwoEntryClasses);
+ copyJar(JOB_LIB_JAR_PATH, userDirHasTwoEntryClasses);
+ createTestFile(userDirHasTwoEntryClasses);
+
+ userDirHasEntryClass = temporaryFolder.newFolder("_test_user_dir_has_entry_class");
+ copyJar(JOB_JAR_PATH, userDirHasEntryClass);
+ copyJar(JOB_LIB_JAR_PATH, userDirHasEntryClass);
+ createTestFile(userDirHasEntryClass);
+
+ userDirHasNotEntryClass = temporaryFolder.newFolder("_test_user_dir_has_not_entry_class");
+ copyJar(JOB_LIB_JAR_PATH, userDirHasNotEntryClass);
+ createTestFile(userDirHasNotEntryClass);
+
+ userDirWithTestJob = temporaryFolder.newFolder("_with_test_job");
+ copyJar(TEST_JOB_JAR_PATH, userDirWithTestJob);
+
+ userDirWithNoJarButTextFile = temporaryFolder.newFolder("_with_no_jar");
+ createTestFile(userDirWithNoJarButTextFile);
+ }
+
+ private static void copyJar(Path sourcePath, File targetDir) throws IOException {
+ Files.copy(sourcePath, targetDir.toPath().resolve(sourcePath.toFile().getName()));
+ }
+
+ private static void createTestFile(File targetDir) throws IOException {
+ Files.createFile(targetDir.toPath().resolve("test.txt"));
+ }
+
+ @Override
+ protected void after() {
+ temporaryFolder.delete();
+ resetSystemClasspath();
+ }
+
+ public String getJobClassName() {
+ return extractEntryClassNameFromJar(JOB_JAR_PATH.toFile());
+ }
+
+ public String getTestJobClassName() {
+ return extractEntryClassNameFromJar(getTestJobJar());
+ }
+
+ public File getTestJobJar() {
+ return TEST_JOB_JAR_PATH.toFile();
+ }
+
+ private static String extractEntryClassNameFromJar(File f) {
+ try {
+ return JarManifestParser.findFirstManifestAttribute(
+ f, PackagedProgram.MANIFEST_ATTRIBUTE_MAIN_CLASS)
+ .orElseThrow(
+ () ->
+ new IllegalArgumentException(
+ "The passed file does not contain a main class: "
+ + f.getAbsolutePath()));
+ } catch (Throwable t) {
+ throw new AssertionError(
+ "Something went wrong with retrieving the main class from "
+ + f.getAbsolutePath(),
+ t);
+ }
+ }
+
+ public File getUserDirectoryWithNoEntryClass() {
+ return userDirHasNotEntryClass;
+ }
+
+ public File getUserDirectoryWithOneEntryClass() {
+ return userDirHasEntryClass;
+ }
+
+ public File getUserDirectoryWithTwoEntryClasses() {
+ return userDirHasTwoEntryClasses;
+ }
+
+ public File getUserDirectoryWithTestJob() {
+ return userDirWithTestJob;
+ }
+
+ public String[] getTestJobArgs(String expectedSuffix) {
+ return new String[] {"--arg", expectedSuffix};
+ }
+
+ public Iterable<URL> getURLUserClasspathWithTwoEntryClasses() throws MalformedURLException {
+ return getURLUserClasspath(userDirHasTwoEntryClasses);
+ }
+
+ public Iterable<URL> getURLUserClasspathWithEntryClass() throws MalformedURLException {
+ return getURLUserClasspath(userDirHasEntryClass);
+ }
+
+ public Iterable<URL> getURLUserClasspathWithoutEntryClass() throws MalformedURLException {
+ return getURLUserClasspath(userDirHasNotEntryClass);
+ }
+
+ public Iterable<URL> getURLUserClasspathWithTestJob() throws MalformedURLException {
+ return getURLUserClasspath(userDirWithTestJob);
+ }
+
+ public Iterable<URL> getURLUserClasspathWithOnlyTextFile() throws MalformedURLException {
+ return getURLUserClasspath(userDirWithNoJarButTextFile);
+ }
+
+ public void setSystemClasspathWithTwoEntryClasses() throws MalformedURLException {
+ setSystemClasspath(getURLUserClasspathWithTwoEntryClasses());
+ }
+
+ public void setSystemClasspathWithEntryClass() throws MalformedURLException {
+ setSystemClasspath(getURLUserClasspathWithEntryClass());
+ }
+
+ public void setSystemClasspathWithoutEntryClass() throws MalformedURLException {
+ setSystemClasspath(getURLUserClasspathWithoutEntryClass());
+ }
+
+ public void setSystemClasspathWithTestJob() throws MalformedURLException {
+ setSystemClasspath(getURLUserClasspathWithTestJob());
+ }
+
+ private void setSystemClasspath(Iterable<URL> classpath) {
Review comment:
If this is called twice the original classpath will not be properly restored; capture the original value in the constructor instead
##########
File path: flink-clients/src/test/java/org/apache/flink/client/program/PackagedProgramRetrieverImplTest.java
##########
@@ -0,0 +1,573 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.client.program;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.api.dag.Pipeline;
+import org.apache.flink.client.deployment.application.EntryClassInformationProvider;
+import org.apache.flink.client.deployment.executors.PipelineExecutorUtils;
+import org.apache.flink.client.testjar.ClasspathProvider;
+import org.apache.flink.configuration.ConfigConstants;
+import org.apache.flink.configuration.ConfigUtils;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.CoreOptions;
+import org.apache.flink.configuration.PipelineOptions;
+import org.apache.flink.configuration.PipelineOptionsInternal;
+import org.apache.flink.runtime.execution.librarycache.FlinkUserCodeClassLoaders;
+import org.apache.flink.runtime.jobgraph.JobGraph;
+import org.apache.flink.runtime.jobgraph.SavepointRestoreSettings;
+import org.apache.flink.util.ChildFirstClassLoader;
+import org.apache.flink.util.ExceptionUtils;
+import org.apache.flink.util.FileUtils;
+import org.apache.flink.util.FlinkException;
+import org.apache.flink.util.TestLogger;
+
+import org.hamcrest.collection.IsIterableContainingInAnyOrder;
+import org.hamcrest.core.IsInstanceOf;
+import org.junit.Assume;
+import org.junit.Ignore;
+import org.junit.Rule;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.core.Is.is;
+import static org.hamcrest.core.IsNot.not;
+import static org.junit.Assert.fail;
+
+/** {@code PackagedProgramRetrieverImplTest} tests {@link PackagedProgramRetrieverImpl}. */
+public class PackagedProgramRetrieverImplTest extends TestLogger {
+
+ @Rule public ClasspathProvider classpathProvider = new ClasspathProvider();
+
+ @Test
+ public void testDeriveEntryClassInformationForPythonBasedOnJobName() throws FlinkException {
+ // TODO: see FLINK-23154
+ Assume.assumeTrue(
+ "This tests only succeeds if " + ConfigConstants.ENV_FLINK_OPT_DIR + " is set.",
+ System.getProperty(ConfigConstants.ENV_FLINK_OPT_DIR) != null);
+ final EntryClassInformationProvider informationProvider =
+ PackagedProgramRetrieverImpl.createEntryClassInformationProvider(
+ null,
+ Collections.emptyList(),
+ null,
+ PackagedProgramUtils.getPythonDriverClassName(),
+ new String[0]);
+ assertThat(informationProvider.getJobClassName().isPresent(), is(true));
+ assertThat(
+ informationProvider.getJobClassName().get(),
+ is(PackagedProgramUtils.getPythonDriverClassName()));
+ assertThat(informationProvider.getJarFile().isPresent(), is(true));
+ assertThat(
+ informationProvider.getJarFile().get(),
+ is(PackagedProgramUtils.getPythonJar().getFile()));
+ }
+
+ @Test
+ public void testDeriveEntryClassInformationForPythonBasedOnParameter() throws FlinkException {
+ // TODO: see FLINK-23154
+ Assume.assumeTrue(
+ "This tests only succeeds if " + ConfigConstants.ENV_FLINK_OPT_DIR + " is set.",
+ System.getProperty(ConfigConstants.ENV_FLINK_OPT_DIR) != null);
+
+ final EntryClassInformationProvider informationProvider =
+ PackagedProgramRetrieverImpl.createEntryClassInformationProvider(
+ null, Collections.emptyList(), null, null, new String[] {"--python"});
+
+ assertThat(informationProvider.getJobClassName().isPresent(), is(true));
+ assertThat(
+ informationProvider.getJobClassName().get(),
+ is(PackagedProgramUtils.getPythonDriverClassName()));
+ assertThat(informationProvider.getJarFile().isPresent(), is(true));
+ assertThat(
+ informationProvider.getJarFile().get(),
+ is(PackagedProgramUtils.getPythonJar().getFile()));
+ }
+
+ @Test
+ public void testDeriveEntryClassInformationForCustomJar()
+ throws FlinkException, MalformedURLException {
+ // make loading from system classpath fail to make sure that it's not triggered
+ classpathProvider.setSystemClasspathWithTwoEntryClasses();
+
+ final String jobClassName = "SomeJobClassName";
+ final File jarFile = new File("some/jar/file.jar");
+ final EntryClassInformationProvider informationProvider =
+ PackagedProgramRetrieverImpl.createEntryClassInformationProvider(
+ null, null, jarFile, jobClassName, new String[0]);
+ assertThat(informationProvider.getJobClassName().isPresent(), is(true));
+ assertThat(informationProvider.getJobClassName().get(), is(jobClassName));
+ assertThat(informationProvider.getJarFile().isPresent(), is(true));
+ assertThat(informationProvider.getJarFile().get(), is(jarFile));
+ }
+
+ @Test
+ // TODO: we might want to change this behavior triggering a failure here
+ public void testDeriveEntryClassInformationFromSystemClasspathWithNonExistingJobClassName()
+ throws IOException, FlinkException {
+ classpathProvider.setSystemClasspathWithEntryClass();
+
+ final String jobClassName = "SomeJobClassNotBeingOnTheSystemClasspath";
+ final EntryClassInformationProvider informationProvider =
+ PackagedProgramRetrieverImpl.createEntryClassInformationProvider(
+ null, null, null, jobClassName, new String[0]);
+ assertThat(informationProvider.getJobClassName().isPresent(), is(true));
+ assertThat(informationProvider.getJobClassName().get(), is(jobClassName));
+ assertThat(informationProvider.getJarFile().isPresent(), is(false));
+ }
+
+ @Test
+ public void testDeriveEntryClassInformationFromSystemClasspathWithExistingJobClassName()
+ throws IOException, FlinkException {
+ classpathProvider.setSystemClasspathWithEntryClass();
+
+ final EntryClassInformationProvider informationProvider =
+ PackagedProgramRetrieverImpl.createEntryClassInformationProvider(
+ null, null, null, classpathProvider.getJobClassName(), new String[0]);
+ assertThat(informationProvider.getJobClassName().isPresent(), is(true));
+ assertThat(
+ informationProvider.getJobClassName().get(),
+ is(classpathProvider.getJobClassName()));
+ assertThat(informationProvider.getJarFile().isPresent(), is(false));
+ }
+
+ @Test
+ public void testDeriveEntryClassInformationFromSystemClasspathExtractingTheJobClassFromThere()
+ throws IOException, FlinkException {
+ classpathProvider.setSystemClasspathWithEntryClass();
+
+ final EntryClassInformationProvider informationProvider =
+ PackagedProgramRetrieverImpl.createEntryClassInformationProvider(
+ null, null, null, null, new String[0]);
+ assertThat(informationProvider.getJobClassName().isPresent(), is(true));
+ assertThat(
+ informationProvider.getJobClassName().get(),
+ is(classpathProvider.getJobClassName()));
+ assertThat(informationProvider.getJarFile().isPresent(), is(false));
+ }
+
+ @Test
+ public void testDeriveEntryClassInformationFromClasspathWithJobClass()
+ throws IOException, FlinkException {
+ final EntryClassInformationProvider informationProvider =
+ PackagedProgramRetrieverImpl.createEntryClassInformationProvider(
+ // the user directory must be specified
+ classpathProvider.getUserDirectoryWithTwoEntryClasses(),
+ // the user classpath is derived from the user directory outside of the
+ // method
+ classpathProvider.getURLUserClasspathWithTwoEntryClasses(),
+ null,
+ // we have to specify the job class - otherwise the call would fail due to
+ // two main method being present
+ classpathProvider.getJobClassName(),
+ new String[0]);
+ assertThat(informationProvider.getJobClassName().isPresent(), is(true));
+ assertThat(
+ informationProvider.getJobClassName().get(),
+ is(classpathProvider.getJobClassName()));
+ assertThat(informationProvider.getJarFile().isPresent(), is(false));
+ }
+
+ @Test
+ public void testDeriveEntryClassInformationFromClasspathWithNoJobClass()
+ throws IOException, FlinkException {
+ final EntryClassInformationProvider informationProvider =
+ PackagedProgramRetrieverImpl.createEntryClassInformationProvider(
+ // the user directory must be specified
+ classpathProvider.getUserDirectoryWithOneEntryClass(),
+ // the user classpath is derived from the user directory outside of the
+ // method
+ classpathProvider.getURLUserClasspathWithEntryClass(),
+ null,
+ // no job class name is specified which enables looking for the entry class
+ // on the user classpath
+ null,
+ new String[0]);
+ assertThat(informationProvider.getJobClassName().isPresent(), is(true));
+ assertThat(
+ informationProvider.getJobClassName().get(),
+ is(classpathProvider.getJobClassName()));
+ assertThat(informationProvider.getJarFile().isPresent(), is(false));
+ }
+
+ @Test
+ public void testCreateWithUserLibDir() throws FlinkException {
+ final PackagedProgramRetriever retriever =
+ PackagedProgramRetrieverImpl.create(
+ classpathProvider.getUserDirectoryWithOneEntryClass(),
+ null,
+ classpathProvider.getJobClassName(),
+ new String[0],
+ new Configuration());
+
+ // the right information is picked up without any error
+ assertThat(
+ retriever.getPackagedProgram().getMainClassName(),
+ is(classpathProvider.getJobClassName()));
+ }
+
+ @Test
+ public void testJobGraphRetrieval()
+ throws IOException, FlinkException, ProgramInvocationException {
+ final int parallelism = 42;
+ final JobID jobId = new JobID();
+
+ final Configuration configuration = new Configuration();
+ configuration.setInteger(CoreOptions.DEFAULT_PARALLELISM, parallelism);
+ configuration.set(PipelineOptionsInternal.PIPELINE_FIXED_JOB_ID, jobId.toHexString());
+
+ final String expectedSuffix = "suffix";
+ final PackagedProgramRetriever retriever =
+ PackagedProgramRetrieverImpl.create(
+ null,
+ classpathProvider.getTestJobClassName(),
+ classpathProvider.getTestJobArgs(expectedSuffix),
+ new Configuration());
+
+ final JobGraph jobGraph = retrieveJobGraph(retriever, configuration);
+
+ assertThat(
+ jobGraph.getName(),
+ is(classpathProvider.getTestJobClassName() + "-" + expectedSuffix));
+ assertThat(jobGraph.getSavepointRestoreSettings(), is(SavepointRestoreSettings.none()));
+ assertThat(jobGraph.getMaximumParallelism(), is(parallelism));
+ assertThat(jobGraph.getJobID(), is(jobId));
+ }
+
+ @Test
+ public void testJobGraphRetrievalFromJar()
+ throws IOException, FlinkException, ProgramInvocationException {
+ final String expectedSuffix = "suffix";
+ final PackagedProgramRetriever retrieverUnderTest =
+ PackagedProgramRetrieverImpl.create(
+ classpathProvider.getUserDirectoryWithTestJob(),
+ null,
+ null,
+ classpathProvider.getTestJobArgs(expectedSuffix),
+ new Configuration());
+
+ final JobGraph jobGraph = retrieveJobGraph(retrieverUnderTest, new Configuration());
+
+ assertThat(
+ jobGraph.getName(),
+ is(classpathProvider.getTestJobClassName() + "-" + expectedSuffix));
+ }
+
+ @Test
+ public void testJobGraphRetrievalJobClassNameHasPrecedenceOverClasspath()
+ throws IOException, FlinkException, ProgramInvocationException {
+ // TODO: I don't get this test: It feels like it does not add any extra value - it does the
+ // same like testTooManyEntryClassesOnSystemClasspath
+ final String expectedSuffix = "suffix";
+ final PackagedProgramRetriever retrieverUnderTest =
+ // Both a class name is specified and a JAR "is" on the class path
+ // The class name should have precedence.
+ PackagedProgramRetrieverImpl.create(
+ null,
+ classpathProvider.getTestJobClassName(),
+ classpathProvider.getTestJobArgs(expectedSuffix),
+ new Configuration());
+
+ final JobGraph jobGraph = retrieveJobGraph(retrieverUnderTest, new Configuration());
+
+ assertThat(jobGraph.getName(), is(classpathProvider.getTestJobClassName() + "-suffix"));
+ }
+
+ @Test
+ public void testSavepointRestoreSettings()
+ throws FlinkException, IOException, ProgramInvocationException {
+ final Configuration configuration = new Configuration();
+ final SavepointRestoreSettings savepointRestoreSettings =
+ SavepointRestoreSettings.forPath("foobar", true);
+ final JobID jobId = new JobID();
+
+ configuration.setString(PipelineOptionsInternal.PIPELINE_FIXED_JOB_ID, jobId.toHexString());
+ SavepointRestoreSettings.toConfiguration(savepointRestoreSettings, configuration);
+
+ final String expectedSuffix = "suffix";
+ final PackagedProgramRetriever retrieverUnderTest =
+ PackagedProgramRetrieverImpl.create(
+ null,
+ classpathProvider.getTestJobClassName(),
+ classpathProvider.getTestJobArgs(expectedSuffix),
+ new Configuration());
+
+ final JobGraph jobGraph = retrieveJobGraph(retrieverUnderTest, configuration);
+
+ assertThat(jobGraph.getSavepointRestoreSettings(), is(savepointRestoreSettings));
+ assertThat(jobGraph.getJobID(), is(jobId));
+ }
+
+ @Test
+ public void testFailIfJobDirDoesNotHaveEntryClass() {
+ try {
+ PackagedProgramRetrieverImpl.create(
+ classpathProvider.getUserDirectoryWithNoEntryClass(),
+ classpathProvider.getTestJobClassName(),
+ classpathProvider.getTestJobArgs("suffix"),
+ new Configuration());
+ fail("This case should throw exception !");
+ } catch (FlinkException e) {
+ assertThat(
+ ExceptionUtils.findThrowableWithMessage(
+ e,
+ String.format(
+ "Could not find the provided job class (%s) in the user lib directory.",
+ classpathProvider.getTestJobClassName()))
+ .isPresent(),
+ is(true));
+ }
+ }
+
+ // TODO: this test checks the same code path as
+ // testDeriveEntryClassInformationFromSystemClasspathWithNonExistingJobClassName
+ // We should make it fail early if the class is not present on the system classpath
+ // Right now, the test is failing because no error is thrown
+ @Ignore
+ @Test(expected = FlinkException.class)
+ public void testEntryClassNotFoundOnSystemClasspath() throws FlinkException {
+ PackagedProgramRetrieverImpl.create(
+ null, "NotExistingClass", new String[0], new Configuration());
+ }
+
+ @Test(expected = FlinkException.class)
+ public void testEntryClassNotFoundOnUserClasspath() throws FlinkException {
+ PackagedProgramRetrieverImpl.create(
+ classpathProvider.getUserDirectoryWithNoEntryClass(),
+ classpathProvider.getTestJobClassName(),
+ new String[0],
+ new Configuration());
+ }
+
+ @Test(expected = FlinkException.class)
+ public void testTooManyEntryClassesOnSystemClasspath() throws FlinkException {
+ PackagedProgramRetrieverImpl.create(null, null, new String[0], new Configuration());
+ }
+
+ @Test
+ public void testTooManyEntryClassesOnUserClasspath() throws FlinkException, IOException {
Review comment:
So if 2 entry points are on the user CP we don't fail?
##########
File path: flink-clients/src/test/java/org/apache/flink/client/testjar/ClasspathProvider.java
##########
@@ -0,0 +1,221 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.client.testjar;
+
+import org.apache.flink.client.deployment.application.JarManifestParser;
+import org.apache.flink.client.program.PackagedProgram;
+
+import org.junit.rules.ExternalResource;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
+
+/** {@code ClasspathProvider} offers methods to generate classpaths based on actual jars. */
+public class ClasspathProvider extends ExternalResource {
Review comment:
The name of the class was a bit surprising; I expected it at first sight to be some `Iterable<URL>/String` provider, not a hodgepodge of getters and setters.
##########
File path: flink-clients/src/test/java/org/apache/flink/client/program/PackagedProgramRetrieverImplTest.java
##########
@@ -0,0 +1,590 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.client.program;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.api.dag.Pipeline;
+import org.apache.flink.client.deployment.application.EntryClassInformationProvider;
+import org.apache.flink.client.deployment.executors.PipelineExecutorUtils;
+import org.apache.flink.client.testjar.ClasspathProvider;
+import org.apache.flink.configuration.ConfigConstants;
+import org.apache.flink.configuration.ConfigUtils;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.CoreOptions;
+import org.apache.flink.configuration.PipelineOptions;
+import org.apache.flink.configuration.PipelineOptionsInternal;
+import org.apache.flink.runtime.execution.librarycache.FlinkUserCodeClassLoaders;
+import org.apache.flink.runtime.jobgraph.JobGraph;
+import org.apache.flink.runtime.jobgraph.SavepointRestoreSettings;
+import org.apache.flink.util.ChildFirstClassLoader;
+import org.apache.flink.util.ExceptionUtils;
+import org.apache.flink.util.FileUtils;
+import org.apache.flink.util.FlinkException;
+import org.apache.flink.util.TestLogger;
+
+import org.hamcrest.collection.IsIterableContainingInAnyOrder;
+import org.hamcrest.core.IsInstanceOf;
+import org.junit.Assume;
+import org.junit.Rule;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.core.Is.is;
+import static org.hamcrest.core.IsNot.not;
+import static org.junit.Assert.fail;
+
+/** {@code PackagedProgramRetrieverImplTest} tests {@link PackagedProgramRetrieverImpl}. */
+public class PackagedProgramRetrieverImplTest extends TestLogger {
+
+ @Rule public ClasspathProvider classpathProvider = new ClasspathProvider();
+
+ @Test
+ public void deriveEntryClassInformationForPythonBasedOnJobName() throws FlinkException {
+ // TODO: see FLINK-23154
+ Assume.assumeTrue(
+ "This tests only succeeds if " + ConfigConstants.ENV_FLINK_OPT_DIR + " is set.",
+ System.getProperty(ConfigConstants.ENV_FLINK_OPT_DIR) != null);
+ final EntryClassInformationProvider informationProvider =
+ PackagedProgramRetrieverImpl.createEntryClassInformationProvider(
+ null,
+ Collections.emptyList(),
+ null,
+ PackagedProgramUtils.getPythonDriverClassName(),
+ new String[0]);
+ assertThat(
+ informationProvider.getJobClassName(),
+ is(PackagedProgramUtils.getPythonDriverClassName()));
+ assertThat(informationProvider.getJarFile().isPresent(), is(true));
+ assertThat(
+ informationProvider.getJarFile().get(),
+ is(PackagedProgramUtils.getPythonJar().getFile()));
+ }
+
+ @Test
+ public void deriveEntryClassInformationForPythonBasedOnParameter() throws FlinkException {
+ // TODO: see FLINK-23154
+ Assume.assumeTrue(
+ "This tests only succeeds if " + ConfigConstants.ENV_FLINK_OPT_DIR + " is set.",
+ System.getProperty(ConfigConstants.ENV_FLINK_OPT_DIR) != null);
+
+ final EntryClassInformationProvider informationProvider =
+ PackagedProgramRetrieverImpl.createEntryClassInformationProvider(
+ null, Collections.emptyList(), null, null, new String[] {"--python"});
+
+ assertThat(
+ informationProvider.getJobClassName(),
+ is(PackagedProgramUtils.getPythonDriverClassName()));
+ assertThat(informationProvider.getJarFile().isPresent(), is(true));
+ assertThat(
+ informationProvider.getJarFile().get(),
+ is(PackagedProgramUtils.getPythonJar().getFile()));
+ }
+
+ @Test
+ public void deriveEntryClassInformationForCustomJar()
+ throws FlinkException, MalformedURLException {
+ // make loading from system classpath fail to make sure that it's not triggered
+ classpathProvider.setSystemClasspathWithTwoEntryClasses();
+
+ final String jobClassName = "SomeJobClassName";
+ final File jarFile = new File("some/jar/file.jar");
+ final EntryClassInformationProvider informationProvider =
+ PackagedProgramRetrieverImpl.createEntryClassInformationProvider(
+ null, null, jarFile, jobClassName, new String[0]);
+ assertThat(informationProvider.getJobClassName(), is(jobClassName));
+ assertThat(informationProvider.getJarFile().isPresent(), is(true));
+ assertThat(informationProvider.getJarFile().get(), is(jarFile));
+ }
+
+ @Test
+ // TODO: we might want to change this behavior triggering a failure here
Review comment:
I don't understand the comment, please elaborate.
##########
File path: flink-clients/src/test/java/org/apache/flink/client/program/PackagedProgramRetrieverImplTest.java
##########
@@ -0,0 +1,573 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.client.program;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.api.dag.Pipeline;
+import org.apache.flink.client.deployment.application.EntryClassInformationProvider;
+import org.apache.flink.client.deployment.executors.PipelineExecutorUtils;
+import org.apache.flink.client.testjar.ClasspathProvider;
+import org.apache.flink.configuration.ConfigConstants;
+import org.apache.flink.configuration.ConfigUtils;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.CoreOptions;
+import org.apache.flink.configuration.PipelineOptions;
+import org.apache.flink.configuration.PipelineOptionsInternal;
+import org.apache.flink.runtime.execution.librarycache.FlinkUserCodeClassLoaders;
+import org.apache.flink.runtime.jobgraph.JobGraph;
+import org.apache.flink.runtime.jobgraph.SavepointRestoreSettings;
+import org.apache.flink.util.ChildFirstClassLoader;
+import org.apache.flink.util.ExceptionUtils;
+import org.apache.flink.util.FileUtils;
+import org.apache.flink.util.FlinkException;
+import org.apache.flink.util.TestLogger;
+
+import org.hamcrest.collection.IsIterableContainingInAnyOrder;
+import org.hamcrest.core.IsInstanceOf;
+import org.junit.Assume;
+import org.junit.Ignore;
+import org.junit.Rule;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.core.Is.is;
+import static org.hamcrest.core.IsNot.not;
+import static org.junit.Assert.fail;
+
+/** {@code PackagedProgramRetrieverImplTest} tests {@link PackagedProgramRetrieverImpl}. */
+public class PackagedProgramRetrieverImplTest extends TestLogger {
+
+ @Rule public ClasspathProvider classpathProvider = new ClasspathProvider();
+
+ @Test
+ public void testDeriveEntryClassInformationForPythonBasedOnJobName() throws FlinkException {
+ // TODO: see FLINK-23154
+ Assume.assumeTrue(
+ "This tests only succeeds if " + ConfigConstants.ENV_FLINK_OPT_DIR + " is set.",
+ System.getProperty(ConfigConstants.ENV_FLINK_OPT_DIR) != null);
+ final EntryClassInformationProvider informationProvider =
+ PackagedProgramRetrieverImpl.createEntryClassInformationProvider(
+ null,
+ Collections.emptyList(),
+ null,
+ PackagedProgramUtils.getPythonDriverClassName(),
+ new String[0]);
+ assertThat(informationProvider.getJobClassName().isPresent(), is(true));
+ assertThat(
+ informationProvider.getJobClassName().get(),
+ is(PackagedProgramUtils.getPythonDriverClassName()));
+ assertThat(informationProvider.getJarFile().isPresent(), is(true));
+ assertThat(
+ informationProvider.getJarFile().get(),
+ is(PackagedProgramUtils.getPythonJar().getFile()));
+ }
+
+ @Test
+ public void testDeriveEntryClassInformationForPythonBasedOnParameter() throws FlinkException {
+ // TODO: see FLINK-23154
+ Assume.assumeTrue(
+ "This tests only succeeds if " + ConfigConstants.ENV_FLINK_OPT_DIR + " is set.",
+ System.getProperty(ConfigConstants.ENV_FLINK_OPT_DIR) != null);
+
+ final EntryClassInformationProvider informationProvider =
+ PackagedProgramRetrieverImpl.createEntryClassInformationProvider(
+ null, Collections.emptyList(), null, null, new String[] {"--python"});
+
+ assertThat(informationProvider.getJobClassName().isPresent(), is(true));
+ assertThat(
+ informationProvider.getJobClassName().get(),
+ is(PackagedProgramUtils.getPythonDriverClassName()));
+ assertThat(informationProvider.getJarFile().isPresent(), is(true));
+ assertThat(
+ informationProvider.getJarFile().get(),
+ is(PackagedProgramUtils.getPythonJar().getFile()));
+ }
+
+ @Test
+ public void testDeriveEntryClassInformationForCustomJar()
+ throws FlinkException, MalformedURLException {
+ // make loading from system classpath fail to make sure that it's not triggered
+ classpathProvider.setSystemClasspathWithTwoEntryClasses();
+
+ final String jobClassName = "SomeJobClassName";
+ final File jarFile = new File("some/jar/file.jar");
+ final EntryClassInformationProvider informationProvider =
+ PackagedProgramRetrieverImpl.createEntryClassInformationProvider(
+ null, null, jarFile, jobClassName, new String[0]);
+ assertThat(informationProvider.getJobClassName().isPresent(), is(true));
+ assertThat(informationProvider.getJobClassName().get(), is(jobClassName));
+ assertThat(informationProvider.getJarFile().isPresent(), is(true));
+ assertThat(informationProvider.getJarFile().get(), is(jarFile));
+ }
+
+ @Test
+ // TODO: we might want to change this behavior triggering a failure here
+ public void testDeriveEntryClassInformationFromSystemClasspathWithNonExistingJobClassName()
+ throws IOException, FlinkException {
+ classpathProvider.setSystemClasspathWithEntryClass();
+
+ final String jobClassName = "SomeJobClassNotBeingOnTheSystemClasspath";
+ final EntryClassInformationProvider informationProvider =
+ PackagedProgramRetrieverImpl.createEntryClassInformationProvider(
+ null, null, null, jobClassName, new String[0]);
+ assertThat(informationProvider.getJobClassName().isPresent(), is(true));
+ assertThat(informationProvider.getJobClassName().get(), is(jobClassName));
+ assertThat(informationProvider.getJarFile().isPresent(), is(false));
+ }
+
+ @Test
+ public void testDeriveEntryClassInformationFromSystemClasspathWithExistingJobClassName()
+ throws IOException, FlinkException {
+ classpathProvider.setSystemClasspathWithEntryClass();
+
+ final EntryClassInformationProvider informationProvider =
+ PackagedProgramRetrieverImpl.createEntryClassInformationProvider(
+ null, null, null, classpathProvider.getJobClassName(), new String[0]);
+ assertThat(informationProvider.getJobClassName().isPresent(), is(true));
+ assertThat(
+ informationProvider.getJobClassName().get(),
+ is(classpathProvider.getJobClassName()));
+ assertThat(informationProvider.getJarFile().isPresent(), is(false));
+ }
+
+ @Test
+ public void testDeriveEntryClassInformationFromSystemClasspathExtractingTheJobClassFromThere()
+ throws IOException, FlinkException {
+ classpathProvider.setSystemClasspathWithEntryClass();
+
+ final EntryClassInformationProvider informationProvider =
+ PackagedProgramRetrieverImpl.createEntryClassInformationProvider(
+ null, null, null, null, new String[0]);
+ assertThat(informationProvider.getJobClassName().isPresent(), is(true));
+ assertThat(
+ informationProvider.getJobClassName().get(),
+ is(classpathProvider.getJobClassName()));
+ assertThat(informationProvider.getJarFile().isPresent(), is(false));
+ }
+
+ @Test
+ public void testDeriveEntryClassInformationFromClasspathWithJobClass()
+ throws IOException, FlinkException {
+ final EntryClassInformationProvider informationProvider =
+ PackagedProgramRetrieverImpl.createEntryClassInformationProvider(
+ // the user directory must be specified
+ classpathProvider.getUserDirectoryWithTwoEntryClasses(),
+ // the user classpath is derived from the user directory outside of the
+ // method
+ classpathProvider.getURLUserClasspathWithTwoEntryClasses(),
+ null,
+ // we have to specify the job class - otherwise the call would fail due to
+ // two main method being present
+ classpathProvider.getJobClassName(),
+ new String[0]);
+ assertThat(informationProvider.getJobClassName().isPresent(), is(true));
+ assertThat(
+ informationProvider.getJobClassName().get(),
+ is(classpathProvider.getJobClassName()));
+ assertThat(informationProvider.getJarFile().isPresent(), is(false));
+ }
+
+ @Test
+ public void testDeriveEntryClassInformationFromClasspathWithNoJobClass()
+ throws IOException, FlinkException {
+ final EntryClassInformationProvider informationProvider =
+ PackagedProgramRetrieverImpl.createEntryClassInformationProvider(
+ // the user directory must be specified
+ classpathProvider.getUserDirectoryWithOneEntryClass(),
+ // the user classpath is derived from the user directory outside of the
+ // method
+ classpathProvider.getURLUserClasspathWithEntryClass(),
Review comment:
which method are you referencing?
##########
File path: flink-clients/src/test/java/org/apache/flink/client/program/PackagedProgramRetrieverImplTest.java
##########
@@ -0,0 +1,573 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.client.program;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.api.dag.Pipeline;
+import org.apache.flink.client.deployment.application.EntryClassInformationProvider;
+import org.apache.flink.client.deployment.executors.PipelineExecutorUtils;
+import org.apache.flink.client.testjar.ClasspathProvider;
+import org.apache.flink.configuration.ConfigConstants;
+import org.apache.flink.configuration.ConfigUtils;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.CoreOptions;
+import org.apache.flink.configuration.PipelineOptions;
+import org.apache.flink.configuration.PipelineOptionsInternal;
+import org.apache.flink.runtime.execution.librarycache.FlinkUserCodeClassLoaders;
+import org.apache.flink.runtime.jobgraph.JobGraph;
+import org.apache.flink.runtime.jobgraph.SavepointRestoreSettings;
+import org.apache.flink.util.ChildFirstClassLoader;
+import org.apache.flink.util.ExceptionUtils;
+import org.apache.flink.util.FileUtils;
+import org.apache.flink.util.FlinkException;
+import org.apache.flink.util.TestLogger;
+
+import org.hamcrest.collection.IsIterableContainingInAnyOrder;
+import org.hamcrest.core.IsInstanceOf;
+import org.junit.Assume;
+import org.junit.Ignore;
+import org.junit.Rule;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.core.Is.is;
+import static org.hamcrest.core.IsNot.not;
+import static org.junit.Assert.fail;
+
+/** {@code PackagedProgramRetrieverImplTest} tests {@link PackagedProgramRetrieverImpl}. */
+public class PackagedProgramRetrieverImplTest extends TestLogger {
+
+ @Rule public ClasspathProvider classpathProvider = new ClasspathProvider();
+
+ @Test
+ public void testDeriveEntryClassInformationForPythonBasedOnJobName() throws FlinkException {
+ // TODO: see FLINK-23154
+ Assume.assumeTrue(
+ "This tests only succeeds if " + ConfigConstants.ENV_FLINK_OPT_DIR + " is set.",
+ System.getProperty(ConfigConstants.ENV_FLINK_OPT_DIR) != null);
+ final EntryClassInformationProvider informationProvider =
+ PackagedProgramRetrieverImpl.createEntryClassInformationProvider(
+ null,
+ Collections.emptyList(),
+ null,
+ PackagedProgramUtils.getPythonDriverClassName(),
+ new String[0]);
+ assertThat(informationProvider.getJobClassName().isPresent(), is(true));
+ assertThat(
+ informationProvider.getJobClassName().get(),
+ is(PackagedProgramUtils.getPythonDriverClassName()));
+ assertThat(informationProvider.getJarFile().isPresent(), is(true));
+ assertThat(
+ informationProvider.getJarFile().get(),
+ is(PackagedProgramUtils.getPythonJar().getFile()));
+ }
+
+ @Test
+ public void testDeriveEntryClassInformationForPythonBasedOnParameter() throws FlinkException {
+ // TODO: see FLINK-23154
+ Assume.assumeTrue(
+ "This tests only succeeds if " + ConfigConstants.ENV_FLINK_OPT_DIR + " is set.",
+ System.getProperty(ConfigConstants.ENV_FLINK_OPT_DIR) != null);
+
+ final EntryClassInformationProvider informationProvider =
+ PackagedProgramRetrieverImpl.createEntryClassInformationProvider(
+ null, Collections.emptyList(), null, null, new String[] {"--python"});
+
+ assertThat(informationProvider.getJobClassName().isPresent(), is(true));
+ assertThat(
+ informationProvider.getJobClassName().get(),
+ is(PackagedProgramUtils.getPythonDriverClassName()));
+ assertThat(informationProvider.getJarFile().isPresent(), is(true));
+ assertThat(
+ informationProvider.getJarFile().get(),
+ is(PackagedProgramUtils.getPythonJar().getFile()));
+ }
+
+ @Test
+ public void testDeriveEntryClassInformationForCustomJar()
+ throws FlinkException, MalformedURLException {
+ // make loading from system classpath fail to make sure that it's not triggered
+ classpathProvider.setSystemClasspathWithTwoEntryClasses();
+
+ final String jobClassName = "SomeJobClassName";
+ final File jarFile = new File("some/jar/file.jar");
+ final EntryClassInformationProvider informationProvider =
+ PackagedProgramRetrieverImpl.createEntryClassInformationProvider(
+ null, null, jarFile, jobClassName, new String[0]);
+ assertThat(informationProvider.getJobClassName().isPresent(), is(true));
+ assertThat(informationProvider.getJobClassName().get(), is(jobClassName));
+ assertThat(informationProvider.getJarFile().isPresent(), is(true));
+ assertThat(informationProvider.getJarFile().get(), is(jarFile));
+ }
+
+ @Test
+ // TODO: we might want to change this behavior triggering a failure here
+ public void testDeriveEntryClassInformationFromSystemClasspathWithNonExistingJobClassName()
+ throws IOException, FlinkException {
+ classpathProvider.setSystemClasspathWithEntryClass();
+
+ final String jobClassName = "SomeJobClassNotBeingOnTheSystemClasspath";
+ final EntryClassInformationProvider informationProvider =
+ PackagedProgramRetrieverImpl.createEntryClassInformationProvider(
+ null, null, null, jobClassName, new String[0]);
+ assertThat(informationProvider.getJobClassName().isPresent(), is(true));
+ assertThat(informationProvider.getJobClassName().get(), is(jobClassName));
+ assertThat(informationProvider.getJarFile().isPresent(), is(false));
+ }
+
+ @Test
+ public void testDeriveEntryClassInformationFromSystemClasspathWithExistingJobClassName()
+ throws IOException, FlinkException {
+ classpathProvider.setSystemClasspathWithEntryClass();
+
+ final EntryClassInformationProvider informationProvider =
+ PackagedProgramRetrieverImpl.createEntryClassInformationProvider(
+ null, null, null, classpathProvider.getJobClassName(), new String[0]);
+ assertThat(informationProvider.getJobClassName().isPresent(), is(true));
+ assertThat(
+ informationProvider.getJobClassName().get(),
+ is(classpathProvider.getJobClassName()));
+ assertThat(informationProvider.getJarFile().isPresent(), is(false));
+ }
+
+ @Test
+ public void testDeriveEntryClassInformationFromSystemClasspathExtractingTheJobClassFromThere()
+ throws IOException, FlinkException {
+ classpathProvider.setSystemClasspathWithEntryClass();
+
+ final EntryClassInformationProvider informationProvider =
+ PackagedProgramRetrieverImpl.createEntryClassInformationProvider(
+ null, null, null, null, new String[0]);
+ assertThat(informationProvider.getJobClassName().isPresent(), is(true));
+ assertThat(
+ informationProvider.getJobClassName().get(),
+ is(classpathProvider.getJobClassName()));
+ assertThat(informationProvider.getJarFile().isPresent(), is(false));
+ }
+
+ @Test
+ public void testDeriveEntryClassInformationFromClasspathWithJobClass()
+ throws IOException, FlinkException {
+ final EntryClassInformationProvider informationProvider =
+ PackagedProgramRetrieverImpl.createEntryClassInformationProvider(
+ // the user directory must be specified
+ classpathProvider.getUserDirectoryWithTwoEntryClasses(),
+ // the user classpath is derived from the user directory outside of the
+ // method
+ classpathProvider.getURLUserClasspathWithTwoEntryClasses(),
Review comment:
Aren't these irrelevant since we explicitly provide the job classname anyway?
##########
File path: flink-clients/src/test/java/org/apache/flink/client/program/PackagedProgramRetrieverImplTest.java
##########
@@ -0,0 +1,557 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.client.program;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.api.dag.Pipeline;
+import org.apache.flink.client.deployment.application.EntryClassInformationProvider;
+import org.apache.flink.client.deployment.executors.PipelineExecutorUtils;
+import org.apache.flink.client.testjar.ClasspathProvider;
+import org.apache.flink.configuration.ConfigConstants;
+import org.apache.flink.configuration.ConfigUtils;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.CoreOptions;
+import org.apache.flink.configuration.PipelineOptions;
+import org.apache.flink.configuration.PipelineOptionsInternal;
+import org.apache.flink.runtime.execution.librarycache.FlinkUserCodeClassLoaders;
+import org.apache.flink.runtime.jobgraph.JobGraph;
+import org.apache.flink.runtime.jobgraph.SavepointRestoreSettings;
+import org.apache.flink.util.ChildFirstClassLoader;
+import org.apache.flink.util.ExceptionUtils;
+import org.apache.flink.util.FileUtils;
+import org.apache.flink.util.FlinkException;
+import org.apache.flink.util.TestLogger;
+
+import org.hamcrest.collection.IsIterableContainingInAnyOrder;
+import org.hamcrest.core.IsInstanceOf;
+import org.junit.Assume;
+import org.junit.Ignore;
+import org.junit.Rule;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.core.Is.is;
+import static org.hamcrest.core.IsNot.not;
+import static org.junit.Assert.fail;
+
+/** {@code PackagedProgramRetrieverImplTest} tests {@link PackagedProgramRetrieverImpl}. */
+public class PackagedProgramRetrieverImplTest extends TestLogger {
+
+ @Rule public ClasspathProvider classpathProvider = new ClasspathProvider();
+
+ @Test
+ public void deriveEntryClassInformationForPythonBasedOnJobName() throws FlinkException {
+ // TODO: see FLINK-23154
+ Assume.assumeTrue(
+ "This tests only succeeds if " + ConfigConstants.ENV_FLINK_OPT_DIR + " is set.",
+ System.getProperty(ConfigConstants.ENV_FLINK_OPT_DIR) != null);
+ final EntryClassInformationProvider informationProvider =
+ PackagedProgramRetrieverImpl.createEntryClassInformationProvider(
+ null,
+ Collections.emptyList(),
+ null,
+ PackagedProgramUtils.getPythonDriverClassName(),
+ new String[0]);
+ assertThat(
+ informationProvider.getJobClassName(),
+ is(PackagedProgramUtils.getPythonDriverClassName()));
+ assertThat(informationProvider.getJarFile().isPresent(), is(true));
+ assertThat(
+ informationProvider.getJarFile().get(),
+ is(PackagedProgramUtils.getPythonJar().getFile()));
+ }
+
+ @Test
+ public void deriveEntryClassInformationForPythonBasedOnParameter() throws FlinkException {
+ // TODO: see FLINK-23154
+ Assume.assumeTrue(
+ "This tests only succeeds if " + ConfigConstants.ENV_FLINK_OPT_DIR + " is set.",
+ System.getProperty(ConfigConstants.ENV_FLINK_OPT_DIR) != null);
+
+ final EntryClassInformationProvider informationProvider =
+ PackagedProgramRetrieverImpl.createEntryClassInformationProvider(
+ null, Collections.emptyList(), null, null, new String[] {"--python"});
+
+ assertThat(
+ informationProvider.getJobClassName(),
+ is(PackagedProgramUtils.getPythonDriverClassName()));
+ assertThat(informationProvider.getJarFile().isPresent(), is(true));
+ assertThat(
+ informationProvider.getJarFile().get(),
+ is(PackagedProgramUtils.getPythonJar().getFile()));
+ }
+
+ @Test
+ public void deriveEntryClassInformationForCustomJar()
+ throws FlinkException, MalformedURLException {
+ // make loading from system classpath fail to make sure that it's not triggered
+ classpathProvider.setSystemClasspathWithTwoEntryClasses();
+
+ final String jobClassName = "SomeJobClassName";
+ final File jarFile = new File("some/jar/file.jar");
+ final EntryClassInformationProvider informationProvider =
+ PackagedProgramRetrieverImpl.createEntryClassInformationProvider(
+ null, null, jarFile, jobClassName, new String[0]);
+ assertThat(informationProvider.getJobClassName(), is(jobClassName));
+ assertThat(informationProvider.getJarFile().isPresent(), is(true));
+ assertThat(informationProvider.getJarFile().get(), is(jarFile));
+ }
+
+ @Test
+ // TODO: we might want to change this behavior triggering a failure here
+ public void testDeriveEntryClassInformationFromSystemClasspathWithNonExistingJobClassName()
+ throws IOException, FlinkException {
+ classpathProvider.setSystemClasspathWithEntryClass();
+
+ final String jobClassName = "SomeJobClassNotBeingOnTheSystemClasspath";
+ final EntryClassInformationProvider informationProvider =
+ PackagedProgramRetrieverImpl.createEntryClassInformationProvider(
+ null, null, null, jobClassName, new String[0]);
+ assertThat(informationProvider.getJobClassName(), is(jobClassName));
+ assertThat(informationProvider.getJarFile().isPresent(), is(false));
+ }
+
+ @Test
+ public void testDeriveEntryClassInformationFromSystemClasspathWithExistingJobClassName()
+ throws IOException, FlinkException {
+ classpathProvider.setSystemClasspathWithEntryClass();
+
+ final EntryClassInformationProvider informationProvider =
+ PackagedProgramRetrieverImpl.createEntryClassInformationProvider(
+ null, null, null, classpathProvider.getJobClassName(), new String[0]);
+ assertThat(informationProvider.getJobClassName(), is(classpathProvider.getJobClassName()));
+ assertThat(informationProvider.getJarFile().isPresent(), is(false));
+ }
+
+ @Test
+ public void testDeriveEntryClassInformationFromSystemClasspathExtractingTheJobClassFromThere()
+ throws IOException, FlinkException {
+ classpathProvider.setSystemClasspathWithEntryClass();
+
+ final EntryClassInformationProvider informationProvider =
+ PackagedProgramRetrieverImpl.createEntryClassInformationProvider(
+ null, null, null, null, new String[0]);
+ assertThat(informationProvider.getJobClassName(), is(classpathProvider.getJobClassName()));
+ assertThat(informationProvider.getJarFile().isPresent(), is(false));
+ }
+
+ @Test
+ public void testDeriveEntryClassInformationFromClasspathWithJobClass()
+ throws IOException, FlinkException {
+ final EntryClassInformationProvider informationProvider =
+ PackagedProgramRetrieverImpl.createEntryClassInformationProvider(
+ // the user directory must be specified
+ classpathProvider.getUserDirectoryWithTwoEntryClasses(),
+ // the user classpath is derived from the user directory outside of the
+ // method
+ classpathProvider.getURLUserClasspathWithTwoEntryClasses(),
+ null,
+ // we have to specify the job class - otherwise the call would fail due to
+ // two main method being present
+ classpathProvider.getJobClassName(),
+ new String[0]);
+ assertThat(informationProvider.getJobClassName(), is(classpathProvider.getJobClassName()));
+ assertThat(informationProvider.getJarFile().isPresent(), is(false));
+ }
+
+ @Test
+ public void testDeriveEntryClassInformationFromClasspathWithNoJobClass()
+ throws IOException, FlinkException {
+ final EntryClassInformationProvider informationProvider =
+ PackagedProgramRetrieverImpl.createEntryClassInformationProvider(
+ // the user directory must be specified
+ classpathProvider.getUserDirectoryWithOneEntryClass(),
+ // the user classpath is derived from the user directory outside of the
+ // method
+ classpathProvider.getURLUserClasspathWithEntryClass(),
+ null,
+ // no job class name is specified which enables looking for the entry class
+ // on the user classpath
+ null,
+ new String[0]);
+ assertThat(informationProvider.getJobClassName(), is(classpathProvider.getJobClassName()));
+ assertThat(informationProvider.getJarFile().isPresent(), is(false));
+ }
+
+ @Test
+ public void testCreateWithUserLibDir() throws FlinkException {
+ final PackagedProgramRetriever retriever =
+ PackagedProgramRetrieverImpl.create(
+ classpathProvider.getUserDirectoryWithOneEntryClass(),
+ null,
+ classpathProvider.getJobClassName(),
+ new String[0],
+ new Configuration());
+
+ // the right information is picked up without any error
+ assertThat(
+ retriever.getPackagedProgram().getMainClassName(),
+ is(classpathProvider.getJobClassName()));
+ }
+
+ @Test
+ public void testJobGraphRetrieval()
+ throws IOException, FlinkException, ProgramInvocationException {
+ final int parallelism = 42;
+ final JobID jobId = new JobID();
+
+ final Configuration configuration = new Configuration();
+ configuration.setInteger(CoreOptions.DEFAULT_PARALLELISM, parallelism);
+ configuration.set(PipelineOptionsInternal.PIPELINE_FIXED_JOB_ID, jobId.toHexString());
+
+ final String expectedSuffix = "suffix";
+ final PackagedProgramRetriever retriever =
+ PackagedProgramRetrieverImpl.create(
+ null,
+ classpathProvider.getTestJobClassName(),
+ classpathProvider.getTestJobArgs(expectedSuffix),
+ new Configuration());
+
+ final JobGraph jobGraph = retrieveJobGraph(retriever, configuration);
+
+ assertThat(
+ jobGraph.getName(),
+ is(classpathProvider.getTestJobClassName() + "-" + expectedSuffix));
+ assertThat(jobGraph.getSavepointRestoreSettings(), is(SavepointRestoreSettings.none()));
+ assertThat(jobGraph.getMaximumParallelism(), is(parallelism));
+ assertThat(jobGraph.getJobID(), is(jobId));
+ }
+
+ @Test
+ public void testJobGraphRetrievalFromJar()
+ throws IOException, FlinkException, ProgramInvocationException {
+ final String expectedSuffix = "suffix";
+ final PackagedProgramRetriever retrieverUnderTest =
+ PackagedProgramRetrieverImpl.create(
+ classpathProvider.getUserDirectoryWithTestJob(),
+ null,
+ null,
+ classpathProvider.getTestJobArgs(expectedSuffix),
+ new Configuration());
+
+ final JobGraph jobGraph = retrieveJobGraph(retrieverUnderTest, new Configuration());
+
+ assertThat(
+ jobGraph.getName(),
+ is(classpathProvider.getTestJobClassName() + "-" + expectedSuffix));
+ }
+
+ @Test
+ public void testJobGraphRetrievalJobClassNameHasPrecedenceOverClasspath()
Review comment:
It doesn't matter where the setter is used in production or not, the purpose of the test is to ensure behavior despite _changes_ in production code (i.e., if the setter were used).
Where else do you test that, if 2 entrypoints are on the CP and the user specifies the entrypoint class, that the user argument is honored?
##########
File path: flink-clients/src/test/java/org/apache/flink/client/program/PackagedProgramRetrieverImplTest.java
##########
@@ -0,0 +1,573 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.client.program;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.api.dag.Pipeline;
+import org.apache.flink.client.deployment.application.EntryClassInformationProvider;
+import org.apache.flink.client.deployment.executors.PipelineExecutorUtils;
+import org.apache.flink.client.testjar.ClasspathProvider;
+import org.apache.flink.configuration.ConfigConstants;
+import org.apache.flink.configuration.ConfigUtils;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.CoreOptions;
+import org.apache.flink.configuration.PipelineOptions;
+import org.apache.flink.configuration.PipelineOptionsInternal;
+import org.apache.flink.runtime.execution.librarycache.FlinkUserCodeClassLoaders;
+import org.apache.flink.runtime.jobgraph.JobGraph;
+import org.apache.flink.runtime.jobgraph.SavepointRestoreSettings;
+import org.apache.flink.util.ChildFirstClassLoader;
+import org.apache.flink.util.ExceptionUtils;
+import org.apache.flink.util.FileUtils;
+import org.apache.flink.util.FlinkException;
+import org.apache.flink.util.TestLogger;
+
+import org.hamcrest.collection.IsIterableContainingInAnyOrder;
+import org.hamcrest.core.IsInstanceOf;
+import org.junit.Assume;
+import org.junit.Ignore;
+import org.junit.Rule;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.core.Is.is;
+import static org.hamcrest.core.IsNot.not;
+import static org.junit.Assert.fail;
+
+/** {@code PackagedProgramRetrieverImplTest} tests {@link PackagedProgramRetrieverImpl}. */
+public class PackagedProgramRetrieverImplTest extends TestLogger {
+
+ @Rule public ClasspathProvider classpathProvider = new ClasspathProvider();
+
+ @Test
+ public void testDeriveEntryClassInformationForPythonBasedOnJobName() throws FlinkException {
+ // TODO: see FLINK-23154
+ Assume.assumeTrue(
+ "This tests only succeeds if " + ConfigConstants.ENV_FLINK_OPT_DIR + " is set.",
+ System.getProperty(ConfigConstants.ENV_FLINK_OPT_DIR) != null);
+ final EntryClassInformationProvider informationProvider =
+ PackagedProgramRetrieverImpl.createEntryClassInformationProvider(
+ null,
+ Collections.emptyList(),
+ null,
+ PackagedProgramUtils.getPythonDriverClassName(),
+ new String[0]);
+ assertThat(informationProvider.getJobClassName().isPresent(), is(true));
+ assertThat(
+ informationProvider.getJobClassName().get(),
+ is(PackagedProgramUtils.getPythonDriverClassName()));
+ assertThat(informationProvider.getJarFile().isPresent(), is(true));
+ assertThat(
+ informationProvider.getJarFile().get(),
+ is(PackagedProgramUtils.getPythonJar().getFile()));
+ }
+
+ @Test
+ public void testDeriveEntryClassInformationForPythonBasedOnParameter() throws FlinkException {
+ // TODO: see FLINK-23154
+ Assume.assumeTrue(
+ "This tests only succeeds if " + ConfigConstants.ENV_FLINK_OPT_DIR + " is set.",
+ System.getProperty(ConfigConstants.ENV_FLINK_OPT_DIR) != null);
+
+ final EntryClassInformationProvider informationProvider =
+ PackagedProgramRetrieverImpl.createEntryClassInformationProvider(
+ null, Collections.emptyList(), null, null, new String[] {"--python"});
+
+ assertThat(informationProvider.getJobClassName().isPresent(), is(true));
+ assertThat(
+ informationProvider.getJobClassName().get(),
+ is(PackagedProgramUtils.getPythonDriverClassName()));
+ assertThat(informationProvider.getJarFile().isPresent(), is(true));
+ assertThat(
+ informationProvider.getJarFile().get(),
+ is(PackagedProgramUtils.getPythonJar().getFile()));
+ }
+
+ @Test
+ public void testDeriveEntryClassInformationForCustomJar()
Review comment:
What is actually being derived from anything? It seems like all we're testing here is that the stuff passed via the method is eventually assigned.
##########
File path: flink-clients/src/test/java/org/apache/flink/client/program/PackagedProgramRetrieverImplTest.java
##########
@@ -0,0 +1,573 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.client.program;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.api.dag.Pipeline;
+import org.apache.flink.client.deployment.application.EntryClassInformationProvider;
+import org.apache.flink.client.deployment.executors.PipelineExecutorUtils;
+import org.apache.flink.client.testjar.ClasspathProvider;
+import org.apache.flink.configuration.ConfigConstants;
+import org.apache.flink.configuration.ConfigUtils;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.CoreOptions;
+import org.apache.flink.configuration.PipelineOptions;
+import org.apache.flink.configuration.PipelineOptionsInternal;
+import org.apache.flink.runtime.execution.librarycache.FlinkUserCodeClassLoaders;
+import org.apache.flink.runtime.jobgraph.JobGraph;
+import org.apache.flink.runtime.jobgraph.SavepointRestoreSettings;
+import org.apache.flink.util.ChildFirstClassLoader;
+import org.apache.flink.util.ExceptionUtils;
+import org.apache.flink.util.FileUtils;
+import org.apache.flink.util.FlinkException;
+import org.apache.flink.util.TestLogger;
+
+import org.hamcrest.collection.IsIterableContainingInAnyOrder;
+import org.hamcrest.core.IsInstanceOf;
+import org.junit.Assume;
+import org.junit.Ignore;
+import org.junit.Rule;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.core.Is.is;
+import static org.hamcrest.core.IsNot.not;
+import static org.junit.Assert.fail;
+
+/** {@code PackagedProgramRetrieverImplTest} tests {@link PackagedProgramRetrieverImpl}. */
+public class PackagedProgramRetrieverImplTest extends TestLogger {
+
+ @Rule public ClasspathProvider classpathProvider = new ClasspathProvider();
+
+ @Test
+ public void testDeriveEntryClassInformationForPythonBasedOnJobName() throws FlinkException {
+ // TODO: see FLINK-23154
+ Assume.assumeTrue(
+ "This tests only succeeds if " + ConfigConstants.ENV_FLINK_OPT_DIR + " is set.",
+ System.getProperty(ConfigConstants.ENV_FLINK_OPT_DIR) != null);
+ final EntryClassInformationProvider informationProvider =
+ PackagedProgramRetrieverImpl.createEntryClassInformationProvider(
+ null,
+ Collections.emptyList(),
+ null,
+ PackagedProgramUtils.getPythonDriverClassName(),
+ new String[0]);
+ assertThat(informationProvider.getJobClassName().isPresent(), is(true));
+ assertThat(
+ informationProvider.getJobClassName().get(),
+ is(PackagedProgramUtils.getPythonDriverClassName()));
+ assertThat(informationProvider.getJarFile().isPresent(), is(true));
+ assertThat(
+ informationProvider.getJarFile().get(),
+ is(PackagedProgramUtils.getPythonJar().getFile()));
+ }
+
+ @Test
+ public void testDeriveEntryClassInformationForPythonBasedOnParameter() throws FlinkException {
+ // TODO: see FLINK-23154
+ Assume.assumeTrue(
+ "This tests only succeeds if " + ConfigConstants.ENV_FLINK_OPT_DIR + " is set.",
+ System.getProperty(ConfigConstants.ENV_FLINK_OPT_DIR) != null);
+
+ final EntryClassInformationProvider informationProvider =
+ PackagedProgramRetrieverImpl.createEntryClassInformationProvider(
+ null, Collections.emptyList(), null, null, new String[] {"--python"});
+
+ assertThat(informationProvider.getJobClassName().isPresent(), is(true));
+ assertThat(
+ informationProvider.getJobClassName().get(),
+ is(PackagedProgramUtils.getPythonDriverClassName()));
+ assertThat(informationProvider.getJarFile().isPresent(), is(true));
+ assertThat(
+ informationProvider.getJarFile().get(),
+ is(PackagedProgramUtils.getPythonJar().getFile()));
+ }
+
+ @Test
+ public void testDeriveEntryClassInformationForCustomJar()
+ throws FlinkException, MalformedURLException {
+ // make loading from system classpath fail to make sure that it's not triggered
+ classpathProvider.setSystemClasspathWithTwoEntryClasses();
+
+ final String jobClassName = "SomeJobClassName";
+ final File jarFile = new File("some/jar/file.jar");
+ final EntryClassInformationProvider informationProvider =
+ PackagedProgramRetrieverImpl.createEntryClassInformationProvider(
+ null, null, jarFile, jobClassName, new String[0]);
+ assertThat(informationProvider.getJobClassName().isPresent(), is(true));
+ assertThat(informationProvider.getJobClassName().get(), is(jobClassName));
+ assertThat(informationProvider.getJarFile().isPresent(), is(true));
+ assertThat(informationProvider.getJarFile().get(), is(jarFile));
+ }
+
+ @Test
+ // TODO: we might want to change this behavior triggering a failure here
+ public void testDeriveEntryClassInformationFromSystemClasspathWithNonExistingJobClassName()
+ throws IOException, FlinkException {
+ classpathProvider.setSystemClasspathWithEntryClass();
+
+ final String jobClassName = "SomeJobClassNotBeingOnTheSystemClasspath";
+ final EntryClassInformationProvider informationProvider =
+ PackagedProgramRetrieverImpl.createEntryClassInformationProvider(
+ null, null, null, jobClassName, new String[0]);
+ assertThat(informationProvider.getJobClassName().isPresent(), is(true));
+ assertThat(informationProvider.getJobClassName().get(), is(jobClassName));
+ assertThat(informationProvider.getJarFile().isPresent(), is(false));
+ }
+
+ @Test
+ public void testDeriveEntryClassInformationFromSystemClasspathWithExistingJobClassName()
+ throws IOException, FlinkException {
+ classpathProvider.setSystemClasspathWithEntryClass();
+
+ final EntryClassInformationProvider informationProvider =
+ PackagedProgramRetrieverImpl.createEntryClassInformationProvider(
+ null, null, null, classpathProvider.getJobClassName(), new String[0]);
+ assertThat(informationProvider.getJobClassName().isPresent(), is(true));
+ assertThat(
+ informationProvider.getJobClassName().get(),
+ is(classpathProvider.getJobClassName()));
+ assertThat(informationProvider.getJarFile().isPresent(), is(false));
+ }
+
+ @Test
+ public void testDeriveEntryClassInformationFromSystemClasspathExtractingTheJobClassFromThere()
+ throws IOException, FlinkException {
+ classpathProvider.setSystemClasspathWithEntryClass();
+
+ final EntryClassInformationProvider informationProvider =
+ PackagedProgramRetrieverImpl.createEntryClassInformationProvider(
+ null, null, null, null, new String[0]);
+ assertThat(informationProvider.getJobClassName().isPresent(), is(true));
+ assertThat(
+ informationProvider.getJobClassName().get(),
+ is(classpathProvider.getJobClassName()));
+ assertThat(informationProvider.getJarFile().isPresent(), is(false));
+ }
+
+ @Test
+ public void testDeriveEntryClassInformationFromClasspathWithJobClass()
+ throws IOException, FlinkException {
+ final EntryClassInformationProvider informationProvider =
+ PackagedProgramRetrieverImpl.createEntryClassInformationProvider(
+ // the user directory must be specified
+ classpathProvider.getUserDirectoryWithTwoEntryClasses(),
+ // the user classpath is derived from the user directory outside of the
+ // method
+ classpathProvider.getURLUserClasspathWithTwoEntryClasses(),
+ null,
+ // we have to specify the job class - otherwise the call would fail due to
+ // two main method being present
+ classpathProvider.getJobClassName(),
+ new String[0]);
+ assertThat(informationProvider.getJobClassName().isPresent(), is(true));
+ assertThat(
+ informationProvider.getJobClassName().get(),
+ is(classpathProvider.getJobClassName()));
+ assertThat(informationProvider.getJarFile().isPresent(), is(false));
+ }
+
+ @Test
+ public void testDeriveEntryClassInformationFromClasspathWithNoJobClass()
+ throws IOException, FlinkException {
+ final EntryClassInformationProvider informationProvider =
+ PackagedProgramRetrieverImpl.createEntryClassInformationProvider(
+ // the user directory must be specified
+ classpathProvider.getUserDirectoryWithOneEntryClass(),
+ // the user classpath is derived from the user directory outside of the
+ // method
+ classpathProvider.getURLUserClasspathWithEntryClass(),
+ null,
+ // no job class name is specified which enables looking for the entry class
+ // on the user classpath
+ null,
+ new String[0]);
+ assertThat(informationProvider.getJobClassName().isPresent(), is(true));
+ assertThat(
+ informationProvider.getJobClassName().get(),
+ is(classpathProvider.getJobClassName()));
+ assertThat(informationProvider.getJarFile().isPresent(), is(false));
+ }
+
+ @Test
+ public void testCreateWithUserLibDir() throws FlinkException {
+ final PackagedProgramRetriever retriever =
+ PackagedProgramRetrieverImpl.create(
+ classpathProvider.getUserDirectoryWithOneEntryClass(),
+ null,
+ classpathProvider.getJobClassName(),
+ new String[0],
+ new Configuration());
+
+ // the right information is picked up without any error
+ assertThat(
+ retriever.getPackagedProgram().getMainClassName(),
+ is(classpathProvider.getJobClassName()));
+ }
+
+ @Test
+ public void testJobGraphRetrieval()
+ throws IOException, FlinkException, ProgramInvocationException {
+ final int parallelism = 42;
+ final JobID jobId = new JobID();
+
+ final Configuration configuration = new Configuration();
+ configuration.setInteger(CoreOptions.DEFAULT_PARALLELISM, parallelism);
+ configuration.set(PipelineOptionsInternal.PIPELINE_FIXED_JOB_ID, jobId.toHexString());
+
+ final String expectedSuffix = "suffix";
+ final PackagedProgramRetriever retriever =
+ PackagedProgramRetrieverImpl.create(
+ null,
+ classpathProvider.getTestJobClassName(),
+ classpathProvider.getTestJobArgs(expectedSuffix),
+ new Configuration());
+
+ final JobGraph jobGraph = retrieveJobGraph(retriever, configuration);
+
+ assertThat(
+ jobGraph.getName(),
+ is(classpathProvider.getTestJobClassName() + "-" + expectedSuffix));
+ assertThat(jobGraph.getSavepointRestoreSettings(), is(SavepointRestoreSettings.none()));
+ assertThat(jobGraph.getMaximumParallelism(), is(parallelism));
+ assertThat(jobGraph.getJobID(), is(jobId));
+ }
+
+ @Test
+ public void testJobGraphRetrievalFromJar()
+ throws IOException, FlinkException, ProgramInvocationException {
+ final String expectedSuffix = "suffix";
+ final PackagedProgramRetriever retrieverUnderTest =
+ PackagedProgramRetrieverImpl.create(
+ classpathProvider.getUserDirectoryWithTestJob(),
+ null,
+ null,
+ classpathProvider.getTestJobArgs(expectedSuffix),
+ new Configuration());
+
+ final JobGraph jobGraph = retrieveJobGraph(retrieverUnderTest, new Configuration());
+
+ assertThat(
+ jobGraph.getName(),
+ is(classpathProvider.getTestJobClassName() + "-" + expectedSuffix));
+ }
+
+ @Test
+ public void testJobGraphRetrievalJobClassNameHasPrecedenceOverClasspath()
+ throws IOException, FlinkException, ProgramInvocationException {
+ // TODO: I don't get this test: It feels like it does not add any extra value - it does the
+ // same like testTooManyEntryClassesOnSystemClasspath
+ final String expectedSuffix = "suffix";
+ final PackagedProgramRetriever retrieverUnderTest =
+ // Both a class name is specified and a JAR "is" on the class path
+ // The class name should have precedence.
+ PackagedProgramRetrieverImpl.create(
+ null,
+ classpathProvider.getTestJobClassName(),
+ classpathProvider.getTestJobArgs(expectedSuffix),
+ new Configuration());
+
+ final JobGraph jobGraph = retrieveJobGraph(retrieverUnderTest, new Configuration());
+
+ assertThat(jobGraph.getName(), is(classpathProvider.getTestJobClassName() + "-suffix"));
+ }
+
+ @Test
+ public void testSavepointRestoreSettings()
+ throws FlinkException, IOException, ProgramInvocationException {
+ final Configuration configuration = new Configuration();
+ final SavepointRestoreSettings savepointRestoreSettings =
+ SavepointRestoreSettings.forPath("foobar", true);
+ final JobID jobId = new JobID();
+
+ configuration.setString(PipelineOptionsInternal.PIPELINE_FIXED_JOB_ID, jobId.toHexString());
+ SavepointRestoreSettings.toConfiguration(savepointRestoreSettings, configuration);
+
+ final String expectedSuffix = "suffix";
+ final PackagedProgramRetriever retrieverUnderTest =
+ PackagedProgramRetrieverImpl.create(
+ null,
+ classpathProvider.getTestJobClassName(),
+ classpathProvider.getTestJobArgs(expectedSuffix),
+ new Configuration());
+
+ final JobGraph jobGraph = retrieveJobGraph(retrieverUnderTest, configuration);
+
+ assertThat(jobGraph.getSavepointRestoreSettings(), is(savepointRestoreSettings));
+ assertThat(jobGraph.getJobID(), is(jobId));
+ }
+
+ @Test
+ public void testFailIfJobDirDoesNotHaveEntryClass() {
+ try {
+ PackagedProgramRetrieverImpl.create(
+ classpathProvider.getUserDirectoryWithNoEntryClass(),
+ classpathProvider.getTestJobClassName(),
+ classpathProvider.getTestJobArgs("suffix"),
+ new Configuration());
+ fail("This case should throw exception !");
+ } catch (FlinkException e) {
+ assertThat(
+ ExceptionUtils.findThrowableWithMessage(
+ e,
+ String.format(
+ "Could not find the provided job class (%s) in the user lib directory.",
+ classpathProvider.getTestJobClassName()))
+ .isPresent(),
+ is(true));
+ }
+ }
+
+ // TODO: this test checks the same code path as
+ // testDeriveEntryClassInformationFromSystemClasspathWithNonExistingJobClassName
+ // We should make it fail early if the class is not present on the system classpath
+ // Right now, the test is failing because no error is thrown
+ @Ignore
+ @Test(expected = FlinkException.class)
+ public void testEntryClassNotFoundOnSystemClasspath() throws FlinkException {
+ PackagedProgramRetrieverImpl.create(
+ null, "NotExistingClass", new String[0], new Configuration());
+ }
+
+ @Test(expected = FlinkException.class)
+ public void testEntryClassNotFoundOnUserClasspath() throws FlinkException {
+ PackagedProgramRetrieverImpl.create(
+ classpathProvider.getUserDirectoryWithNoEntryClass(),
+ classpathProvider.getTestJobClassName(),
+ new String[0],
+ new Configuration());
+ }
+
+ @Test(expected = FlinkException.class)
+ public void testTooManyEntryClassesOnSystemClasspath() throws FlinkException {
+ PackagedProgramRetrieverImpl.create(null, null, new String[0], new Configuration());
+ }
+
+ @Test
+ public void testTooManyEntryClassesOnUserClasspath() throws FlinkException, IOException {
+ classpathProvider.setSystemClasspathWithTwoEntryClasses();
+ final PackagedProgramRetrieverImpl retriever =
+ PackagedProgramRetrieverImpl.create(
+ classpathProvider.getUserDirectoryWithTwoEntryClasses(),
+ classpathProvider.getTestJobClassName(),
+ new String[0],
+ new Configuration());
+ assertThat(
+ retriever.getPackagedProgram().getMainClassName(),
+ is(classpathProvider.getTestJobClassName()));
+ }
+
+ @Test
+ public void testRetrieveCorrectUserClasspathsWithoutSpecifiedEntryClass()
+ throws IOException, FlinkException, ProgramInvocationException {
+ final PackagedProgramRetriever retrieverUnderTest =
+ PackagedProgramRetrieverImpl.create(
+ classpathProvider.getUserDirectoryWithOneEntryClass(),
+ null,
+ classpathProvider.getTestJobArgs("suffix"),
+ new Configuration());
+ final JobGraph jobGraph = retrieveJobGraph(retrieverUnderTest, new Configuration());
+ final List<String> actualClasspath =
+ jobGraph.getClasspaths().stream().map(URL::toString).collect(Collectors.toList());
+
+ final Path workingDirectory = FileUtils.getCurrentWorkingDirectory();
+ final List<String> expectedClasspath = new ArrayList<>();
+ for (File file : classpathProvider.getUserDirectoryWithOneEntryClass().listFiles()) {
+ if (!file.getName().endsWith("jar")) {
+ // only jars are expected
+ continue;
+ }
Review comment:
Why should we silently ignore this? Wouldn't it be a bug in the ClassPathProvider?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@flink.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org