You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by kk...@apache.org on 2019/12/14 12:40:55 UTC

[flink] 02/07: [FLINK-15179] Introduce generic ExecutorCLI

This is an automated email from the ASF dual-hosted git repository.

kkloudas pushed a commit to branch release-1.10
in repository https://gitbox.apache.org/repos/asf/flink.git

commit a50decf2e70e76157d00488398bee4568bc831ce
Author: Kostas Kloudas <kk...@gmail.com>
AuthorDate: Tue Dec 10 18:31:30 2019 +0100

    [FLINK-15179] Introduce generic ExecutorCLI
---
 .../org/apache/flink/client/cli/CliFrontend.java   |   2 +
 .../org/apache/flink/client/cli/ExecutorCLI.java   | 125 +++++++++++++++++++
 .../apache/flink/client/cli/ExecutorCLITest.java   | 135 +++++++++++++++++++++
 3 files changed, 262 insertions(+)

diff --git a/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontend.java b/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontend.java
index 4b5a1a0..0c4a98b 100644
--- a/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontend.java
+++ b/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontend.java
@@ -1070,6 +1070,8 @@ public class CliFrontend {
 			LOG.warn("Could not load CLI class {}.", kubeSessionCLI, e);
 		}
 
+		customCommandLines.add(new ExecutorCLI(configuration));
+
 		//	Tips: DefaultCLI must be added at last, because getActiveCustomCommandLine(..) will get the
 		//	      active CustomCommandLine in order and DefaultCLI isActive always return true.
 		customCommandLines.add(new DefaultCLI(configuration));
diff --git a/flink-clients/src/main/java/org/apache/flink/client/cli/ExecutorCLI.java b/flink-clients/src/main/java/org/apache/flink/client/cli/ExecutorCLI.java
new file mode 100644
index 0000000..9beac1a
--- /dev/null
+++ b/flink-clients/src/main/java/org/apache/flink/client/cli/ExecutorCLI.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.client.cli;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.DeploymentOptions;
+import org.apache.flink.configuration.UnmodifiableConfiguration;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Properties;
+
+import static org.apache.flink.util.Preconditions.checkNotNull;
+import static org.apache.flink.util.Preconditions.checkState;
+
+/**
+ * A generic implementation of the {@link CustomCommandLine} that only expects
+ * the execution.target parameter to be explicitly specified and simply forwards the
+ * rest of the options specified with -D to the corresponding {@link org.apache.flink.core.execution.Executor}
+ * for further parsing.
+ */
+@Internal
+public class ExecutorCLI implements CustomCommandLine {
+
+	private static final Logger LOG = LoggerFactory.getLogger(ExecutorCLI.class);
+
+	private static final String ID = "Executor-CLI";
+
+	private final Option executorOption = new Option("e", "executor", true,
+			"The name of the executor to be used for executing the given job, e.g. \"local-executor\"." +
+					" This is equivalent to the \"" + DeploymentOptions.TARGET.key() + "\" config option.");
+
+	/**
+	 * Dynamic properties allow the user to specify additional configuration values with -D, such as
+	 * <tt> -Dfs.overwrite-files=true  -Dtaskmanager.memory.shuffle.min=536346624</tt>.
+	 */
+	private final Option dynamicProperties = Option.builder("D")
+			.argName("property=value")
+			.numberOfArgs(2)
+			.valueSeparator('=')
+			.desc("use value for given property")
+			.build();
+
+	private final Configuration baseConfiguration;
+
+	public ExecutorCLI(final Configuration configuration) {
+		this.baseConfiguration = new UnmodifiableConfiguration(checkNotNull(configuration));
+	}
+
+	@Override
+	public boolean isActive(CommandLine commandLine) {
+		return baseConfiguration.getOptional(DeploymentOptions.TARGET).isPresent()
+				|| commandLine.hasOption(executorOption.getOpt());
+	}
+
+	@Override
+	public String getId() {
+		return ID;
+	}
+
+	@Override
+	public void addRunOptions(Options baseOptions) {
+		// nothing to add here
+	}
+
+	@Override
+	public void addGeneralOptions(Options baseOptions) {
+		baseOptions.addOption(executorOption);
+		baseOptions.addOption(dynamicProperties);
+	}
+
+	@Override
+	public Configuration applyCommandLineOptionsToConfiguration(final CommandLine commandLine) {
+		checkState(isActive(commandLine));
+
+		final Configuration effectiveConfiguration = new Configuration(baseConfiguration);
+
+		final String executorName = commandLine.getOptionValue(executorOption.getOpt());
+		if (executorName != null) {
+			effectiveConfiguration.setString(DeploymentOptions.TARGET, executorName);
+		}
+
+		encodeDynamicProperties(commandLine, effectiveConfiguration);
+
+		if (LOG.isDebugEnabled()) {
+			LOG.debug("Effective Configuration: {}", effectiveConfiguration);
+		}
+
+		return effectiveConfiguration;
+	}
+
+	private void encodeDynamicProperties(final CommandLine commandLine, final Configuration effectiveConfiguration) {
+		final Properties properties = commandLine.getOptionProperties(dynamicProperties.getOpt());
+		properties.stringPropertyNames()
+				.forEach(key -> {
+					final String value = properties.getProperty(key);
+					if (value != null) {
+						effectiveConfiguration.setString(key, value);
+					} else {
+						effectiveConfiguration.setString(key, "true");
+					}
+				});
+	}
+}
diff --git a/flink-clients/src/test/java/org/apache/flink/client/cli/ExecutorCLITest.java b/flink-clients/src/test/java/org/apache/flink/client/cli/ExecutorCLITest.java
new file mode 100644
index 0000000..7fac8ff
--- /dev/null
+++ b/flink-clients/src/test/java/org/apache/flink/client/cli/ExecutorCLITest.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.client.cli;
+
+import org.apache.flink.configuration.ConfigOption;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.CoreOptions;
+import org.apache.flink.configuration.DeploymentOptions;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Options;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import static org.apache.flink.configuration.ConfigOptions.key;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+
+/**
+ * Unit tests for the {@link ExecutorCLI}.
+ */
+public class ExecutorCLITest {
+
+	private Options testOptions;
+
+	@Before
+	public void initOptions() {
+		testOptions = new Options();
+
+		final ExecutorCLI cliUnderTest = new ExecutorCLI(new Configuration());
+		cliUnderTest.addGeneralOptions(testOptions);
+	}
+
+	@Test(expected = IllegalStateException.class)
+	public void testExceptionWithUnspecifiedExecutor() throws CliArgsException {
+		final ExecutorCLI cliUnderTest = new ExecutorCLI(new Configuration());
+
+		final String[] args = {"-Dtest=tsetD-"};
+		final CommandLine commandLine = CliFrontendParser.parse(testOptions, args, true);
+
+		cliUnderTest.applyCommandLineOptionsToConfiguration(commandLine);
+	}
+
+	@Test
+	public void testExecutorInBaseConfigIsPickedUp() throws CliArgsException {
+		final String expectedExecutorName = "test-executor";
+		final Configuration loadedConfig = new Configuration();
+		loadedConfig.set(DeploymentOptions.TARGET, expectedExecutorName);
+
+		final ExecutorCLI cliUnderTest = new ExecutorCLI(loadedConfig);
+		final CommandLine emptyCommandLine = CliFrontendParser.parse(testOptions, new String[0], true);
+
+		final Configuration configuration = cliUnderTest.applyCommandLineOptionsToConfiguration(emptyCommandLine);
+		assertEquals(expectedExecutorName, configuration.get(DeploymentOptions.TARGET));
+	}
+
+	@Test
+	public void testWithPreexistingConfigurationInConstructor() throws CliArgsException {
+		final Configuration loadedConfig = new Configuration();
+		loadedConfig.setInteger(CoreOptions.DEFAULT_PARALLELISM, 2);
+		loadedConfig.setBoolean(DeploymentOptions.ATTACHED, false);
+
+		final ConfigOption<List<Integer>> listOption = key("test.list")
+				.intType()
+				.asList()
+				.noDefaultValue();
+
+		final List<Integer> listValue = Arrays.asList(41, 42, 23);
+		final String encodedListValue = listValue
+				.stream()
+				.map(Object::toString)
+				.collect(Collectors.joining(";"));
+
+		final String[] args = {
+				"-e", "test-executor",
+				"-D" + listOption.key() + "=" + encodedListValue,
+				"-D" + CoreOptions.DEFAULT_PARALLELISM.key() + "=5"
+		};
+
+		final ExecutorCLI cliUnderTest = new ExecutorCLI(loadedConfig);
+		final CommandLine commandLine = CliFrontendParser.parse(testOptions, args, true);
+
+		final Configuration configuration = cliUnderTest.applyCommandLineOptionsToConfiguration(commandLine);
+
+		assertEquals("test-executor", configuration.getString(DeploymentOptions.TARGET));
+		assertEquals(5, configuration.getInteger(CoreOptions.DEFAULT_PARALLELISM));
+		assertFalse(configuration.getBoolean(DeploymentOptions.ATTACHED));
+		assertEquals(listValue, configuration.get(listOption));
+	}
+
+	@Test
+	public void testIsActiveLong() throws CliArgsException {
+		testIsActiveHelper("--executor");
+	}
+
+	@Test
+	public void testIsActiveShort() throws CliArgsException {
+		testIsActiveHelper("-e");
+	}
+
+	private void testIsActiveHelper(final String executorOption) throws CliArgsException {
+		final String expectedExecutorName = "test-executor";
+		final ConfigOption<Integer> configOption = key("test.int").intType().noDefaultValue();
+		final int expectedValue = 42;
+
+		final ExecutorCLI cliUnderTest = new ExecutorCLI(new Configuration());
+
+		final String[] args = {executorOption, expectedExecutorName, "-D" + configOption.key() + "=" + expectedValue};
+		final CommandLine commandLine = CliFrontendParser.parse(testOptions, args, true);
+
+		final Configuration configuration = cliUnderTest.applyCommandLineOptionsToConfiguration(commandLine);
+		assertEquals(expectedExecutorName, configuration.get(DeploymentOptions.TARGET));
+		assertEquals(expectedValue, configuration.getInteger(configOption));
+	}
+}