You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by kk...@apache.org on 2019/12/14 12:40:55 UTC
[flink] 02/07: [FLINK-15179] Introduce generic ExecutorCLI
This is an automated email from the ASF dual-hosted git repository.
kkloudas pushed a commit to branch release-1.10
in repository https://gitbox.apache.org/repos/asf/flink.git
commit a50decf2e70e76157d00488398bee4568bc831ce
Author: Kostas Kloudas <kk...@gmail.com>
AuthorDate: Tue Dec 10 18:31:30 2019 +0100
[FLINK-15179] Introduce generic ExecutorCLI
---
.../org/apache/flink/client/cli/CliFrontend.java | 2 +
.../org/apache/flink/client/cli/ExecutorCLI.java | 125 +++++++++++++++++++
.../apache/flink/client/cli/ExecutorCLITest.java | 135 +++++++++++++++++++++
3 files changed, 262 insertions(+)
diff --git a/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontend.java b/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontend.java
index 4b5a1a0..0c4a98b 100644
--- a/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontend.java
+++ b/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontend.java
@@ -1070,6 +1070,8 @@ public class CliFrontend {
LOG.warn("Could not load CLI class {}.", kubeSessionCLI, e);
}
+ customCommandLines.add(new ExecutorCLI(configuration));
+
// Tips: DefaultCLI must be added at last, because getActiveCustomCommandLine(..) will get the
// active CustomCommandLine in order and DefaultCLI isActive always return true.
customCommandLines.add(new DefaultCLI(configuration));
diff --git a/flink-clients/src/main/java/org/apache/flink/client/cli/ExecutorCLI.java b/flink-clients/src/main/java/org/apache/flink/client/cli/ExecutorCLI.java
new file mode 100644
index 0000000..9beac1a
--- /dev/null
+++ b/flink-clients/src/main/java/org/apache/flink/client/cli/ExecutorCLI.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.client.cli;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.DeploymentOptions;
+import org.apache.flink.configuration.UnmodifiableConfiguration;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Properties;
+
+import static org.apache.flink.util.Preconditions.checkNotNull;
+import static org.apache.flink.util.Preconditions.checkState;
+
+/**
+ * A generic implementation of the {@link CustomCommandLine} that only expects
+ * the execution.target parameter to be explicitly specified and simply forwards the
+ * rest of the options specified with -D to the corresponding {@link org.apache.flink.core.execution.Executor}
+ * for further parsing.
+ */
+@Internal
+public class ExecutorCLI implements CustomCommandLine {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ExecutorCLI.class);
+
+ private static final String ID = "Executor-CLI";
+
+ private final Option executorOption = new Option("e", "executor", true,
+ "The name of the executor to be used for executing the given job, e.g. \"local-executor\"." +
+ " This is equivalent to the \"" + DeploymentOptions.TARGET.key() + "\" config option.");
+
+ /**
+ * Dynamic properties allow the user to specify additional configuration values with -D, such as
+ * <tt> -Dfs.overwrite-files=true -Dtaskmanager.memory.shuffle.min=536346624</tt>.
+ */
+ private final Option dynamicProperties = Option.builder("D")
+ .argName("property=value")
+ .numberOfArgs(2)
+ .valueSeparator('=')
+ .desc("use value for given property")
+ .build();
+
+ private final Configuration baseConfiguration;
+
+ public ExecutorCLI(final Configuration configuration) {
+ this.baseConfiguration = new UnmodifiableConfiguration(checkNotNull(configuration));
+ }
+
+ @Override
+ public boolean isActive(CommandLine commandLine) {
+ return baseConfiguration.getOptional(DeploymentOptions.TARGET).isPresent()
+ || commandLine.hasOption(executorOption.getOpt());
+ }
+
+ @Override
+ public String getId() {
+ return ID;
+ }
+
+ @Override
+ public void addRunOptions(Options baseOptions) {
+ // nothing to add here
+ }
+
+ @Override
+ public void addGeneralOptions(Options baseOptions) {
+ baseOptions.addOption(executorOption);
+ baseOptions.addOption(dynamicProperties);
+ }
+
+ @Override
+ public Configuration applyCommandLineOptionsToConfiguration(final CommandLine commandLine) {
+ checkState(isActive(commandLine));
+
+ final Configuration effectiveConfiguration = new Configuration(baseConfiguration);
+
+ final String executorName = commandLine.getOptionValue(executorOption.getOpt());
+ if (executorName != null) {
+ effectiveConfiguration.setString(DeploymentOptions.TARGET, executorName);
+ }
+
+ encodeDynamicProperties(commandLine, effectiveConfiguration);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Effective Configuration: {}", effectiveConfiguration);
+ }
+
+ return effectiveConfiguration;
+ }
+
+ private void encodeDynamicProperties(final CommandLine commandLine, final Configuration effectiveConfiguration) {
+ final Properties properties = commandLine.getOptionProperties(dynamicProperties.getOpt());
+ properties.stringPropertyNames()
+ .forEach(key -> {
+ final String value = properties.getProperty(key);
+ if (value != null) {
+ effectiveConfiguration.setString(key, value);
+ } else {
+ effectiveConfiguration.setString(key, "true");
+ }
+ });
+ }
+}
diff --git a/flink-clients/src/test/java/org/apache/flink/client/cli/ExecutorCLITest.java b/flink-clients/src/test/java/org/apache/flink/client/cli/ExecutorCLITest.java
new file mode 100644
index 0000000..7fac8ff
--- /dev/null
+++ b/flink-clients/src/test/java/org/apache/flink/client/cli/ExecutorCLITest.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.client.cli;
+
+import org.apache.flink.configuration.ConfigOption;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.CoreOptions;
+import org.apache.flink.configuration.DeploymentOptions;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Options;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import static org.apache.flink.configuration.ConfigOptions.key;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+
+/**
+ * Unit tests for the {@link ExecutorCLI}.
+ */
+public class ExecutorCLITest {
+
+ private Options testOptions;
+
+ @Before
+ public void initOptions() {
+ testOptions = new Options();
+
+ final ExecutorCLI cliUnderTest = new ExecutorCLI(new Configuration());
+ cliUnderTest.addGeneralOptions(testOptions);
+ }
+
+ @Test(expected = IllegalStateException.class)
+ public void testExceptionWithUnspecifiedExecutor() throws CliArgsException {
+ final ExecutorCLI cliUnderTest = new ExecutorCLI(new Configuration());
+
+ final String[] args = {"-Dtest=tsetD-"};
+ final CommandLine commandLine = CliFrontendParser.parse(testOptions, args, true);
+
+ cliUnderTest.applyCommandLineOptionsToConfiguration(commandLine);
+ }
+
+ @Test
+ public void testExecutorInBaseConfigIsPickedUp() throws CliArgsException {
+ final String expectedExecutorName = "test-executor";
+ final Configuration loadedConfig = new Configuration();
+ loadedConfig.set(DeploymentOptions.TARGET, expectedExecutorName);
+
+ final ExecutorCLI cliUnderTest = new ExecutorCLI(loadedConfig);
+ final CommandLine emptyCommandLine = CliFrontendParser.parse(testOptions, new String[0], true);
+
+ final Configuration configuration = cliUnderTest.applyCommandLineOptionsToConfiguration(emptyCommandLine);
+ assertEquals(expectedExecutorName, configuration.get(DeploymentOptions.TARGET));
+ }
+
+ @Test
+ public void testWithPreexistingConfigurationInConstructor() throws CliArgsException {
+ final Configuration loadedConfig = new Configuration();
+ loadedConfig.setInteger(CoreOptions.DEFAULT_PARALLELISM, 2);
+ loadedConfig.setBoolean(DeploymentOptions.ATTACHED, false);
+
+ final ConfigOption<List<Integer>> listOption = key("test.list")
+ .intType()
+ .asList()
+ .noDefaultValue();
+
+ final List<Integer> listValue = Arrays.asList(41, 42, 23);
+ final String encodedListValue = listValue
+ .stream()
+ .map(Object::toString)
+ .collect(Collectors.joining(";"));
+
+ final String[] args = {
+ "-e", "test-executor",
+ "-D" + listOption.key() + "=" + encodedListValue,
+ "-D" + CoreOptions.DEFAULT_PARALLELISM.key() + "=5"
+ };
+
+ final ExecutorCLI cliUnderTest = new ExecutorCLI(loadedConfig);
+ final CommandLine commandLine = CliFrontendParser.parse(testOptions, args, true);
+
+ final Configuration configuration = cliUnderTest.applyCommandLineOptionsToConfiguration(commandLine);
+
+ assertEquals("test-executor", configuration.getString(DeploymentOptions.TARGET));
+ assertEquals(5, configuration.getInteger(CoreOptions.DEFAULT_PARALLELISM));
+ assertFalse(configuration.getBoolean(DeploymentOptions.ATTACHED));
+ assertEquals(listValue, configuration.get(listOption));
+ }
+
+ @Test
+ public void testIsActiveLong() throws CliArgsException {
+ testIsActiveHelper("--executor");
+ }
+
+ @Test
+ public void testIsActiveShort() throws CliArgsException {
+ testIsActiveHelper("-e");
+ }
+
+ private void testIsActiveHelper(final String executorOption) throws CliArgsException {
+ final String expectedExecutorName = "test-executor";
+ final ConfigOption<Integer> configOption = key("test.int").intType().noDefaultValue();
+ final int expectedValue = 42;
+
+ final ExecutorCLI cliUnderTest = new ExecutorCLI(new Configuration());
+
+ final String[] args = {executorOption, expectedExecutorName, "-D" + configOption.key() + "=" + expectedValue};
+ final CommandLine commandLine = CliFrontendParser.parse(testOptions, args, true);
+
+ final Configuration configuration = cliUnderTest.applyCommandLineOptionsToConfiguration(commandLine);
+ assertEquals(expectedExecutorName, configuration.get(DeploymentOptions.TARGET));
+ assertEquals(expectedValue, configuration.getInteger(configOption));
+ }
+}