You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by tr...@apache.org on 2018/02/15 10:42:46 UTC
[03/22] flink git commit: [FLINK-8605] [rest] Enable job cancellation
from the web UI
http://git-wip-us.apache.org/repos/asf/flink/blob/2a154173/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/job/JobTerminationHandler.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/job/JobTerminationHandler.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/job/JobTerminationHandler.java
index 96f57f8..39d9bea 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/job/JobTerminationHandler.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/job/JobTerminationHandler.java
@@ -35,6 +35,7 @@ import org.apache.flink.runtime.rest.messages.TerminationModeQueryParameter;
import org.apache.flink.runtime.webmonitor.RestfulGateway;
import org.apache.flink.runtime.webmonitor.retriever.GatewayRetriever;
import org.apache.flink.util.ExceptionUtils;
+import org.apache.flink.util.Preconditions;
import org.apache.flink.shaded.netty4.io.netty.handler.codec.http.HttpResponseStatus;
@@ -49,13 +50,18 @@ import java.util.concurrent.TimeoutException;
*/
public class JobTerminationHandler extends AbstractRestHandler<RestfulGateway, EmptyRequestBody, EmptyResponseBody, JobTerminationMessageParameters> {
+ private final TerminationModeQueryParameter.TerminationMode defaultTerminationMode;
+
public JobTerminationHandler(
CompletableFuture<String> localRestAddress,
GatewayRetriever<? extends RestfulGateway> leaderRetriever,
Time timeout,
Map<String, String> headers,
- MessageHeaders<EmptyRequestBody, EmptyResponseBody, JobTerminationMessageParameters> messageHeaders) {
+ MessageHeaders<EmptyRequestBody, EmptyResponseBody, JobTerminationMessageParameters> messageHeaders,
+ TerminationModeQueryParameter.TerminationMode defaultTerminationMode) {
super(localRestAddress, leaderRetriever, timeout, headers, messageHeaders);
+
+ this.defaultTerminationMode = Preconditions.checkNotNull(defaultTerminationMode);
}
@Override
@@ -65,7 +71,7 @@ public class JobTerminationHandler extends AbstractRestHandler<RestfulGateway, E
final TerminationModeQueryParameter.TerminationMode terminationMode;
if (terminationModes.isEmpty()) {
- terminationMode = TerminationModeQueryParameter.TerminationMode.CANCEL;
+ terminationMode = defaultTerminationMode;
} else {
// picking the first termination mode value
terminationMode = terminationModes.get(0);
http://git-wip-us.apache.org/repos/asf/flink/blob/2a154173/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/YarnCancelJobTerminationHeaders.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/YarnCancelJobTerminationHeaders.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/YarnCancelJobTerminationHeaders.java
new file mode 100644
index 0000000..5333960
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/YarnCancelJobTerminationHeaders.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.rest.messages;
+
+import org.apache.flink.runtime.rest.HttpMethodWrapper;
+import org.apache.flink.runtime.rest.handler.RestHandlerSpecification;
+import org.apache.flink.runtime.rest.handler.job.JobTerminationHandler;
+
+/**
+ * {@link RestHandlerSpecification} for the {@link JobTerminationHandler} which is registered for
+ * compatibility with the Yarn proxy as a GET call.
+ *
+ * <p>For more information @see <a href="https://issues.apache.org/jira/browse/YARN-2031">YARN-2031</a>.
+ *
+ * @deprecated This should be removed once we can send arbitrary REST calls via the Yarn proxy.
+ */
+@Deprecated
+public class YarnCancelJobTerminationHeaders implements RestHandlerSpecification {
+
+ private static final YarnCancelJobTerminationHeaders INSTANCE = new YarnCancelJobTerminationHeaders();
+
+ private static final String URL = String.format("/jobs/:%s/yarn-cancel", JobIDPathParameter.KEY);
+
+ private YarnCancelJobTerminationHeaders() {}
+
+ @Override
+ public HttpMethodWrapper getHttpMethod() {
+ return HttpMethodWrapper.GET;
+ }
+
+ @Override
+ public String getTargetRestEndpointURL() {
+ return URL;
+ }
+
+ public static YarnCancelJobTerminationHeaders getInstance() {
+ return INSTANCE;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/2a154173/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/YarnStopJobTerminationHeaders.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/YarnStopJobTerminationHeaders.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/YarnStopJobTerminationHeaders.java
new file mode 100644
index 0000000..bbd36c0
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/YarnStopJobTerminationHeaders.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.rest.messages;
+
+import org.apache.flink.runtime.rest.HttpMethodWrapper;
+import org.apache.flink.runtime.rest.handler.RestHandlerSpecification;
+import org.apache.flink.runtime.rest.handler.job.JobTerminationHandler;
+
+/**
+ * {@link RestHandlerSpecification} for the {@link JobTerminationHandler} which is registered for
+ * compatibility with the Yarn proxy as a GET call.
+ *
+ * <p>For more information @see <a href="https://issues.apache.org/jira/browse/YARN-2031">YARN-2031</a>.
+ *
+ * @deprecated This should be removed once we can send arbitrary REST calls via the Yarn proxy.
+ */
+@Deprecated
+public class YarnStopJobTerminationHeaders implements RestHandlerSpecification {
+
+ private static final YarnStopJobTerminationHeaders INSTANCE = new YarnStopJobTerminationHeaders();
+
+ private static final String URL = String.format("/jobs/:%s/yarn-stop", JobIDPathParameter.KEY);
+
+ private YarnStopJobTerminationHeaders() {}
+
+ @Override
+ public HttpMethodWrapper getHttpMethod() {
+ return HttpMethodWrapper.GET;
+ }
+
+ @Override
+ public String getTargetRestEndpointURL() {
+ return URL;
+ }
+
+ public static YarnStopJobTerminationHeaders getInstance() {
+ return INSTANCE;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/2a154173/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorEndpoint.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorEndpoint.java b/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorEndpoint.java
index 5aa3271..19efcbb 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorEndpoint.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorEndpoint.java
@@ -85,6 +85,9 @@ import org.apache.flink.runtime.rest.messages.JobVertexBackPressureHeaders;
import org.apache.flink.runtime.rest.messages.JobVertexTaskManagersHeaders;
import org.apache.flink.runtime.rest.messages.JobsOverviewHeaders;
import org.apache.flink.runtime.rest.messages.SubtasksTimesHeaders;
+import org.apache.flink.runtime.rest.messages.TerminationModeQueryParameter;
+import org.apache.flink.runtime.rest.messages.YarnCancelJobTerminationHeaders;
+import org.apache.flink.runtime.rest.messages.YarnStopJobTerminationHeaders;
import org.apache.flink.runtime.rest.messages.checkpoints.CheckpointConfigHeaders;
import org.apache.flink.runtime.rest.messages.checkpoints.CheckpointStatisticDetailsHeaders;
import org.apache.flink.runtime.rest.messages.checkpoints.CheckpointingStatisticsHeaders;
@@ -451,12 +454,21 @@ public class WebMonitorEndpoint<T extends RestfulGateway> extends RestServerEndp
responseHeaders,
JobVertexBackPressureHeaders.getInstance());
- final JobTerminationHandler jobTerminationHandler = new JobTerminationHandler(
+ final JobTerminationHandler jobCancelTerminationHandler = new JobTerminationHandler(
restAddressFuture,
leaderRetriever,
timeout,
responseHeaders,
- JobTerminationHeaders.getInstance());
+ JobTerminationHeaders.getInstance(),
+ TerminationModeQueryParameter.TerminationMode.CANCEL);
+
+ final JobTerminationHandler jobStopTerminationHandler = new JobTerminationHandler(
+ restAddressFuture,
+ leaderRetriever,
+ timeout,
+ responseHeaders,
+ JobTerminationHeaders.getInstance(),
+ TerminationModeQueryParameter.TerminationMode.STOP);
final File tmpDir = restConfiguration.getTmpDir();
@@ -504,7 +516,11 @@ public class WebMonitorEndpoint<T extends RestfulGateway> extends RestServerEndp
handlers.add(Tuple2.of(SubtaskCurrentAttemptDetailsHeaders.getInstance(), subtaskCurrentAttemptDetailsHandler));
handlers.add(Tuple2.of(JobVertexTaskManagersHeaders.getInstance(), jobVertexTaskManagersHandler));
handlers.add(Tuple2.of(JobVertexBackPressureHeaders.getInstance(), jobVertexBackPressureHandler));
- handlers.add(Tuple2.of(JobTerminationHeaders.getInstance(), jobTerminationHandler));
+ handlers.add(Tuple2.of(JobTerminationHeaders.getInstance(), jobCancelTerminationHandler));
+
+ // TODO: Remove once the Yarn proxy can forward all REST verbs
+ handlers.add(Tuple2.of(YarnCancelJobTerminationHeaders.getInstance(), jobCancelTerminationHandler));
+ handlers.add(Tuple2.of(YarnStopJobTerminationHeaders.getInstance(), jobStopTerminationHandler));
optWebContent.ifPresent(
webContent -> handlers.add(Tuple2.of(WebContentHandlerSpecification.getInstance(), webContent)));
http://git-wip-us.apache.org/repos/asf/flink/blob/2a154173/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/YarnCancelJobTerminationHeadersTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/YarnCancelJobTerminationHeadersTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/YarnCancelJobTerminationHeadersTest.java
new file mode 100644
index 0000000..f1a016b
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/YarnCancelJobTerminationHeadersTest.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.rest.messages;
+
+import org.apache.flink.runtime.rest.HttpMethodWrapper;
+import org.apache.flink.util.TestLogger;
+
+import org.junit.Test;
+
+import static org.hamcrest.Matchers.endsWith;
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertThat;
+
+/**
+ * Tests for the {@link YarnCancelJobTerminationHeaders}.
+ */
+public class YarnCancelJobTerminationHeadersTest extends TestLogger {
+
+ // instance under test
+ private static final YarnCancelJobTerminationHeaders instance = YarnCancelJobTerminationHeaders.getInstance();
+
+ @Test
+ public void testMethod() {
+ assertThat(instance.getHttpMethod(), is(HttpMethodWrapper.GET));
+ }
+
+ @Test
+ public void testURL() {
+ assertThat(instance.getTargetRestEndpointURL(), endsWith("yarn-cancel"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/2a154173/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/YarnStopJobTerminationHeadersTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/YarnStopJobTerminationHeadersTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/YarnStopJobTerminationHeadersTest.java
new file mode 100644
index 0000000..a5d814f
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/YarnStopJobTerminationHeadersTest.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.rest.messages;
+
+import org.apache.flink.runtime.rest.HttpMethodWrapper;
+import org.apache.flink.util.TestLogger;
+
+import org.junit.Test;
+
+import static org.hamcrest.Matchers.endsWith;
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertThat;
+
+/**
+ * Tests for {@link YarnStopJobTerminationHeaders}.
+ */
+public class YarnStopJobTerminationHeadersTest extends TestLogger {
+
+ // instance under test
+ private static final YarnStopJobTerminationHeaders instance = YarnStopJobTerminationHeaders.getInstance();
+
+ @Test
+ public void testMethod() {
+ assertThat(instance.getHttpMethod(), is(HttpMethodWrapper.GET));
+ }
+
+ @Test
+ public void testURL() {
+ assertThat(instance.getTargetRestEndpointURL(), endsWith("yarn-stop"));
+ }
+
+}