You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ga...@apache.org on 2019/01/18 12:52:26 UTC
[flink] branch master updated (12260a2 -> 3cf593c)
This is an automated email from the ASF dual-hosted git repository.
gary pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git.
from 12260a2 [FLINK-8739] [table] Optimize DISTINCE aggregates to use the same distinct accumulator if possible
new 6516659 [hotfix][tests] Fix checkstyle violation and extract NoDataSource
new 3cf593c [FLINK-10665][tests] Remove legacy test YARNSessionFIFOITCase#testJavaAPI
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../java/org/apache/flink/yarn/NoDataSource.java | 24 ++-----
.../java/org/apache/flink/yarn/YARNITCase.java | 12 ----
.../apache/flink/yarn/YARNSessionFIFOITCase.java | 74 +---------------------
.../flink/yarn/YARNSessionFIFOSecuredITCase.java | 3 -
4 files changed, 8 insertions(+), 105 deletions(-)
copy flink-metrics/flink-metrics-core/src/test/java/org/apache/flink/metrics/util/TestMeter.java => flink-yarn-tests/src/test/java/org/apache/flink/yarn/NoDataSource.java (68%)
[flink] 02/02: [FLINK-10665][tests] Remove legacy test
YARNSessionFIFOITCase#testJavaAPI
Posted by ga...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gary pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 3cf593cb5400fcaece1f458c703d5093c202d807
Author: tison <wa...@gmail.com>
AuthorDate: Fri Jan 18 01:37:10 2019 +0800
[FLINK-10665][tests] Remove legacy test YARNSessionFIFOITCase#testJavaAPI
This closes #6917.
---
.../apache/flink/yarn/YARNSessionFIFOITCase.java | 75 +---------------------
.../flink/yarn/YARNSessionFIFOSecuredITCase.java | 3 -
2 files changed, 1 insertion(+), 77 deletions(-)
diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionFIFOITCase.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionFIFOITCase.java
index f6e097f..c8a90d7 100644
--- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionFIFOITCase.java
+++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionFIFOITCase.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -18,21 +18,15 @@
package org.apache.flink.yarn;
-import org.apache.flink.client.deployment.ClusterSpecification;
-import org.apache.flink.client.program.ClusterClient;
-import org.apache.flink.configuration.ConfigConstants;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.configuration.GlobalConfiguration;
import org.apache.flink.configuration.SecurityOptions;
-import org.apache.flink.runtime.clusterframework.messages.GetClusterStatusResponse;
import org.apache.flink.test.testdata.WordCountData;
import org.apache.flink.test.util.SecureTestEnvironment;
import org.apache.flink.yarn.cli.FlinkYarnSessionCli;
import org.apache.flink.yarn.configuration.YarnConfigOptions;
-import org.apache.flink.yarn.util.YarnTestUtils;
import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
@@ -52,7 +46,6 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.EnumSet;
import java.util.List;
import java.util.concurrent.TimeUnit;
@@ -271,70 +264,4 @@ public class YARNSessionFIFOITCase extends YarnTestBase {
checkForLogString("There is not enough memory available in the YARN cluster. The TaskManager(s) require 3840MB each. NodeManagers available: [4096, 4096]\n" +
"After allocating the JobManager (512MB) and (1/2) TaskManagers, the following NodeManagers are available: [3584, 256]");
}
-
- /**
- * Test the YARN Java API.
- */
- @Test
- public void testJavaAPI() throws Exception {
- final int waitTime = 15;
- LOG.info("Starting testJavaAPI()");
-
- String confDirPath = System.getenv(ConfigConstants.ENV_FLINK_CONF_DIR);
- Configuration configuration = GlobalConfiguration.loadConfiguration();
-
- try (final AbstractYarnClusterDescriptor clusterDescriptor = new LegacyYarnClusterDescriptor(
- configuration,
- getYarnConfiguration(),
- confDirPath,
- getYarnClient(),
- true)) {
- Assert.assertNotNull("unable to get yarn client", clusterDescriptor);
- clusterDescriptor.setLocalJarPath(new Path(flinkUberjar.getAbsolutePath()));
- clusterDescriptor.addShipFiles(Arrays.asList(flinkLibFolder.listFiles()));
-
- final ClusterSpecification clusterSpecification = new ClusterSpecification.ClusterSpecificationBuilder()
- .setMasterMemoryMB(768)
- .setTaskManagerMemoryMB(1024)
- .setNumberTaskManagers(1)
- .setSlotsPerTaskManager(1)
- .createClusterSpecification();
- // deploy
- ClusterClient<ApplicationId> yarnClusterClient = null;
- try {
- yarnClusterClient = clusterDescriptor.deploySessionCluster(clusterSpecification);
-
- GetClusterStatusResponse expectedStatus = new GetClusterStatusResponse(1, 1);
- for (int second = 0; second < waitTime * 2; second++) { // run "forever"
- try {
- Thread.sleep(1000);
- } catch (InterruptedException e) {
- LOG.warn("Interrupted", e);
- }
- GetClusterStatusResponse status = yarnClusterClient.getClusterStatus();
- if (status != null && status.equals(expectedStatus)) {
- LOG.info("ClusterClient reached status " + status);
- break; // all good, cluster started
- }
- if (second > waitTime) {
- // we waited for 15 seconds. cluster didn't come up correctly
- Assert.fail("The custer didn't start after " + waitTime + " seconds");
- }
- }
-
- // use the cluster
- Assert.assertNotNull(yarnClusterClient.getClusterConnectionInfo());
- Assert.assertNotNull(yarnClusterClient.getWebInterfaceURL());
- LOG.info("All tests passed.");
- } finally {
- if (yarnClusterClient != null) {
- // shutdown cluster
- LOG.info("Shutting down the Flink Yarn application.");
- yarnClusterClient.shutDownCluster();
- yarnClusterClient.shutdown();
- }
- }
- }
- LOG.info("Finished testJavaAPI()");
- }
}
diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionFIFOSecuredITCase.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionFIFOSecuredITCase.java
index 97f60fc..d9a79b6 100644
--- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionFIFOSecuredITCase.java
+++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionFIFOSecuredITCase.java
@@ -129,7 +129,4 @@ public class YARNSessionFIFOSecuredITCase extends YARNSessionFIFOITCase {
@Override
public void testfullAlloc() {}
-
- @Override
- public void testJavaAPI() {}
}
[flink] 01/02: [hotfix][tests] Fix checkstyle violation and extract
NoDataSource
Posted by ga...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gary pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 6516659e603939344d6d673f34e3e233ef7b7593
Author: Tzu-Li Chen <wa...@gmail.com>
AuthorDate: Wed Oct 24 23:41:27 2018 +0800
[hotfix][tests] Fix checkstyle violation and extract NoDataSource
---
.../java/org/apache/flink/yarn/NoDataSource.java | 35 ++++++++++++++++++++++
.../java/org/apache/flink/yarn/YARNITCase.java | 12 --------
.../apache/flink/yarn/YARNSessionFIFOITCase.java | 1 +
3 files changed, 36 insertions(+), 12 deletions(-)
diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/NoDataSource.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/NoDataSource.java
new file mode 100644
index 0000000..34d8d70
--- /dev/null
+++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/NoDataSource.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.yarn;
+
+import org.apache.flink.streaming.api.functions.source.ParallelSourceFunction;
+
+/**
+ * Parallel data source that produces no data, i.e., finishes immediately.
+ */
+public class NoDataSource implements ParallelSourceFunction<Integer> {
+
+ private static final long serialVersionUID = 1642561062000662861L;
+
+ @Override
+ public void run(SourceContext<Integer> ctx) {}
+
+ @Override
+ public void cancel() {}
+}
diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNITCase.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNITCase.java
index 7f9ea7e..4420f89 100644
--- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNITCase.java
+++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNITCase.java
@@ -28,7 +28,6 @@ import org.apache.flink.runtime.jobgraph.JobGraph;
import org.apache.flink.runtime.jobmaster.JobResult;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.sink.DiscardingSink;
-import org.apache.flink.streaming.api.functions.source.ParallelSourceFunction;
import org.apache.flink.yarn.util.YarnTestUtils;
import org.apache.hadoop.fs.Path;
@@ -123,15 +122,4 @@ public class YARNITCase extends YarnTestBase {
}
}
}
-
- private static class NoDataSource implements ParallelSourceFunction<Integer> {
-
- private static final long serialVersionUID = 1642561062000662861L;
-
- @Override
- public void run(SourceContext<Integer> ctx) {}
-
- @Override
- public void cancel() {}
- }
}
diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionFIFOITCase.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionFIFOITCase.java
index fb2c6cc..f6e097f 100644
--- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionFIFOITCase.java
+++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionFIFOITCase.java
@@ -29,6 +29,7 @@ import org.apache.flink.test.testdata.WordCountData;
import org.apache.flink.test.util.SecureTestEnvironment;
import org.apache.flink.yarn.cli.FlinkYarnSessionCli;
import org.apache.flink.yarn.configuration.YarnConfigOptions;
+import org.apache.flink.yarn.util.YarnTestUtils;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.fs.Path;