You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by rm...@apache.org on 2016/07/05 14:38:38 UTC
[01/15] flink git commit: [FLINK-3995] [build] Properly structure
test scopes and dependencies
Repository: flink
Updated Branches:
refs/heads/master 5c0c7e36e -> 8a2c03e1a
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-test-utils/src/test/java/org/apache/flink/test/util/TestBaseUtils.java
----------------------------------------------------------------------
diff --git a/flink-test-utils/src/test/java/org/apache/flink/test/util/TestBaseUtils.java b/flink-test-utils/src/test/java/org/apache/flink/test/util/TestBaseUtils.java
deleted file mode 100644
index 4dda4cf..0000000
--- a/flink-test-utils/src/test/java/org/apache/flink/test/util/TestBaseUtils.java
+++ /dev/null
@@ -1,666 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.test.util;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import akka.actor.ActorRef;
-import akka.dispatch.Futures;
-import akka.pattern.Patterns;
-import akka.util.Timeout;
-
-import com.google.common.collect.Lists;
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.io.IOUtils;
-import org.apache.flink.api.java.tuple.Tuple;
-import org.apache.flink.configuration.ConfigConstants;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.runtime.testingUtils.TestingTaskManagerMessages;
-import org.apache.flink.runtime.testingUtils.TestingUtils;
-import org.apache.flink.util.TestLogger;
-import org.apache.hadoop.fs.FileSystem;
-import org.junit.Assert;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import scala.concurrent.Await;
-import scala.concurrent.Future;
-import scala.concurrent.duration.FiniteDuration;
-
-import java.io.BufferedInputStream;
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileReader;
-import java.io.FilenameFilter;
-import java.io.IOException;
-import java.io.InputStream;
-import java.lang.reflect.Field;
-import java.net.HttpURLConnection;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.net.URL;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-public class TestBaseUtils extends TestLogger {
-
- private static final Logger LOG = LoggerFactory.getLogger(TestBaseUtils.class);
-
- protected static final int MINIMUM_HEAP_SIZE_MB = 192;
-
- protected static final long TASK_MANAGER_MEMORY_SIZE = 80;
-
- protected static final long DEFAULT_AKKA_ASK_TIMEOUT = 1000;
-
- protected static final String DEFAULT_AKKA_STARTUP_TIMEOUT = "60 s";
-
- protected static FiniteDuration DEFAULT_TIMEOUT = new FiniteDuration(DEFAULT_AKKA_ASK_TIMEOUT, TimeUnit.SECONDS);
-
- // ------------------------------------------------------------------------
-
- protected static File logDir;
-
- protected TestBaseUtils(){
- verifyJvmOptions();
- }
-
- private static void verifyJvmOptions() {
- long heap = Runtime.getRuntime().maxMemory() >> 20;
- Assert.assertTrue("Insufficient java heap space " + heap + "mb - set JVM option: -Xmx" + MINIMUM_HEAP_SIZE_MB
- + "m", heap > MINIMUM_HEAP_SIZE_MB - 50);
- }
-
-
- public static ForkableFlinkMiniCluster startCluster(
- int numTaskManagers,
- int taskManagerNumSlots,
- boolean startWebserver,
- boolean startZooKeeper,
- boolean singleActorSystem) throws Exception {
-
- Configuration config = new Configuration();
-
- config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, numTaskManagers);
- config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, taskManagerNumSlots);
-
- config.setBoolean(ConfigConstants.LOCAL_START_WEBSERVER, startWebserver);
-
- if (startZooKeeper) {
- config.setInteger(ConfigConstants.LOCAL_NUMBER_JOB_MANAGER, 3);
- config.setString(ConfigConstants.RECOVERY_MODE, "zookeeper");
- }
-
- return startCluster(config, singleActorSystem);
- }
-
- public static ForkableFlinkMiniCluster startCluster(
- Configuration config,
- boolean singleActorSystem) throws Exception {
-
- logDir = File.createTempFile("TestBaseUtils-logdir", null);
- Assert.assertTrue("Unable to delete temp file", logDir.delete());
- Assert.assertTrue("Unable to create temp directory", logDir.mkdir());
- Path logFile = Files.createFile(new File(logDir, "jobmanager.log").toPath());
- Files.createFile(new File(logDir, "jobmanager.out").toPath());
-
- config.setLong(ConfigConstants.TASK_MANAGER_MEMORY_SIZE_KEY, TASK_MANAGER_MEMORY_SIZE);
- config.setBoolean(ConfigConstants.FILESYSTEM_DEFAULT_OVERWRITE_KEY, true);
-
- config.setString(ConfigConstants.AKKA_ASK_TIMEOUT, DEFAULT_AKKA_ASK_TIMEOUT + "s");
- config.setString(ConfigConstants.AKKA_STARTUP_TIMEOUT, DEFAULT_AKKA_STARTUP_TIMEOUT);
-
- config.setInteger(ConfigConstants.JOB_MANAGER_WEB_PORT_KEY, 8081);
- config.setString(ConfigConstants.JOB_MANAGER_WEB_LOG_PATH_KEY, logFile.toString());
-
- config.setString(ConfigConstants.TASK_MANAGER_LOG_PATH_KEY, logFile.toString());
-
- ForkableFlinkMiniCluster cluster = new ForkableFlinkMiniCluster(config, singleActorSystem);
-
- cluster.start();
-
- return cluster;
- }
-
-
- public static void stopCluster(ForkableFlinkMiniCluster executor, FiniteDuration timeout) throws Exception {
- if (logDir != null) {
- FileUtils.deleteDirectory(logDir);
- }
- if (executor != null) {
- int numUnreleasedBCVars = 0;
- int numActiveConnections = 0;
-
- if (executor.running()) {
- List<ActorRef> tms = executor.getTaskManagersAsJava();
- List<Future<Object>> bcVariableManagerResponseFutures = new ArrayList<>();
- List<Future<Object>> numActiveConnectionsResponseFutures = new ArrayList<>();
-
- for (ActorRef tm : tms) {
- bcVariableManagerResponseFutures.add(Patterns.ask(tm, TestingTaskManagerMessages
- .RequestBroadcastVariablesWithReferences$.MODULE$, new Timeout(timeout)));
-
- numActiveConnectionsResponseFutures.add(Patterns.ask(tm, TestingTaskManagerMessages
- .RequestNumActiveConnections$.MODULE$, new Timeout(timeout)));
- }
-
- Future<Iterable<Object>> bcVariableManagerFutureResponses = Futures.sequence(
- bcVariableManagerResponseFutures, TestingUtils.defaultExecutionContext());
-
- Iterable<Object> responses = Await.result(bcVariableManagerFutureResponses, timeout);
-
- for (Object response : responses) {
- numUnreleasedBCVars += ((TestingTaskManagerMessages
- .ResponseBroadcastVariablesWithReferences) response).number();
- }
-
- Future<Iterable<Object>> numActiveConnectionsFutureResponses = Futures.sequence(
- numActiveConnectionsResponseFutures, TestingUtils.defaultExecutionContext());
-
- responses = Await.result(numActiveConnectionsFutureResponses, timeout);
-
- for (Object response : responses) {
- numActiveConnections += ((TestingTaskManagerMessages
- .ResponseNumActiveConnections) response).number();
- }
- }
-
- executor.stop();
- FileSystem.closeAll();
- System.gc();
-
- Assert.assertEquals("Not all broadcast variables were released.", 0, numUnreleasedBCVars);
- Assert.assertEquals("Not all TCP connections were released.", 0, numActiveConnections);
- }
-
- }
-
- // --------------------------------------------------------------------------------------------
- // Result Checking
- // --------------------------------------------------------------------------------------------
-
- public static BufferedReader[] getResultReader(String resultPath) throws IOException {
- return getResultReader(resultPath, new String[]{}, false);
- }
-
- public static BufferedReader[] getResultReader(String resultPath, String[] excludePrefixes,
- boolean inOrderOfFiles) throws IOException {
- File[] files = getAllInvolvedFiles(resultPath, excludePrefixes);
-
- if (inOrderOfFiles) {
- // sort the files after their name (1, 2, 3, 4)...
- // we cannot sort by path, because strings sort by prefix
- Arrays.sort(files, new Comparator<File>() {
-
- @Override
- public int compare(File o1, File o2) {
- try {
- int f1 = Integer.parseInt(o1.getName());
- int f2 = Integer.parseInt(o2.getName());
- return f1 < f2 ? -1 : (f1 > f2 ? 1 : 0);
- }
- catch (NumberFormatException e) {
- throw new RuntimeException("The file names are no numbers and cannot be ordered: " +
- o1.getName() + "/" + o2.getName());
- }
- }
- });
- }
-
- BufferedReader[] readers = new BufferedReader[files.length];
- for (int i = 0; i < files.length; i++) {
- readers[i] = new BufferedReader(new FileReader(files[i]));
- }
- return readers;
- }
-
- public static BufferedInputStream[] getResultInputStream(String resultPath) throws IOException {
- return getResultInputStream(resultPath, new String[]{});
- }
-
- public static BufferedInputStream[] getResultInputStream(String resultPath, String[]
- excludePrefixes) throws IOException {
- File[] files = getAllInvolvedFiles(resultPath, excludePrefixes);
- BufferedInputStream[] inStreams = new BufferedInputStream[files.length];
- for (int i = 0; i < files.length; i++) {
- inStreams[i] = new BufferedInputStream(new FileInputStream(files[i]));
- }
- return inStreams;
- }
-
- public static void readAllResultLines(List<String> target, String resultPath) throws IOException {
- readAllResultLines(target, resultPath, new String[]{});
- }
-
- public static void readAllResultLines(List<String> target, String resultPath, String[] excludePrefixes)
- throws IOException {
-
- readAllResultLines(target, resultPath, excludePrefixes, false);
- }
-
- public static void readAllResultLines(List<String> target, String resultPath,
- String[] excludePrefixes, boolean inOrderOfFiles) throws IOException {
-
- final BufferedReader[] readers = getResultReader(resultPath, excludePrefixes, inOrderOfFiles);
- try {
- for (BufferedReader reader : readers) {
- String s;
- while ((s = reader.readLine()) != null) {
- target.add(s);
- }
- }
- }
- finally {
- for (BufferedReader reader : readers) {
- try {
- reader.close();
- }
- catch (Exception e) {
- // ignore, this is best-effort cleanup
- }
- }
- }
- }
-
- public static void compareResultsByLinesInMemory(String expectedResultStr, String resultPath) throws Exception {
- compareResultsByLinesInMemory(expectedResultStr, resultPath, new String[0]);
- }
-
- public static void compareResultsByLinesInMemory(String expectedResultStr, String resultPath,
- String[] excludePrefixes) throws Exception {
- ArrayList<String> list = new ArrayList<>();
- readAllResultLines(list, resultPath, excludePrefixes, false);
-
- String[] result = list.toArray(new String[list.size()]);
- Arrays.sort(result);
-
- String[] expected = expectedResultStr.isEmpty() ? new String[0] : expectedResultStr.split("\n");
- Arrays.sort(expected);
-
- Assert.assertEquals("Different number of lines in expected and obtained result.", expected.length, result.length);
- Assert.assertArrayEquals(expected, result);
- }
-
- public static void compareResultsByLinesInMemoryWithStrictOrder(String expectedResultStr,
- String resultPath) throws Exception {
- compareResultsByLinesInMemoryWithStrictOrder(expectedResultStr, resultPath, new String[]{});
- }
-
- public static void compareResultsByLinesInMemoryWithStrictOrder(String expectedResultStr,
- String resultPath, String[] excludePrefixes) throws Exception {
- ArrayList<String> list = new ArrayList<>();
- readAllResultLines(list, resultPath, excludePrefixes, true);
-
- String[] result = list.toArray(new String[list.size()]);
-
- String[] expected = expectedResultStr.split("\n");
-
- Assert.assertEquals("Different number of lines in expected and obtained result.", expected.length, result.length);
- Assert.assertArrayEquals(expected, result);
- }
-
- public static void checkLinesAgainstRegexp(String resultPath, String regexp){
- Pattern pattern = Pattern.compile(regexp);
- Matcher matcher = pattern.matcher("");
-
- ArrayList<String> list = new ArrayList<>();
- try {
- readAllResultLines(list, resultPath, new String[]{}, false);
- } catch (IOException e1) {
- Assert.fail("Error reading the result");
- }
-
- for (String line : list){
- matcher.reset(line);
- if (!matcher.find()){
- String msg = "Line is not well-formed: " + line;
- Assert.fail(msg);
- }
- }
- }
-
- public static void compareKeyValuePairsWithDelta(String expectedLines, String resultPath,
- String delimiter, double maxDelta) throws Exception {
- compareKeyValuePairsWithDelta(expectedLines, resultPath, new String[]{}, delimiter, maxDelta);
- }
-
- public static void compareKeyValuePairsWithDelta(String expectedLines, String resultPath,
- String[] excludePrefixes, String delimiter, double maxDelta) throws Exception {
- ArrayList<String> list = new ArrayList<>();
- readAllResultLines(list, resultPath, excludePrefixes, false);
-
- String[] result = list.toArray(new String[list.size()]);
- String[] expected = expectedLines.isEmpty() ? new String[0] : expectedLines.split("\n");
-
- Assert.assertEquals("Wrong number of result lines.", expected.length, result.length);
-
- Arrays.sort(result);
- Arrays.sort(expected);
-
- for (int i = 0; i < expected.length; i++) {
- String[] expectedFields = expected[i].split(delimiter);
- String[] resultFields = result[i].split(delimiter);
-
- double expectedPayLoad = Double.parseDouble(expectedFields[1]);
- double resultPayLoad = Double.parseDouble(resultFields[1]);
-
- Assert.assertTrue("Values differ by more than the permissible delta", Math.abs(expectedPayLoad - resultPayLoad) < maxDelta);
- }
- }
-
- public static <X> void compareResultCollections(List<X> expected, List<X> actual,
- Comparator<X> comparator) {
- Assert.assertEquals(expected.size(), actual.size());
-
- Collections.sort(expected, comparator);
- Collections.sort(actual, comparator);
-
- for (int i = 0; i < expected.size(); i++) {
- Assert.assertEquals(expected.get(i), actual.get(i));
- }
- }
-
- private static File[] getAllInvolvedFiles(String resultPath, String[] excludePrefixes) {
- final String[] exPrefs = excludePrefixes;
- File result = asFile(resultPath);
- if (!result.exists()) {
- Assert.fail("Result file was not written");
- }
- if (result.isDirectory()) {
- return result.listFiles(new FilenameFilter() {
-
- @Override
- public boolean accept(File dir, String name) {
- for(String p: exPrefs) {
- if(name.startsWith(p)) {
- return false;
- }
- }
- return true;
- }
- });
- } else {
- return new File[] { result };
- }
- }
-
- protected static File asFile(String path) {
- try {
- URI uri = new URI(path);
- if (uri.getScheme().equals("file")) {
- return new File(uri.getPath());
- } else {
- throw new IllegalArgumentException("This path does not denote a local file.");
- }
- } catch (URISyntaxException | NullPointerException e) {
- throw new IllegalArgumentException("This path does not describe a valid local file URI.");
- }
- }
-
- // --------------------------------------------------------------------------------------------
- // Comparison methods for tests using collect()
- // --------------------------------------------------------------------------------------------
-
- public static <T> void compareResultAsTuples(List<T> result, String expected) {
- compareResult(result, expected, true, true);
- }
-
- public static <T> void compareResultAsText(List<T> result, String expected) {
- compareResult(result, expected,
- false, true);
- }
-
- public static <T> void compareOrderedResultAsText(List<T> result, String expected) {
- compareResult(result, expected, false, false);
- }
-
- public static <T> void compareOrderedResultAsText(List<T> result, String expected, boolean asTuples) {
- compareResult(result, expected, asTuples, false);
- }
-
- private static <T> void compareResult(List<T> result, String expected, boolean asTuples, boolean sort) {
- String[] expectedStrings = expected.split("\n");
- String[] resultStrings = new String[result.size()];
-
- for (int i = 0; i < resultStrings.length; i++) {
- T val = result.get(i);
-
- if (asTuples) {
- if (val instanceof Tuple) {
- Tuple t = (Tuple) val;
- Object first = t.getField(0);
- StringBuilder bld = new StringBuilder(first == null ? "null" : first.toString());
- for (int pos = 1; pos < t.getArity(); pos++) {
- Object next = t.getField(pos);
- bld.append(',').append(next == null ? "null" : next.toString());
- }
- resultStrings[i] = bld.toString();
- }
- else {
- throw new IllegalArgumentException(val + " is no tuple");
- }
- }
- else {
- resultStrings[i] = (val == null) ? "null" : val.toString();
- }
- }
-
- assertEquals("Wrong number of elements result", expectedStrings.length, resultStrings.length);
-
- if (sort) {
- Arrays.sort(expectedStrings);
- Arrays.sort(resultStrings);
- }
-
- for (int i = 0; i < expectedStrings.length; i++) {
- assertEquals(expectedStrings[i], resultStrings[i]);
- }
- }
-
- // --------------------------------------------------------------------------------------------
- // Comparison methods for tests using sample
- // --------------------------------------------------------------------------------------------
-
- /**
- * The expected string contains all expected results separate with line break, check whether all elements in result
- * are contained in the expected string.
- * @param result The test result.
- * @param expected The expected string value combination.
- * @param <T> The result type.
- */
- public static <T> void containsResultAsText(List<T> result, String expected) {
- String[] expectedStrings = expected.split("\n");
- List<String> resultStrings = Lists.newLinkedList();
-
- for (int i = 0; i < result.size(); i++) {
- T val = result.get(i);
- String str = (val == null) ? "null" : val.toString();
- resultStrings.add(str);
- }
-
- List<String> expectedStringList = Arrays.asList(expectedStrings);
-
- for (String element : resultStrings) {
- assertTrue(expectedStringList.contains(element));
- }
- }
-
- // --------------------------------------------------------------------------------------------
- // Miscellaneous helper methods
- // --------------------------------------------------------------------------------------------
-
- protected static Collection<Object[]> toParameterList(Configuration ... testConfigs) {
- ArrayList<Object[]> configs = new ArrayList<>();
- for (Configuration testConfig : testConfigs) {
- Object[] c = { testConfig };
- configs.add(c);
- }
- return configs;
- }
-
- protected static Collection<Object[]> toParameterList(List<Configuration> testConfigs) {
- LinkedList<Object[]> configs = new LinkedList<>();
- for (Configuration testConfig : testConfigs) {
- Object[] c = { testConfig };
- configs.add(c);
- }
- return configs;
- }
-
- // This code is taken from: http://stackoverflow.com/a/7201825/568695
- // it changes the environment variables of this JVM. Use only for testing purposes!
- @SuppressWarnings("unchecked")
- public static void setEnv(Map<String, String> newenv) {
- try {
- Class<?> processEnvironmentClass = Class.forName("java.lang.ProcessEnvironment");
- Field theEnvironmentField = processEnvironmentClass.getDeclaredField("theEnvironment");
- theEnvironmentField.setAccessible(true);
- Map<String, String> env = (Map<String, String>) theEnvironmentField.get(null);
- env.putAll(newenv);
- Field theCaseInsensitiveEnvironmentField = processEnvironmentClass.getDeclaredField("theCaseInsensitiveEnvironment");
- theCaseInsensitiveEnvironmentField.setAccessible(true);
- Map<String, String> cienv = (Map<String, String>) theCaseInsensitiveEnvironmentField.get(null);
- cienv.putAll(newenv);
- } catch (NoSuchFieldException e) {
- try {
- Class<?>[] classes = Collections.class.getDeclaredClasses();
- Map<String, String> env = System.getenv();
- for (Class<?> cl : classes) {
- if ("java.util.Collections$UnmodifiableMap".equals(cl.getName())) {
- Field field = cl.getDeclaredField("m");
- field.setAccessible(true);
- Object obj = field.get(env);
- Map<String, String> map = (Map<String, String>) obj;
- map.clear();
- map.putAll(newenv);
- }
- }
- } catch (Exception e2) {
- throw new RuntimeException(e2);
- }
- } catch (Exception e1) {
- throw new RuntimeException(e1);
- }
- }
- // --------------------------------------------------------------------------------------------
- // File helper methods
- // --------------------------------------------------------------------------------------------
-
- protected static void deleteRecursively(File f) throws IOException {
- if (f.isDirectory()) {
- FileUtils.deleteDirectory(f);
- } else if (!f.delete()) {
- System.err.println("Failed to delete file " + f.getAbsolutePath());
- }
- }
-
- public static String constructTestPath(Class<?> forClass, String folder) {
- // we create test path that depends on class to prevent name clashes when two tests
- // create temp files with the same name
- String path = System.getProperty("java.io.tmpdir");
- if (!(path.endsWith("/") || path.endsWith("\\")) ) {
- path += System.getProperty("file.separator");
- }
- path += (forClass.getName() + "-" + folder);
- return path;
- }
-
- public static String constructTestURI(Class<?> forClass, String folder) {
- return new File(constructTestPath(forClass, folder)).toURI().toString();
- }
-
- //---------------------------------------------------------------------------------------------
- // Web utils
- //---------------------------------------------------------------------------------------------
-
- public static String getFromHTTP(String url) throws Exception {
- URL u = new URL(url);
- LOG.info("Accessing URL "+url+" as URL: "+u);
- HttpURLConnection connection = (HttpURLConnection) u.openConnection();
- connection.setConnectTimeout(100000);
- connection.connect();
- InputStream is;
- if(connection.getResponseCode() >= 400) {
- // error!
- LOG.warn("HTTP Response code when connecting to {} was {}", url, connection.getResponseCode());
- is = connection.getErrorStream();
- } else {
- is = connection.getInputStream();
- }
-
- return IOUtils.toString(is, connection.getContentEncoding() != null ? connection.getContentEncoding() : "UTF-8");
- }
-
- public static class TupleComparator<T extends Tuple> implements Comparator<T> {
-
- @Override
- public int compare(T o1, T o2) {
- if (o1 == null || o2 == null) {
- throw new IllegalArgumentException("Cannot compare null tuples");
- }
- else if (o1.getArity() != o2.getArity()) {
- return o1.getArity() - o2.getArity();
- }
- else {
- for (int i = 0; i < o1.getArity(); i++) {
- Object val1 = o1.getField(i);
- Object val2 = o2.getField(i);
-
- int cmp;
- if (val1 != null && val2 != null) {
- cmp = compareValues(val1, val2);
- }
- else {
- cmp = val1 == null ? (val2 == null ? 0 : -1) : 1;
- }
-
- if (cmp != 0) {
- return cmp;
- }
- }
-
- return 0;
- }
- }
-
- @SuppressWarnings("unchecked")
- private static <X extends Comparable<X>> int compareValues(Object o1, Object o2) {
- if (o1 instanceof Comparable && o2 instanceof Comparable) {
- X c1 = (X) o1;
- X c2 = (X) o2;
- return c1.compareTo(c2);
- }
- else {
- throw new IllegalArgumentException("Cannot compare tuples with non comparable elements");
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-test-utils/src/test/java/org/apache/flink/test/util/TestEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-test-utils/src/test/java/org/apache/flink/test/util/TestEnvironment.java b/flink-test-utils/src/test/java/org/apache/flink/test/util/TestEnvironment.java
deleted file mode 100644
index 7cb88be..0000000
--- a/flink-test-utils/src/test/java/org/apache/flink/test/util/TestEnvironment.java
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.test.util;
-
-import org.apache.flink.api.common.JobExecutionResult;
-import org.apache.flink.api.common.Plan;
-import org.apache.flink.api.common.CodeAnalysisMode;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.ExecutionEnvironmentFactory;
-import org.apache.flink.optimizer.DataStatistics;
-import org.apache.flink.optimizer.Optimizer;
-import org.apache.flink.optimizer.plan.OptimizedPlan;
-import org.apache.flink.optimizer.plandump.PlanJSONDumpGenerator;
-import org.apache.flink.optimizer.plantranslate.JobGraphGenerator;
-import org.apache.flink.runtime.jobgraph.JobGraph;
-
-public class TestEnvironment extends ExecutionEnvironment {
-
- private final ForkableFlinkMiniCluster executor;
-
- private TestEnvironment lastEnv = null;
-
- @Override
- public JobExecutionResult getLastJobExecutionResult() {
- if (lastEnv == null) {
- return this.lastJobExecutionResult;
- }
- else {
- return lastEnv.getLastJobExecutionResult();
- }
- }
-
- public TestEnvironment(ForkableFlinkMiniCluster executor, int parallelism) {
- this.executor = executor;
- setParallelism(parallelism);
-
- // disabled to improve build time
- getConfig().setCodeAnalysisMode(CodeAnalysisMode.DISABLE);
- }
-
- public TestEnvironment(ForkableFlinkMiniCluster executor, int parallelism, boolean isObjectReuseEnabled) {
- this(executor, parallelism);
-
- if (isObjectReuseEnabled) {
- getConfig().enableObjectReuse();
- } else {
- getConfig().disableObjectReuse();
- }
- }
-
- @Override
- public void startNewSession() throws Exception {
- }
-
- @Override
- public JobExecutionResult execute(String jobName) throws Exception {
- OptimizedPlan op = compileProgram(jobName);
-
- JobGraphGenerator jgg = new JobGraphGenerator();
- JobGraph jobGraph = jgg.compileJobGraph(op);
-
- this.lastJobExecutionResult = executor.submitJobAndWait(jobGraph, false);
- return this.lastJobExecutionResult;
- }
-
-
- @Override
- public String getExecutionPlan() throws Exception {
- OptimizedPlan op = compileProgram("unused");
-
- PlanJSONDumpGenerator jsonGen = new PlanJSONDumpGenerator();
- return jsonGen.getOptimizerPlanAsJSON(op);
- }
-
-
- private OptimizedPlan compileProgram(String jobName) {
- Plan p = createProgramPlan(jobName);
-
- Optimizer pc = new Optimizer(new DataStatistics(), this.executor.configuration());
- return pc.compile(p);
- }
-
- public void setAsContext() {
- ExecutionEnvironmentFactory factory = new ExecutionEnvironmentFactory() {
- @Override
- public ExecutionEnvironment createExecutionEnvironment() {
- lastEnv = new TestEnvironment(executor, getParallelism(), getConfig().isObjectReuseEnabled());
- return lastEnv;
- }
- };
-
- initializeContextEnvironment(factory);
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-test-utils/src/test/scala/org/apache/flink/test/util/FlinkTestBase.scala
----------------------------------------------------------------------
diff --git a/flink-test-utils/src/test/scala/org/apache/flink/test/util/FlinkTestBase.scala b/flink-test-utils/src/test/scala/org/apache/flink/test/util/FlinkTestBase.scala
deleted file mode 100644
index 1f00c7e..0000000
--- a/flink-test-utils/src/test/scala/org/apache/flink/test/util/FlinkTestBase.scala
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.test.util
-
-import org.scalatest.{Suite, BeforeAndAfter}
-
-/** Mixin to start and stop a ForkableFlinkMiniCluster automatically for Scala based tests.
- * Additionally a TestEnvironment with the started cluster is created and set as the default
- * [[org.apache.flink.api.java.ExecutionEnvironment]].
- *
- * This mixin starts a ForkableFlinkMiniCluster with one TaskManager and a number of slots given
- * by parallelism. This value can be overridden in a sub class in order to start the cluster
- * with a different number of slots.
- *
- * The cluster is started once before starting the tests and is re-used for the individual tests.
- * After all tests have been executed, the cluster is shutdown.
- *
- * The cluster is used by obtaining the default [[org.apache.flink.api.java.ExecutionEnvironment]].
- *
- * @example
- * {{{
- * def testSomething: Unit = {
- * // Obtain TestEnvironment with started ForkableFlinkMiniCluster
- * val env = ExecutionEnvironment.getExecutionEnvironment
- *
- * env.fromCollection(...)
- *
- * env.execute
- * }
- * }}}
- *
- */
-trait FlinkTestBase extends BeforeAndAfter {
- that: Suite =>
-
- var cluster: Option[ForkableFlinkMiniCluster] = None
- val parallelism = 4
-
- before {
- val cl = TestBaseUtils.startCluster(
- 1,
- parallelism,
- false,
- false,
- true)
-
- val clusterEnvironment = new TestEnvironment(cl, parallelism)
- clusterEnvironment.setAsContext()
-
- cluster = Some(cl)
- }
-
- after {
- cluster.foreach(c => TestBaseUtils.stopCluster(c, TestBaseUtils.DEFAULT_TIMEOUT))
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-test-utils/src/test/scala/org/apache/flink/test/util/ForkableFlinkMiniCluster.scala
----------------------------------------------------------------------
diff --git a/flink-test-utils/src/test/scala/org/apache/flink/test/util/ForkableFlinkMiniCluster.scala b/flink-test-utils/src/test/scala/org/apache/flink/test/util/ForkableFlinkMiniCluster.scala
deleted file mode 100644
index 8a348a1..0000000
--- a/flink-test-utils/src/test/scala/org/apache/flink/test/util/ForkableFlinkMiniCluster.scala
+++ /dev/null
@@ -1,305 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.test.util
-
-import java.util.concurrent.TimeoutException
-
-import akka.actor.{ActorRef, ActorSystem}
-import akka.pattern.Patterns._
-import akka.pattern.ask
-import org.apache.curator.test.TestingCluster
-import org.apache.flink.configuration.{ConfigConstants, Configuration}
-import org.apache.flink.runtime.akka.AkkaUtils
-import org.apache.flink.runtime.clusterframework.FlinkResourceManager
-import org.apache.flink.runtime.clusterframework.types.ResourceID
-import org.apache.flink.runtime.jobmanager.{JobManager, RecoveryMode}
-import org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster
-import org.apache.flink.runtime.taskmanager.TaskManager
-import org.apache.flink.runtime.testingUtils.TestingTaskManagerMessages.NotifyWhenRegisteredAtJobManager
-import org.apache.flink.runtime.testingUtils.{TestingJobManager, TestingMemoryArchivist, TestingTaskManager, TestingUtils}
-
-import org.apache.flink.runtime.testutils.TestingResourceManager
-
-import scala.concurrent.{Await, Future}
-
-/**
- * A forkable mini cluster is a special case of the mini cluster, used for parallel test execution
- * on build servers. If multiple tests run in parallel, the cluster picks up the fork number and
- * uses it to avoid port conflicts.
- *
- * @param userConfiguration Configuration object with the user provided configuration values
- * @param singleActorSystem true, if all actors (JobManager and TaskManager) shall be run in the
- * same [[ActorSystem]], otherwise false.
- */
-class ForkableFlinkMiniCluster(
- userConfiguration: Configuration,
- singleActorSystem: Boolean)
- extends LocalFlinkMiniCluster(userConfiguration, singleActorSystem) {
-
- def this(userConfiguration: Configuration) = this(userConfiguration, true)
-
- // --------------------------------------------------------------------------
-
- var zookeeperCluster: Option[TestingCluster] = None
-
- override def generateConfiguration(userConfiguration: Configuration): Configuration = {
- val forkNumberString = System.getProperty("forkNumber")
-
- val forkNumber = try {
- Integer.parseInt(forkNumberString)
- }
- catch {
- case e: NumberFormatException => -1
- }
-
- val config = userConfiguration.clone()
-
- if (forkNumber != -1) {
- val jobManagerRPC = 1024 + forkNumber*400
- val taskManagerRPC = 1024 + forkNumber*400 + 100
- val taskManagerData = 1024 + forkNumber*400 + 200
- val resourceManagerRPC = 1024 + forkNumber*400 + 300
-
- config.setInteger(ConfigConstants.JOB_MANAGER_IPC_PORT_KEY, jobManagerRPC)
- config.setInteger(ConfigConstants.TASK_MANAGER_IPC_PORT_KEY, taskManagerRPC)
- config.setInteger(ConfigConstants.TASK_MANAGER_DATA_PORT_KEY, taskManagerData)
- config.setInteger(ConfigConstants.RESOURCE_MANAGER_IPC_PORT_KEY, resourceManagerRPC)
- }
-
- super.generateConfiguration(config)
- }
-
- override def startJobManager(index: Int, actorSystem: ActorSystem): ActorRef = {
- val config = configuration.clone()
-
- val jobManagerName = getJobManagerName(index)
- val archiveName = getArchiveName(index)
-
- val jobManagerPort = config.getInteger(
- ConfigConstants.JOB_MANAGER_IPC_PORT_KEY,
- ConfigConstants.DEFAULT_JOB_MANAGER_IPC_PORT)
-
- if (jobManagerPort > 0) {
- config.setInteger(ConfigConstants.JOB_MANAGER_IPC_PORT_KEY, jobManagerPort + index)
- }
-
- val (jobManager, _) = JobManager.startJobManagerActors(
- config,
- actorSystem,
- Some(jobManagerName),
- Some(archiveName),
- classOf[TestingJobManager],
- classOf[TestingMemoryArchivist])
-
- jobManager
- }
-
- override def startResourceManager(index: Int, system: ActorSystem): ActorRef = {
- val config = configuration.clone()
-
- val resourceManagerName = getResourceManagerName(index)
-
- val resourceManagerPort = config.getInteger(
- ConfigConstants.RESOURCE_MANAGER_IPC_PORT_KEY,
- ConfigConstants.DEFAULT_RESOURCE_MANAGER_IPC_PORT)
-
- if (resourceManagerPort > 0) {
- config.setInteger(ConfigConstants.RESOURCE_MANAGER_IPC_PORT_KEY, resourceManagerPort + index)
- }
-
- val resourceManager = FlinkResourceManager.startResourceManagerActors(
- config,
- system,
- createLeaderRetrievalService(),
- classOf[TestingResourceManager],
- resourceManagerName)
-
- resourceManager
- }
-
- override def startTaskManager(index: Int, system: ActorSystem): ActorRef = {
- val config = configuration.clone()
-
- val rpcPort = config.getInteger(
- ConfigConstants.TASK_MANAGER_IPC_PORT_KEY,
- ConfigConstants.DEFAULT_TASK_MANAGER_IPC_PORT)
-
- val dataPort = config.getInteger(
- ConfigConstants.TASK_MANAGER_DATA_PORT_KEY,
- ConfigConstants.DEFAULT_TASK_MANAGER_DATA_PORT)
-
- if (rpcPort > 0) {
- config.setInteger(ConfigConstants.TASK_MANAGER_IPC_PORT_KEY, rpcPort + index)
- }
- if (dataPort > 0) {
- config.setInteger(ConfigConstants.TASK_MANAGER_DATA_PORT_KEY, dataPort + index)
- }
-
- val localExecution = numTaskManagers == 1
-
- TaskManager.startTaskManagerComponentsAndActor(
- config,
- ResourceID.generate(),
- system,
- hostname,
- Some(TaskManager.TASK_MANAGER_NAME + index),
- Some(createLeaderRetrievalService()),
- localExecution,
- classOf[TestingTaskManager])
- }
-
- def restartLeadingJobManager(): Unit = {
- this.synchronized {
- (jobManagerActorSystems, jobManagerActors) match {
- case (Some(jmActorSystems), Some(jmActors)) =>
- val leader = getLeaderGateway(AkkaUtils.getTimeout(configuration))
- val index = getLeaderIndex(AkkaUtils.getTimeout(configuration))
-
- clearLeader()
-
- val stopped = gracefulStop(leader.actor(), TestingUtils.TESTING_DURATION)
- Await.result(stopped, TestingUtils.TESTING_DURATION)
-
- if(!singleActorSystem) {
- jmActorSystems(index).shutdown()
- jmActorSystems(index).awaitTermination()
- }
-
- val newJobManagerActorSystem = if(!singleActorSystem) {
- startJobManagerActorSystem(index)
- } else {
- jmActorSystems.head
- }
-
- val newJobManagerActor = startJobManager(index, newJobManagerActorSystem)
-
- jobManagerActors = Some(jmActors.patch(index, Seq(newJobManagerActor), 1))
- jobManagerActorSystems = Some(jmActorSystems.patch(
- index,
- Seq(newJobManagerActorSystem),
- 1))
-
- val lrs = createLeaderRetrievalService()
-
- jobManagerLeaderRetrievalService = Some(lrs)
- lrs.start(this)
-
- case _ => throw new Exception("The JobManager of the ForkableFlinkMiniCluster have not " +
- "been started properly.")
- }
- }
- }
-
-
- def restartTaskManager(index: Int): Unit = {
- (taskManagerActorSystems, taskManagerActors) match {
- case (Some(tmActorSystems), Some(tmActors)) =>
- val stopped = gracefulStop(tmActors(index), TestingUtils.TESTING_DURATION)
- Await.result(stopped, TestingUtils.TESTING_DURATION)
-
- if(!singleActorSystem) {
- tmActorSystems(index).shutdown()
- tmActorSystems(index).awaitTermination()
- }
-
- val taskManagerActorSystem = if(!singleActorSystem) {
- startTaskManagerActorSystem(index)
- } else {
- tmActorSystems.head
- }
-
- val taskManagerActor = startTaskManager(index, taskManagerActorSystem)
-
- taskManagerActors = Some(tmActors.patch(index, Seq(taskManagerActor), 1))
- taskManagerActorSystems = Some(tmActorSystems.patch(index, Seq(taskManagerActorSystem), 1))
-
- case _ => throw new Exception("The TaskManager of the ForkableFlinkMiniCluster have not " +
- "been started properly.")
- }
- }
-
- override def start(): Unit = {
- val zookeeperURL = configuration.getString(ConfigConstants.ZOOKEEPER_QUORUM_KEY, "")
-
- zookeeperCluster = if(recoveryMode == RecoveryMode.ZOOKEEPER && zookeeperURL.equals("")) {
- LOG.info("Starting ZooKeeper cluster.")
-
- val testingCluster = new TestingCluster(1)
-
- configuration.setString(ConfigConstants.ZOOKEEPER_QUORUM_KEY, testingCluster.getConnectString)
-
- testingCluster.start()
-
- Some(testingCluster)
- } else {
- None
- }
-
- super.start()
- }
-
- override def stop(): Unit = {
- super.stop()
-
- zookeeperCluster.foreach{
- LOG.info("Stopping ZooKeeper cluster.")
- _.close()
- }
- }
-
- def waitForTaskManagersToBeRegisteredAtJobManager(jobManager: ActorRef): Unit = {
- val futures = taskManagerActors.map {
- _.map {
- tm => (tm ? NotifyWhenRegisteredAtJobManager(jobManager))(timeout)
- }
- }.getOrElse(Seq())
-
- try {
- Await.ready(Future.sequence(futures), timeout)
- } catch {
- case t: TimeoutException =>
- throw new Exception("Timeout while waiting for TaskManagers to register at " +
- s"${jobManager.path}")
- }
-
- }
-}
-
-object ForkableFlinkMiniCluster {
-
- import org.apache.flink.runtime.testingUtils.TestingUtils.DEFAULT_AKKA_ASK_TIMEOUT
-
- def startCluster(
- numSlots: Int,
- numTaskManagers: Int,
- timeout: String = DEFAULT_AKKA_ASK_TIMEOUT)
- : ForkableFlinkMiniCluster = {
-
- val config = new Configuration()
- config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, numSlots)
- config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, numTaskManagers)
- config.setString(ConfigConstants.AKKA_ASK_TIMEOUT, timeout)
-
- val cluster = new ForkableFlinkMiniCluster(config)
-
- cluster.start()
-
- cluster
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-tests/pom.xml
----------------------------------------------------------------------
diff --git a/flink-tests/pom.xml b/flink-tests/pom.xml
index 83f004b..9fd8c3e 100644
--- a/flink-tests/pom.xml
+++ b/flink-tests/pom.xml
@@ -43,7 +43,15 @@ under the License.
<version>${project.version}</version>
<scope>test</scope>
</dependency>
-
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-core</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
+
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-optimizer_2.10</artifactId>
@@ -69,8 +77,8 @@ under the License.
<groupId>org.apache.flink</groupId>
<artifactId>flink-runtime-web_2.10</artifactId>
<version>${project.version}</version>
- <type>test-jar</type>
<scope>test</scope>
+ <type>test-jar</type>
</dependency>
<dependency>
@@ -105,7 +113,6 @@ under the License.
<groupId>org.apache.flink</groupId>
<artifactId>flink-test-utils_2.10</artifactId>
<version>${project.version}</version>
- <type>test-jar</type>
<scope>test</scope>
</dependency>
@@ -115,14 +122,6 @@ under the License.
<version>${project.version}</version>
<scope>test</scope>
</dependency>
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-core</artifactId>
- <version>${project.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
<dependency>
<groupId>org.apache.flink</groupId>
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-tests/src/test/scala/org/apache/flink/api/scala/io/CsvInputFormatTest.scala
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/scala/org/apache/flink/api/scala/io/CsvInputFormatTest.scala b/flink-tests/src/test/scala/org/apache/flink/api/scala/io/CsvInputFormatTest.scala
index 47f99ca..24d86e7 100644
--- a/flink-tests/src/test/scala/org/apache/flink/api/scala/io/CsvInputFormatTest.scala
+++ b/flink-tests/src/test/scala/org/apache/flink/api/scala/io/CsvInputFormatTest.scala
@@ -536,6 +536,4 @@ class CsvInputFormatTest {
assert(expected.sameElements(actual))
}
-
-
}
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-yarn-tests/pom.xml
----------------------------------------------------------------------
diff --git a/flink-yarn-tests/pom.xml b/flink-yarn-tests/pom.xml
index e849211..dcb5334 100644
--- a/flink-yarn-tests/pom.xml
+++ b/flink-yarn-tests/pom.xml
@@ -38,11 +38,13 @@ under the License.
<packaging>jar</packaging>
<dependencies>
+
+ <!-- test dependencies -->
+
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-test-utils_2.10</artifactId>
<version>${project.version}</version>
- <type>test-jar</type>
<scope>test</scope>
</dependency>
@@ -51,18 +53,21 @@ under the License.
<groupId>org.apache.flink</groupId>
<artifactId>flink-streaming-java_2.10</artifactId>
<version>${project.version}</version>
+ <scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-yarn_2.10</artifactId>
<version>${project.version}</version>
+ <scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>${shading-artifact.name}</artifactId>
<version>${project.version}</version>
+ <scope>test</scope>
</dependency>
<dependency>
@@ -80,7 +85,7 @@ under the License.
<dependency>
<groupId>com.typesafe.akka</groupId>
<artifactId>akka-testkit_${scala.binary.version}</artifactId>
- <scope>compile</scope>
+ <scope>test</scope>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 342fc56..c038844 100644
--- a/pom.xml
+++ b/pom.xml
@@ -68,7 +68,7 @@ under the License.
<module>flink-examples</module>
<module>flink-clients</module>
<module>flink-tests</module>
- <module>flink-test-utils</module>
+ <module>flink-test-utils-parent</module>
<module>flink-libraries</module>
<module>flink-scala-shell</module>
<module>flink-quickstart</module>
@@ -105,6 +105,7 @@ under the License.
<curator.version>2.8.0</curator.version>
<jackson.version>2.7.4</jackson.version>
<metrics.version>3.1.0</metrics.version>
+ <junit.version>4.11</junit.version>
</properties>
<dependencies>
@@ -148,7 +149,7 @@ under the License.
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
- <version>4.11</version>
+ <version>${junit.version}</version>
<type>jar</type>
<scope>test</scope>
</dependency>
[15/15] flink git commit: [FLINK-3995] [build] Fix failing tests and
add missing dependencies
Posted by rm...@apache.org.
[FLINK-3995] [build] Fix failing tests and add missing dependencies
This closes #2148
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/8a2c03e1
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/8a2c03e1
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/8a2c03e1
Branch: refs/heads/master
Commit: 8a2c03e1a5fd560f2f204ca1e420a7d7c1e606c7
Parents: b9f42e9
Author: Robert Metzger <rm...@apache.org>
Authored: Mon Jun 20 14:20:59 2016 +0200
Committer: Robert Metzger <rm...@apache.org>
Committed: Tue Jul 5 16:31:34 2016 +0200
----------------------------------------------------------------------
.../apache/flink/util/InstantiationUtilTest.java | 6 +++---
flink-fs-tests/pom.xml | 1 +
flink-libraries/flink-cep/pom.xml | 15 +++++++++++++++
.../distributed/DistributedRowMatrixSuite.scala | 2 +-
.../flink/ml/preprocessing/SplitterITSuite.scala | 3 +--
flink-metrics/flink-metrics-dropwizard/pom.xml | 17 ++++++++++-------
flink-metrics/flink-metrics-statsd/pom.xml | 5 +++--
flink-runtime-web/pom.xml | 6 ++++++
.../executiongraph/ExecutionGraphRestartTest.java | 4 ++--
.../flink-connector-cassandra/pom.xml | 1 +
.../flink-connector-kafka-0.8/pom.xml | 8 ++++++++
.../flink-connector-kafka-0.9/pom.xml | 8 ++++++++
flink-streaming-java/pom.xml | 11 +++++++++++
.../org/apache/flink/test/util/TestBaseUtils.java | 2 +-
14 files changed, 71 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/8a2c03e1/flink-core/src/test/java/org/apache/flink/util/InstantiationUtilTest.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/util/InstantiationUtilTest.java b/flink-core/src/test/java/org/apache/flink/util/InstantiationUtilTest.java
index b369763..cf27ca2 100644
--- a/flink-core/src/test/java/org/apache/flink/util/InstantiationUtilTest.java
+++ b/flink-core/src/test/java/org/apache/flink/util/InstantiationUtilTest.java
@@ -182,12 +182,12 @@ public class InstantiationUtilTest extends TestLogger {
}
}
- private static final class WritableType implements IOReadableWritable {
+ public static final class WritableType implements IOReadableWritable {
private int aInt;
private long aLong;
-
- private WritableType() {
+
+ public WritableType() {
Random rnd = new Random();
this.aInt = rnd.nextInt();
this.aLong = rnd.nextLong();
http://git-wip-us.apache.org/repos/asf/flink/blob/8a2c03e1/flink-fs-tests/pom.xml
----------------------------------------------------------------------
diff --git a/flink-fs-tests/pom.xml b/flink-fs-tests/pom.xml
index d6ffbb5..11ab419 100644
--- a/flink-fs-tests/pom.xml
+++ b/flink-fs-tests/pom.xml
@@ -47,6 +47,7 @@ under the License.
<groupId>org.apache.flink</groupId>
<artifactId>flink-streaming-java_2.10</artifactId>
<version>${project.version}</version>
+ <type>test-jar</type>
<scope>test</scope>
</dependency>
http://git-wip-us.apache.org/repos/asf/flink/blob/8a2c03e1/flink-libraries/flink-cep/pom.xml
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-cep/pom.xml b/flink-libraries/flink-cep/pom.xml
index 6b6d431..32b26d9 100644
--- a/flink-libraries/flink-cep/pom.xml
+++ b/flink-libraries/flink-cep/pom.xml
@@ -52,6 +52,12 @@ under the License.
<scope>provided</scope>
</dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>${guava.version}</version>
+ </dependency>
+
<!-- test dependencies -->
<dependency>
@@ -68,6 +74,15 @@ under the License.
<type>test-jar</type>
<scope>test</scope>
</dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-runtime_2.10</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/flink/blob/8a2c03e1/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/math/distributed/DistributedRowMatrixSuite.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/math/distributed/DistributedRowMatrixSuite.scala b/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/math/distributed/DistributedRowMatrixSuite.scala
index 598b0f3..696b37f 100644
--- a/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/math/distributed/DistributedRowMatrixSuite.scala
+++ b/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/math/distributed/DistributedRowMatrixSuite.scala
@@ -19,7 +19,7 @@
package org.apache.flink.ml.math.distributed
import org.apache.flink.api.scala._
-import org.apache.flink.test.util.FlinkTestBase
+import org.apache.flink.ml.util.FlinkTestBase
import org.scalatest.{FlatSpec, Matchers}
class DistributedRowMatrixSuite extends FlatSpec with Matchers with FlinkTestBase {
http://git-wip-us.apache.org/repos/asf/flink/blob/8a2c03e1/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/preprocessing/SplitterITSuite.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/preprocessing/SplitterITSuite.scala b/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/preprocessing/SplitterITSuite.scala
index 231cb69..4f04438 100644
--- a/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/preprocessing/SplitterITSuite.scala
+++ b/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/preprocessing/SplitterITSuite.scala
@@ -19,9 +19,8 @@ package org.apache.flink.ml.preprocessing
import org.apache.flink.api.scala.ExecutionEnvironment
import org.apache.flink.api.scala._
-import org.apache.flink.test.util.FlinkTestBase
+import org.apache.flink.ml.util.FlinkTestBase
import org.scalatest.{Matchers, FlatSpec}
-import org.apache.flink.ml.math.Vector
import org.apache.flink.api.scala.utils._
http://git-wip-us.apache.org/repos/asf/flink/blob/8a2c03e1/flink-metrics/flink-metrics-dropwizard/pom.xml
----------------------------------------------------------------------
diff --git a/flink-metrics/flink-metrics-dropwizard/pom.xml b/flink-metrics/flink-metrics-dropwizard/pom.xml
index 90dbc00..3d3ecb7 100644
--- a/flink-metrics/flink-metrics-dropwizard/pom.xml
+++ b/flink-metrics/flink-metrics-dropwizard/pom.xml
@@ -41,18 +41,21 @@ under the License.
</dependency>
<dependency>
+ <groupId>io.dropwizard.metrics</groupId>
+ <artifactId>metrics-core</artifactId>
+ <version>${metrics.version}</version>
+ </dependency>
+
+
+ <!-- test dependencies -->
+
+ <dependency>
<groupId>org.apache.flink</groupId>
- <artifactId>flink-core</artifactId>
+ <artifactId>flink-test-utils-junit</artifactId>
<version>${project.version}</version>
- <type>test-jar</type>
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>io.dropwizard.metrics</groupId>
- <artifactId>metrics-core</artifactId>
- <version>${metrics.version}</version>
- </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/flink/blob/8a2c03e1/flink-metrics/flink-metrics-statsd/pom.xml
----------------------------------------------------------------------
diff --git a/flink-metrics/flink-metrics-statsd/pom.xml b/flink-metrics/flink-metrics-statsd/pom.xml
index 8ee0b56..3569e72 100644
--- a/flink-metrics/flink-metrics-statsd/pom.xml
+++ b/flink-metrics/flink-metrics-statsd/pom.xml
@@ -40,11 +40,12 @@ under the License.
<scope>provided</scope>
</dependency>
+ <!-- test dependencies -->
+
<dependency>
<groupId>org.apache.flink</groupId>
- <artifactId>flink-core</artifactId>
+ <artifactId>flink-test-utils-junit</artifactId>
<version>${project.version}</version>
- <type>test-jar</type>
<scope>test</scope>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/flink/blob/8a2c03e1/flink-runtime-web/pom.xml
----------------------------------------------------------------------
diff --git a/flink-runtime-web/pom.xml b/flink-runtime-web/pom.xml
index 8fd0e2f..e0dc3cb 100644
--- a/flink-runtime-web/pom.xml
+++ b/flink-runtime-web/pom.xml
@@ -67,6 +67,12 @@ under the License.
<version>3.18.2-GA</version>
</dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>${guava.version}</version>
+ </dependency>
+
<!-- ===================================================
Testing
=================================================== -->
http://git-wip-us.apache.org/repos/asf/flink/blob/8a2c03e1/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphRestartTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphRestartTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphRestartTest.java
index c12f346..12b866e 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphRestartTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphRestartTest.java
@@ -286,7 +286,7 @@ public class ExecutionGraphRestartTest extends TestLogger {
new JobID(),
"TestJob",
new Configuration(),
- ExecutionConfigTest.getSerializedConfig(),
+ new SerializedValue<>(new ExecutionConfig()),
AkkaUtils.getDefaultTimeout(),
// We want to manually control the restart and delay
new FixedDelayRestartStrategy(Integer.MAX_VALUE, Long.MAX_VALUE));
@@ -703,7 +703,7 @@ public class ExecutionGraphRestartTest extends TestLogger {
new JobID(),
"Test job",
new Configuration(),
- ExecutionConfigTest.getSerializedConfig(),
+ new SerializedValue<>(new ExecutionConfig()),
AkkaUtils.getDefaultTimeout(),
controllableRestartStrategy);
http://git-wip-us.apache.org/repos/asf/flink/blob/8a2c03e1/flink-streaming-connectors/flink-connector-cassandra/pom.xml
----------------------------------------------------------------------
diff --git a/flink-streaming-connectors/flink-connector-cassandra/pom.xml b/flink-streaming-connectors/flink-connector-cassandra/pom.xml
index 5a95659..8d65984 100644
--- a/flink-streaming-connectors/flink-connector-cassandra/pom.xml
+++ b/flink-streaming-connectors/flink-connector-cassandra/pom.xml
@@ -65,6 +65,7 @@ under the License.
<goal>shade</goal>
</goals>
<configuration combine.self="override">
+ <dependencyReducedPomLocation>${project.basedir}/target/dependency-reduced-pom.xml</dependencyReducedPomLocation>
<artifactSet>
<includes>
<include>com.datastax.cassandra:cassandra-driver-core</include>
http://git-wip-us.apache.org/repos/asf/flink/blob/8a2c03e1/flink-streaming-connectors/flink-connector-kafka-0.8/pom.xml
----------------------------------------------------------------------
diff --git a/flink-streaming-connectors/flink-connector-kafka-0.8/pom.xml b/flink-streaming-connectors/flink-connector-kafka-0.8/pom.xml
index 79fa7bd..6f6b96e 100644
--- a/flink-streaming-connectors/flink-connector-kafka-0.8/pom.xml
+++ b/flink-streaming-connectors/flink-connector-kafka-0.8/pom.xml
@@ -137,6 +137,14 @@ under the License.
<dependency>
<groupId>org.apache.flink</groupId>
+ <artifactId>flink-runtime_2.10</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
<artifactId>flink-test-utils_2.10</artifactId>
<version>${project.version}</version>
<scope>test</scope>
http://git-wip-us.apache.org/repos/asf/flink/blob/8a2c03e1/flink-streaming-connectors/flink-connector-kafka-0.9/pom.xml
----------------------------------------------------------------------
diff --git a/flink-streaming-connectors/flink-connector-kafka-0.9/pom.xml b/flink-streaming-connectors/flink-connector-kafka-0.9/pom.xml
index 8ad909a..fe4db7c 100644
--- a/flink-streaming-connectors/flink-connector-kafka-0.9/pom.xml
+++ b/flink-streaming-connectors/flink-connector-kafka-0.9/pom.xml
@@ -119,6 +119,14 @@ under the License.
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-runtime_2.10</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/flink/blob/8a2c03e1/flink-streaming-java/pom.xml
----------------------------------------------------------------------
diff --git a/flink-streaming-java/pom.xml b/flink-streaming-java/pom.xml
index 790161d..b309d97 100644
--- a/flink-streaming-java/pom.xml
+++ b/flink-streaming-java/pom.xml
@@ -104,6 +104,17 @@ under the License.
<reuseForks>false</reuseForks>
</configuration>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
http://git-wip-us.apache.org/repos/asf/flink/blob/8a2c03e1/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/TestBaseUtils.java
----------------------------------------------------------------------
diff --git a/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/TestBaseUtils.java b/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/TestBaseUtils.java
index 6acc653..c51e666 100644
--- a/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/TestBaseUtils.java
+++ b/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/TestBaseUtils.java
@@ -87,7 +87,7 @@ public class TestBaseUtils extends TestLogger {
protected static final String DEFAULT_AKKA_STARTUP_TIMEOUT = "60 s";
- protected static FiniteDuration DEFAULT_TIMEOUT = new FiniteDuration(DEFAULT_AKKA_ASK_TIMEOUT, TimeUnit.SECONDS);
+ public static FiniteDuration DEFAULT_TIMEOUT = new FiniteDuration(DEFAULT_AKKA_ASK_TIMEOUT, TimeUnit.SECONDS);
// ------------------------------------------------------------------------
[12/15] flink git commit: [FLINK-3995] [build] flink-test-utils also
contains the streaming test utilities.
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/b9f42e91/flink-streaming-java/src/test/java/org/apache/flink/streaming/timestamp/TimestampITCase.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/timestamp/TimestampITCase.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/timestamp/TimestampITCase.java
deleted file mode 100644
index 6c1d0e6..0000000
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/timestamp/TimestampITCase.java
+++ /dev/null
@@ -1,875 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.timestamp;
-
-import org.apache.flink.api.common.JobID;
-import org.apache.flink.api.common.functions.MapFunction;
-import org.apache.flink.api.common.functions.ReduceFunction;
-import org.apache.flink.api.common.functions.StoppableFunction;
-import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.configuration.ConfigConstants;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.core.testutils.MultiShotLatch;
-import org.apache.flink.streaming.api.TimeCharacteristic;
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.api.functions.timestamps.AscendingTimestampExtractor;
-import org.apache.flink.streaming.api.functions.AssignerWithPeriodicWatermarks;
-import org.apache.flink.streaming.api.functions.AssignerWithPunctuatedWatermarks;
-import org.apache.flink.streaming.api.functions.co.CoMapFunction;
-import org.apache.flink.streaming.api.functions.source.SourceFunction;
-import org.apache.flink.streaming.api.operators.AbstractStreamOperator;
-import org.apache.flink.streaming.api.operators.ChainingStrategy;
-import org.apache.flink.streaming.api.operators.OneInputStreamOperator;
-import org.apache.flink.streaming.api.watermark.Watermark;
-import org.apache.flink.streaming.api.windowing.assigners.TumblingEventTimeWindows;
-import org.apache.flink.streaming.api.windowing.time.Time;
-import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
-import org.apache.flink.streaming.util.NoOpSink;
-import org.apache.flink.test.util.ForkableFlinkMiniCluster;
-
-import org.apache.flink.util.TestLogger;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotEquals;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-/**
- * Tests for timestamps, watermarks, and event-time sources.
- */
-@SuppressWarnings("serial")
-public class TimestampITCase extends TestLogger {
-
- private static final int NUM_TASK_MANAGERS = 2;
- private static final int NUM_TASK_SLOTS = 3;
- private static final int PARALLELISM = NUM_TASK_MANAGERS * NUM_TASK_SLOTS;
-
- // this is used in some tests to synchronize
- static MultiShotLatch latch;
-
-
- private static ForkableFlinkMiniCluster cluster;
-
- @Before
- public void setupLatch() {
- // ensure that we get a fresh latch for each test
- latch = new MultiShotLatch();
- }
-
-
- @BeforeClass
- public static void startCluster() {
- try {
- Configuration config = new Configuration();
- config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, NUM_TASK_MANAGERS);
- config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, NUM_TASK_SLOTS);
- config.setInteger(ConfigConstants.TASK_MANAGER_MEMORY_SIZE_KEY, 12);
-
- cluster = new ForkableFlinkMiniCluster(config, false);
-
- cluster.start();
- }
- catch (Exception e) {
- e.printStackTrace();
- fail("Failed to start test cluster: " + e.getMessage());
- }
- }
-
- @AfterClass
- public static void shutdownCluster() {
- try {
- cluster.shutdown();
- cluster = null;
- }
- catch (Exception e) {
- e.printStackTrace();
- fail("Failed to stop test cluster: " + e.getMessage());
- }
- }
-
- /**
- * These check whether custom timestamp emission works at sources and also whether timestamps
- * arrive at operators throughout a topology.
- *
- * <p>
- * This also checks whether watermarks keep propagating if a source closes early.
- *
- * <p>
- * This only uses map to test the workings of watermarks in a complete, running topology. All
- * tasks and stream operators have dedicated tests that test the watermark propagation
- * behaviour.
- */
- @Test
- public void testWatermarkPropagation() throws Exception {
- final int NUM_WATERMARKS = 10;
-
- long initialTime = 0L;
-
- StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment(
- "localhost", cluster.getLeaderRPCPort());
-
- env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
- env.setParallelism(PARALLELISM);
- env.getConfig().disableSysoutLogging();
-
- DataStream<Integer> source1 = env.addSource(new MyTimestampSource(initialTime, NUM_WATERMARKS));
- DataStream<Integer> source2 = env.addSource(new MyTimestampSource(initialTime, NUM_WATERMARKS / 2));
-
- source1.union(source2)
- .map(new IdentityMap())
- .connect(source2).map(new IdentityCoMap())
- .transform("Custom Operator", BasicTypeInfo.INT_TYPE_INFO, new CustomOperator(true))
- .addSink(new NoOpSink<Integer>());
-
- env.execute();
-
- // verify that all the watermarks arrived at the final custom operator
- for (int i = 0; i < PARALLELISM; i++) {
- // we are only guaranteed to see NUM_WATERMARKS / 2 watermarks because the
- // other source stops emitting after that
- for (int j = 0; j < NUM_WATERMARKS / 2; j++) {
- if (!CustomOperator.finalWatermarks[i].get(j).equals(new Watermark(initialTime + j))) {
- System.err.println("All Watermarks: ");
- for (int k = 0; k <= NUM_WATERMARKS / 2; k++) {
- System.err.println(CustomOperator.finalWatermarks[i].get(k));
- }
-
- fail("Wrong watermark.");
- }
- }
-
- assertEquals(Watermark.MAX_WATERMARK,
- CustomOperator.finalWatermarks[i].get(CustomOperator.finalWatermarks[i].size()-1));
- }
- }
-
- @Test
- public void testWatermarkPropagationNoFinalWatermarkOnStop() throws Exception {
-
- // for this test to work, we need to be sure that no other jobs are being executed
- while (!cluster.getCurrentlyRunningJobsJava().isEmpty()) {
- Thread.sleep(100);
- }
-
- final int NUM_WATERMARKS = 10;
-
- long initialTime = 0L;
-
- StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment(
- "localhost", cluster.getLeaderRPCPort());
-
- env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
- env.setParallelism(PARALLELISM);
- env.getConfig().disableSysoutLogging();
-
- DataStream<Integer> source1 = env.addSource(new MyTimestampSourceInfinite(initialTime, NUM_WATERMARKS));
- DataStream<Integer> source2 = env.addSource(new MyTimestampSourceInfinite(initialTime, NUM_WATERMARKS / 2));
-
- source1.union(source2)
- .map(new IdentityMap())
- .connect(source2).map(new IdentityCoMap())
- .transform("Custom Operator", BasicTypeInfo.INT_TYPE_INFO, new CustomOperator(true))
- .addSink(new NoOpSink<Integer>());
-
- new Thread("stopper") {
- @Override
- public void run() {
- try {
- // try until we get the running jobs
- List<JobID> running;
- while ((running = cluster.getCurrentlyRunningJobsJava()).isEmpty()) {
- Thread.sleep(50);
- }
-
- JobID id = running.get(0);
-
- // send stop until the job is stopped
- do {
- cluster.stopJob(id);
- Thread.sleep(50);
- } while (!cluster.getCurrentlyRunningJobsJava().isEmpty());
- }
- catch (Throwable t) {
- t.printStackTrace();
- }
- }
- }.start();
-
- env.execute();
-
- // verify that all the watermarks arrived at the final custom operator
- for (List<Watermark> subtaskWatermarks : CustomOperator.finalWatermarks) {
-
- // we are only guaranteed to see NUM_WATERMARKS / 2 watermarks because the
- // other source stops emitting after that
- for (int j = 0; j < subtaskWatermarks.size(); j++) {
- if (subtaskWatermarks.get(j).getTimestamp() != initialTime + j) {
- System.err.println("All Watermarks: ");
- for (int k = 0; k <= NUM_WATERMARKS / 2; k++) {
- System.err.println(subtaskWatermarks.get(k));
- }
-
- fail("Wrong watermark.");
- }
- }
-
- // if there are watermarks, the final one must not be the MAX watermark
- if (subtaskWatermarks.size() > 0) {
- assertNotEquals(Watermark.MAX_WATERMARK,
- subtaskWatermarks.get(subtaskWatermarks.size()-1));
- }
- }
- }
-
- /**
- * These check whether timestamps are properly assigned at the sources and handled in
- * network transmission and between chained operators when timestamps are enabled.
- */
- @Test
- public void testTimestampHandling() throws Exception {
- final int NUM_ELEMENTS = 10;
-
-
- StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment(
- "localhost", cluster.getLeaderRPCPort());
-
- env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
- env.setParallelism(PARALLELISM);
- env.getConfig().disableSysoutLogging();
-
- DataStream<Integer> source1 = env.addSource(new MyTimestampSource(0L, NUM_ELEMENTS));
- DataStream<Integer> source2 = env.addSource(new MyTimestampSource(0L, NUM_ELEMENTS));
-
- source1
- .map(new IdentityMap())
- .connect(source2).map(new IdentityCoMap())
- .transform("Custom Operator", BasicTypeInfo.INT_TYPE_INFO, new TimestampCheckingOperator())
- .addSink(new NoOpSink<Integer>());
-
-
- env.execute();
- }
-
- /**
- * These check whether timestamps are properly ignored when they are disabled.
- */
- @Test
- public void testDisabledTimestamps() throws Exception {
- final int NUM_ELEMENTS = 10;
-
- StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment(
- "localhost", cluster.getLeaderRPCPort());
-
- env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime);
- env.setParallelism(PARALLELISM);
- env.getConfig().disableSysoutLogging();
-
- DataStream<Integer> source1 = env.addSource(new MyNonWatermarkingSource(NUM_ELEMENTS));
- DataStream<Integer> source2 = env.addSource(new MyNonWatermarkingSource(NUM_ELEMENTS));
-
- source1
- .map(new IdentityMap())
- .connect(source2).map(new IdentityCoMap())
- .transform("Custom Operator", BasicTypeInfo.INT_TYPE_INFO, new DisabledTimestampCheckingOperator())
- .addSink(new NoOpSink<Integer>());
-
- env.execute();
- }
-
- /**
- * This tests whether timestamps are properly extracted in the timestamp
- * extractor and whether watermarks are also correctly forwared from this with the auto watermark
- * interval.
- */
- @Test
- public void testTimestampExtractorWithAutoInterval() throws Exception {
- final int NUM_ELEMENTS = 10;
-
- StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", cluster.getLeaderRPCPort());
-
- env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
- env.getConfig().setAutoWatermarkInterval(10);
- env.setParallelism(1);
- env.getConfig().disableSysoutLogging();
-
-
- DataStream<Integer> source1 = env.addSource(new SourceFunction<Integer>() {
- @Override
- public void run(SourceContext<Integer> ctx) throws Exception {
- int index = 1;
- while (index <= NUM_ELEMENTS) {
- ctx.collect(index);
- latch.await();
- index++;
- }
- }
-
- @Override
- public void cancel() {}
- });
-
- DataStream<Integer> extractOp = source1.assignTimestampsAndWatermarks(
- new AscendingTimestampExtractor<Integer>() {
- @Override
- public long extractAscendingTimestamp(Integer element) {
- return element;
- }
- });
-
- extractOp
- .transform("Watermark Check", BasicTypeInfo.INT_TYPE_INFO, new CustomOperator(true))
- .transform("Timestamp Check",
- BasicTypeInfo.INT_TYPE_INFO,
- new TimestampCheckingOperator());
-
- // verify that extractor picks up source parallelism
- Assert.assertEquals(extractOp.getTransformation().getParallelism(), source1.getTransformation().getParallelism());
-
- env.execute();
-
- // verify that we get NUM_ELEMENTS watermarks
- for (int j = 0; j < NUM_ELEMENTS; j++) {
- if (!CustomOperator.finalWatermarks[0].get(j).equals(new Watermark(j))) {
- long wm = CustomOperator.finalWatermarks[0].get(j).getTimestamp();
- Assert.fail("Wrong watermark. Expected: " + j + " Found: " + wm + " All: " + CustomOperator.finalWatermarks[0]);
- }
- }
-
- // the input is finite, so it should have a MAX Watermark
- assertEquals(Watermark.MAX_WATERMARK,
- CustomOperator.finalWatermarks[0].get(CustomOperator.finalWatermarks[0].size() - 1));
- }
-
- /**
- * This thests whether timestamps are properly extracted in the timestamp
- * extractor and whether watermark are correctly forwarded from the custom watermark emit
- * function.
- */
- @Test
- public void testTimestampExtractorWithCustomWatermarkEmit() throws Exception {
- final int NUM_ELEMENTS = 10;
-
- StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", cluster.getLeaderRPCPort());
-
- env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
- env.getConfig().setAutoWatermarkInterval(10);
- env.setParallelism(1);
- env.getConfig().disableSysoutLogging();
-
-
- DataStream<Integer> source1 = env.addSource(new SourceFunction<Integer>() {
- @Override
- public void run(SourceContext<Integer> ctx) throws Exception {
- int index = 1;
- while (index <= NUM_ELEMENTS) {
- ctx.collect(index);
- latch.await();
- index++;
- }
- }
-
- @Override
- public void cancel() {}
- });
-
- source1
- .assignTimestampsAndWatermarks(new AssignerWithPunctuatedWatermarks<Integer>() {
-
- @Override
- public long extractTimestamp(Integer element, long currentTimestamp) {
- return element;
- }
-
- @Override
- public Watermark checkAndGetNextWatermark(Integer element, long extractedTimestamp) {
- return new Watermark(extractedTimestamp - 1);
- }
- })
- .transform("Watermark Check", BasicTypeInfo.INT_TYPE_INFO, new CustomOperator(true))
- .transform("Timestamp Check", BasicTypeInfo.INT_TYPE_INFO, new TimestampCheckingOperator());
-
-
- env.execute();
-
- // verify that we get NUM_ELEMENTS watermarks
- for (int j = 0; j < NUM_ELEMENTS; j++) {
- if (!CustomOperator.finalWatermarks[0].get(j).equals(new Watermark(j))) {
- Assert.fail("Wrong watermark.");
- }
- }
-
- // the input is finite, so it should have a MAX Watermark
- assertEquals(Watermark.MAX_WATERMARK,
- CustomOperator.finalWatermarks[0].get(CustomOperator.finalWatermarks[0].size() - 1));
- }
-
- /**
- * This test verifies that the timestamp extractor does not emit decreasing watermarks even
- *
- */
- @Test
- public void testTimestampExtractorWithDecreasingCustomWatermarkEmit() throws Exception {
- final int NUM_ELEMENTS = 10;
-
- StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", cluster.getLeaderRPCPort());
-
- env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
- env.getConfig().setAutoWatermarkInterval(1);
- env.setParallelism(1);
- env.getConfig().disableSysoutLogging();
-
-
- DataStream<Integer> source1 = env.addSource(new SourceFunction<Integer>() {
- @Override
- public void run(SourceContext<Integer> ctx) throws Exception {
- int index = 1;
- while (index <= NUM_ELEMENTS) {
- ctx.collect(index);
- Thread.sleep(100);
- ctx.collect(index - 1);
- latch.await();
- index++;
- }
- }
-
- @Override
- public void cancel() {}
- });
-
- source1
- .assignTimestampsAndWatermarks(new AssignerWithPunctuatedWatermarks<Integer>() {
-
- @Override
- public long extractTimestamp(Integer element, long previousTimestamp) {
- return element;
- }
-
- @Override
- public Watermark checkAndGetNextWatermark(Integer element, long extractedTimestamp) {
- return new Watermark(extractedTimestamp - 1);
- }
- })
- .transform("Watermark Check", BasicTypeInfo.INT_TYPE_INFO, new CustomOperator(true))
- .transform("Timestamp Check", BasicTypeInfo.INT_TYPE_INFO, new TimestampCheckingOperator());
-
-
- env.execute();
-
- // verify that we get NUM_ELEMENTS watermarks
- for (int j = 0; j < NUM_ELEMENTS; j++) {
- if (!CustomOperator.finalWatermarks[0].get(j).equals(new Watermark(j))) {
- Assert.fail("Wrong watermark.");
- }
- }
- // the input is finite, so it should have a MAX Watermark
- assertEquals(Watermark.MAX_WATERMARK,
- CustomOperator.finalWatermarks[0].get(CustomOperator.finalWatermarks[0].size() - 1));
- }
-
- /**
- * This test verifies that the timestamp extractor forwards Long.MAX_VALUE watermarks.
- */
- @Test
- public void testTimestampExtractorWithLongMaxWatermarkFromSource() throws Exception {
- final int NUM_ELEMENTS = 10;
-
- StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", cluster.getLeaderRPCPort());
-
- env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
- env.getConfig().setAutoWatermarkInterval(1);
- env.setParallelism(2);
- env.getConfig().disableSysoutLogging();
-
-
- DataStream<Integer> source1 = env.addSource(new SourceFunction<Integer>() {
- @Override
- public void run(SourceContext<Integer> ctx) throws Exception {
- int index = 1;
- while (index <= NUM_ELEMENTS) {
- ctx.collectWithTimestamp(index, index);
- ctx.collectWithTimestamp(index - 1, index - 1);
- index++;
- ctx.emitWatermark(new Watermark(index-2));
- }
-
- // emit the final Long.MAX_VALUE watermark, do it twice and verify that
- // we only see one in the result
- ctx.emitWatermark(new Watermark(Long.MAX_VALUE));
- ctx.emitWatermark(new Watermark(Long.MAX_VALUE));
- }
-
- @Override
- public void cancel() {}
- });
-
- source1
- .assignTimestampsAndWatermarks(new AssignerWithPunctuatedWatermarks<Integer>() {
-
- @Override
- public long extractTimestamp(Integer element, long currentTimestamp) {
- return element;
- }
-
- @Override
- public Watermark checkAndGetNextWatermark(Integer element, long extractedTimestamp) {
- return null;
- }
- })
- .transform("Watermark Check", BasicTypeInfo.INT_TYPE_INFO, new CustomOperator(true));
-
-
- env.execute();
-
- Assert.assertTrue(CustomOperator.finalWatermarks[0].size() == 1);
- Assert.assertTrue(CustomOperator.finalWatermarks[0].get(0).getTimestamp() == Long.MAX_VALUE);
- }
-
- /**
- * This test verifies that the timestamp extractor forwards Long.MAX_VALUE watermarks.
- *
- * Same test as before, but using a different timestamp extractor
- */
- @Test
- public void testTimestampExtractorWithLongMaxWatermarkFromSource2() throws Exception {
- final int NUM_ELEMENTS = 10;
-
- StreamExecutionEnvironment env = StreamExecutionEnvironment
- .createRemoteEnvironment("localhost", cluster.getLeaderRPCPort());
-
- env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
- env.getConfig().setAutoWatermarkInterval(10);
- env.setParallelism(2);
- env.getConfig().disableSysoutLogging();
-
- DataStream<Integer> source1 = env.addSource(new SourceFunction<Integer>() {
- @Override
- public void run(SourceContext<Integer> ctx) throws Exception {
- int index = 1;
- while (index <= NUM_ELEMENTS) {
- ctx.collectWithTimestamp(index, index);
- ctx.collectWithTimestamp(index - 1, index - 1);
- index++;
- ctx.emitWatermark(new Watermark(index-2));
- }
-
- // emit the final Long.MAX_VALUE watermark, do it twice and verify that
- // we only see one in the result
- ctx.emitWatermark(new Watermark(Long.MAX_VALUE));
- ctx.emitWatermark(new Watermark(Long.MAX_VALUE));
- }
-
- @Override
- public void cancel() {}
- });
-
- source1
- .assignTimestampsAndWatermarks(new AssignerWithPeriodicWatermarks<Integer>() {
-
- @Override
- public long extractTimestamp(Integer element, long currentTimestamp) {
- return element;
- }
-
- @Override
- public Watermark getCurrentWatermark() {
- return null;
- }
- })
- .transform("Watermark Check", BasicTypeInfo.INT_TYPE_INFO, new CustomOperator(true));
-
- env.execute();
-
- Assert.assertTrue(CustomOperator.finalWatermarks[0].size() == 1);
- Assert.assertTrue(CustomOperator.finalWatermarks[0].get(0).getTimestamp() == Long.MAX_VALUE);
- }
-
- /**
- * This verifies that an event time source works when setting stream time characteristic to
- * processing time. In this case, the watermarks should just be swallowed.
- */
- @Test
- public void testEventTimeSourceWithProcessingTime() throws Exception {
- StreamExecutionEnvironment env =
- StreamExecutionEnvironment.createRemoteEnvironment("localhost", cluster.getLeaderRPCPort());
-
- env.setParallelism(2);
- env.getConfig().disableSysoutLogging();
- env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime);
-
- DataStream<Integer> source1 = env.addSource(new MyTimestampSource(0, 10));
-
- source1
- .map(new IdentityMap())
- .transform("Watermark Check", BasicTypeInfo.INT_TYPE_INFO, new CustomOperator(false));
-
- env.execute();
-
- // verify that we don't get any watermarks, the source is used as watermark source in
- // other tests, so it normally emits watermarks
- Assert.assertTrue(CustomOperator.finalWatermarks[0].size() == 0);
- }
-
- @Test
- public void testErrorOnEventTimeOverProcessingTime() {
- StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment(
- "localhost", cluster.getLeaderRPCPort());
-
- env.setParallelism(2);
- env.getConfig().disableSysoutLogging();
- env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime);
-
- DataStream<Tuple2<String, Integer>> source1 =
- env.fromElements(new Tuple2<>("a", 1), new Tuple2<>("b", 2));
-
- source1
- .keyBy(0)
- .window(TumblingEventTimeWindows.of(Time.seconds(5)))
- .reduce(new ReduceFunction<Tuple2<String, Integer>>() {
- @Override
- public Tuple2<String, Integer> reduce(Tuple2<String, Integer> value1, Tuple2<String, Integer> value2) {
- return value1;
- }
- })
- .print();
-
- try {
- env.execute();
- fail("this should fail with an exception");
- } catch (Exception e) {
- // expected
- }
- }
-
- @Test
- public void testErrorOnEventTimeWithoutTimestamps() {
- StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment(
- "localhost", cluster.getLeaderRPCPort());
-
- env.setParallelism(2);
- env.getConfig().disableSysoutLogging();
- env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
-
- DataStream<Tuple2<String, Integer>> source1 =
- env.fromElements(new Tuple2<>("a", 1), new Tuple2<>("b", 2));
-
- source1
- .keyBy(0)
- .window(TumblingEventTimeWindows.of(Time.seconds(5)))
- .reduce(new ReduceFunction<Tuple2<String, Integer>>() {
- @Override
- public Tuple2<String, Integer> reduce(Tuple2<String, Integer> value1, Tuple2<String, Integer> value2) {
- return value1;
- }
- })
- .print();
-
- try {
- env.execute();
- fail("this should fail with an exception");
- } catch (Exception e) {
- // expected
- }
- }
-
- // ------------------------------------------------------------------------
- // Custom Operators and Functions
- // ------------------------------------------------------------------------
-
- @SuppressWarnings("unchecked")
- public static class CustomOperator extends AbstractStreamOperator<Integer> implements OneInputStreamOperator<Integer, Integer> {
-
- List<Watermark> watermarks;
- public static List<Watermark>[] finalWatermarks = new List[PARALLELISM];
- private final boolean timestampsEnabled;
-
- public CustomOperator(boolean timestampsEnabled) {
- setChainingStrategy(ChainingStrategy.ALWAYS);
- this.timestampsEnabled = timestampsEnabled;
- }
-
- @Override
- public void processElement(StreamRecord<Integer> element) throws Exception {
- if (timestampsEnabled) {
- if (element.getTimestamp() != element.getValue()) {
- Assert.fail("Timestamps are not properly handled.");
- }
- }
- output.collect(element);
- }
-
- @Override
- public void processWatermark(Watermark mark) throws Exception {
- for (Watermark previousMark: watermarks) {
- assertTrue(previousMark.getTimestamp() < mark.getTimestamp());
- }
- watermarks.add(mark);
- latch.trigger();
- output.emitWatermark(mark);
- }
-
- @Override
- public void open() throws Exception {
- super.open();
- watermarks = new ArrayList<>();
- }
-
- @Override
- public void close() throws Exception {
- super.close();
- finalWatermarks[getRuntimeContext().getIndexOfThisSubtask()] = watermarks;
- }
- }
-
- public static class TimestampCheckingOperator extends AbstractStreamOperator<Integer> implements OneInputStreamOperator<Integer, Integer> {
-
- public TimestampCheckingOperator() {
- setChainingStrategy(ChainingStrategy.ALWAYS);
- }
-
- @Override
- public void processElement(StreamRecord<Integer> element) throws Exception {
- if (element.getTimestamp() != element.getValue()) {
- Assert.fail("Timestamps are not properly handled.");
- }
- output.collect(element);
- }
-
- @Override
- public void processWatermark(Watermark mark) throws Exception {}
- }
-
- public static class DisabledTimestampCheckingOperator extends AbstractStreamOperator<Integer> implements OneInputStreamOperator<Integer, Integer> {
-
- @Override
- public void processElement(StreamRecord<Integer> element) throws Exception {
- if (element.hasTimestamp()) {
- Assert.fail("Timestamps are not properly handled.");
- }
- output.collect(element);
- }
-
- @Override
- public void processWatermark(Watermark mark) throws Exception {}
- }
-
- public static class IdentityCoMap implements CoMapFunction<Integer, Integer, Integer> {
- @Override
- public Integer map1(Integer value) throws Exception {
- return value;
- }
-
- @Override
- public Integer map2(Integer value) throws Exception {
- return value;
- }
- }
-
- public static class IdentityMap implements MapFunction<Integer, Integer> {
- @Override
- public Integer map(Integer value) throws Exception {
- return value;
- }
- }
-
- public static class MyTimestampSource implements SourceFunction<Integer> {
-
- private final long initialTime;
- private final int numWatermarks;
-
- public MyTimestampSource(long initialTime, int numWatermarks) {
- this.initialTime = initialTime;
- this.numWatermarks = numWatermarks;
- }
-
- @Override
- public void run(SourceContext<Integer> ctx) throws Exception {
- for (int i = 0; i < numWatermarks; i++) {
- ctx.collectWithTimestamp(i, initialTime + i);
- ctx.emitWatermark(new Watermark(initialTime + i));
- }
- }
-
- @Override
- public void cancel() {}
- }
-
- public static class MyTimestampSourceInfinite implements SourceFunction<Integer>, StoppableFunction {
-
- private final long initialTime;
- private final int numWatermarks;
-
- private volatile boolean running = true;
-
- public MyTimestampSourceInfinite(long initialTime, int numWatermarks) {
- this.initialTime = initialTime;
- this.numWatermarks = numWatermarks;
- }
-
- @Override
- public void run(SourceContext<Integer> ctx) throws Exception {
- for (int i = 0; i < numWatermarks; i++) {
- ctx.collectWithTimestamp(i, initialTime + i);
- ctx.emitWatermark(new Watermark(initialTime + i));
- }
-
- while (running) {
- Thread.sleep(20);
- }
- }
-
- @Override
- public void cancel() {
- running = false;
- }
-
- @Override
- public void stop() {
- running = false;
- }
- }
-
- public static class MyNonWatermarkingSource implements SourceFunction<Integer> {
-
- int numWatermarks;
-
- public MyNonWatermarkingSource(int numWatermarks) {
- this.numWatermarks = numWatermarks;
- }
-
- @Override
- public void run(SourceContext<Integer> ctx) throws Exception {
- for (int i = 0; i < numWatermarks; i++) {
- ctx.collect(i);
- }
- }
-
- @Override
- public void cancel() {}
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/b9f42e91/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/NoOpSink.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/NoOpSink.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/NoOpSink.java
deleted file mode 100644
index d398121..0000000
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/NoOpSink.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.streaming.util;
-
-import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
-
-public final class NoOpSink<T> extends RichSinkFunction<T> {
- public void invoke(T tuple) {
-
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/b9f42e91/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/ReceiveCheckNoOpSink.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/ReceiveCheckNoOpSink.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/ReceiveCheckNoOpSink.java
deleted file mode 100644
index a46ff55..0000000
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/ReceiveCheckNoOpSink.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.streaming.util;
-
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import static org.junit.Assert.assertTrue;
-
-public final class ReceiveCheckNoOpSink<T> extends RichSinkFunction<T> {
- private List<T> received;
-
- public void invoke(T tuple) {
- received.add(tuple);
- }
-
- public void open(Configuration conf) {
- received = new ArrayList<T>();
- }
-
- public void close() {
- assertTrue(received.size() > 0);
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/b9f42e91/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/SocketOutputTestBase.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/SocketOutputTestBase.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/SocketOutputTestBase.java
deleted file mode 100644
index 7d6a6d0..0000000
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/SocketOutputTestBase.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.util;
-
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.api.java.typeutils.TypeExtractor;
-import org.apache.flink.streaming.util.serialization.DeserializationSchema;
-import org.apache.flink.streaming.util.serialization.SerializationSchema;
-import org.apache.flink.streaming.util.serialization.SimpleStringSchema;
-import org.apache.flink.test.testdata.WordCountData;
-import org.apache.flink.util.NetUtils;
-
-import org.junit.Assert;
-
-import java.io.BufferedReader;
-import java.io.InputStreamReader;
-import java.net.ServerSocket;
-import java.net.Socket;
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.Set;
-
-/**
- * Test base for streaming programs relying on an open server socket to write to.
- */
-public abstract class SocketOutputTestBase extends StreamingProgramTestBase {
-
- protected static final String HOST = "localhost";
- protected static Integer port;
- protected Set<String> dataReadFromSocket = new HashSet<String>();
-
- @Override
- protected void preSubmit() throws Exception {
- port = NetUtils.getAvailablePort();
- temporarySocket = createLocalSocket(port);
- }
-
- @Override
- protected void postSubmit() throws Exception {
- Set<String> expectedData = new HashSet<String>(Arrays.asList(WordCountData.STREAMING_COUNTS_AS_TUPLES.split("\n")));
- Assert.assertEquals(expectedData, dataReadFromSocket);
- temporarySocket.close();
- }
-
- protected ServerSocket temporarySocket;
-
- public ServerSocket createLocalSocket(int port) throws Exception {
- ServerSocket serverSocket = new ServerSocket(port);
- ServerThread st = new ServerThread(serverSocket);
- st.start();
- return serverSocket;
- }
-
- protected class ServerThread extends Thread {
-
- private ServerSocket serverSocket;
- private Thread t;
-
- public ServerThread(ServerSocket serverSocket) {
- this.serverSocket = serverSocket;
- t = new Thread(this);
- }
-
- public void waitForAccept() throws Exception {
- Socket socket = serverSocket.accept();
- BufferedReader in = new BufferedReader(new InputStreamReader(socket.getInputStream()));
- DeserializationSchema<String> schema = new SimpleStringSchema();
- String rawData = in.readLine();
- while (rawData != null){
- String string = schema.deserialize(rawData.getBytes());
- dataReadFromSocket.add(string);
- rawData = in.readLine();
- }
- socket.close();
- }
-
- public void run() {
- try {
- waitForAccept();
- } catch (Exception e) {
- Assert.fail();
- throw new RuntimeException(e);
- }
- }
-
- @Override
- public void start() {
- t.start();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/b9f42e91/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/StreamingMultipleProgramsTestBase.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/StreamingMultipleProgramsTestBase.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/StreamingMultipleProgramsTestBase.java
deleted file mode 100644
index 8cdedd5..0000000
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/StreamingMultipleProgramsTestBase.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.util;
-
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.test.util.AbstractTestBase;
-import org.apache.flink.test.util.ForkableFlinkMiniCluster;
-import org.apache.flink.test.util.TestBaseUtils;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-
-/**
- * Base class for streaming unit tests that run multiple tests and want to reuse the same
- * Flink cluster. This saves a significant amount of time, since the startup and
- * shutdown of the Flink clusters (including actor systems, etc) usually dominates
- * the execution of the actual tests.
- *
- * To write a unit test against this test base, simply extend it and add
- * one or more regular test methods and retrieve the StreamExecutionEnvironment from
- * the context:
- *
- * <pre>
- * {@literal @}Test
- * public void someTest() {
- * StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
- * // test code
- * env.execute();
- * }
- *
- * {@literal @}Test
- * public void anotherTest() {
- * StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
- * // test code
- * env.execute();
- * }
- *
- * </pre>
- */
-public class StreamingMultipleProgramsTestBase extends AbstractTestBase {
-
- // ------------------------------------------------------------------------
- // The mini cluster that is shared across tests
- // ------------------------------------------------------------------------
-
- protected static final int DEFAULT_PARALLELISM = 4;
-
- protected static ForkableFlinkMiniCluster cluster;
-
- public StreamingMultipleProgramsTestBase() {
- super(new Configuration());
- }
-
- // ------------------------------------------------------------------------
- // Cluster setup & teardown
- // ------------------------------------------------------------------------
-
- @BeforeClass
- public static void setup() throws Exception {
- cluster = TestBaseUtils.startCluster(1, DEFAULT_PARALLELISM, false, false, true);
- TestStreamEnvironment.setAsContext(cluster, DEFAULT_PARALLELISM);
- }
-
- @AfterClass
- public static void teardown() throws Exception {
- TestStreamEnvironment.unsetAsContext();
- stopCluster(cluster, TestBaseUtils.DEFAULT_TIMEOUT);
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/b9f42e91/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/StreamingProgramTestBase.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/StreamingProgramTestBase.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/StreamingProgramTestBase.java
deleted file mode 100644
index 50ed1cf..0000000
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/StreamingProgramTestBase.java
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.util;
-
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.test.util.AbstractTestBase;
-
-import org.junit.Test;
-
-import static org.junit.Assert.fail;
-
-public abstract class StreamingProgramTestBase extends AbstractTestBase {
-
- protected static final int DEFAULT_PARALLELISM = 4;
-
- private int parallelism;
-
-
- public StreamingProgramTestBase() {
- super(new Configuration());
- setParallelism(DEFAULT_PARALLELISM);
- }
-
-
- public void setParallelism(int parallelism) {
- this.parallelism = parallelism;
- setTaskManagerNumSlots(parallelism);
- }
-
- public int getParallelism() {
- return parallelism;
- }
-
-
- // --------------------------------------------------------------------------------------------
- // Methods to create the test program and for pre- and post- test work
- // --------------------------------------------------------------------------------------------
-
- protected abstract void testProgram() throws Exception;
-
- protected void preSubmit() throws Exception {}
-
- protected void postSubmit() throws Exception {}
-
- // --------------------------------------------------------------------------------------------
- // Test entry point
- // --------------------------------------------------------------------------------------------
-
- @Test
- public void testJob() throws Exception {
- try {
- // pre-submit
- try {
- preSubmit();
- }
- catch (Exception e) {
- System.err.println(e.getMessage());
- e.printStackTrace();
- fail("Pre-submit work caused an error: " + e.getMessage());
- }
-
- // prepare the test environment
- startCluster();
-
- TestStreamEnvironment.setAsContext(this.executor, getParallelism());
-
- // call the test program
- try {
- testProgram();
- }
- catch (Exception e) {
- System.err.println(e.getMessage());
- e.printStackTrace();
- fail("Error while calling the test program: " + e.getMessage());
- }
- finally {
- TestStreamEnvironment.unsetAsContext();
- }
-
- // post-submit
- try {
- postSubmit();
- }
- catch (Exception e) {
- System.err.println(e.getMessage());
- e.printStackTrace();
- fail("Post-submit work caused an error: " + e.getMessage());
- }
- }
- finally {
- stopCluster();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/b9f42e91/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/TestListResultSink.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/TestListResultSink.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/TestListResultSink.java
deleted file mode 100644
index 423d08e..0000000
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/TestListResultSink.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.util;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.TreeSet;
-
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
-
-public class TestListResultSink<T> extends RichSinkFunction<T> {
-
- private static final long serialVersionUID = 1L;
- private int resultListId;
-
- public TestListResultSink() {
- this.resultListId = TestListWrapper.getInstance().createList();
- }
-
- @Override
- public void open(Configuration parameters) throws Exception {
- super.open(parameters);
- }
-
- @Override
- public void invoke(T value) throws Exception {
- synchronized (resultList()) {
- resultList().add(value);
- }
- }
-
- @Override
- public void close() throws Exception {
- super.close();
- }
-
- @SuppressWarnings("unchecked")
- private List<T> resultList() {
- synchronized (TestListWrapper.getInstance()) {
- return (List<T>) TestListWrapper.getInstance().getList(resultListId);
- }
- }
-
- public List<T> getResult() {
- synchronized (resultList()) {
- ArrayList<T> copiedList = new ArrayList<T>(resultList());
- return copiedList;
- }
- }
-
- public List<T> getSortedResult() {
- synchronized (resultList()) {
- TreeSet<T> treeSet = new TreeSet<T>(resultList());
- ArrayList<T> sortedList = new ArrayList<T>(treeSet);
- return sortedList;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/b9f42e91/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/TestListWrapper.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/TestListWrapper.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/TestListWrapper.java
deleted file mode 100644
index 751f836..0000000
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/TestListWrapper.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.util;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
-public class TestListWrapper {
-
- private static TestListWrapper instance;
-
- @SuppressWarnings("rawtypes")
- private List<List<? extends Comparable>> lists;
-
- @SuppressWarnings("rawtypes")
- private TestListWrapper() {
- lists = Collections.synchronizedList(new ArrayList<List<? extends Comparable>>());
- }
-
- public static TestListWrapper getInstance() {
- if (instance == null) {
- instance = new TestListWrapper();
- }
- return instance;
- }
-
- /**
- * Creates and stores a list, returns with the id.
- *
- * @return The ID of the list.
- */
- @SuppressWarnings("rawtypes")
- public int createList() {
- lists.add(new ArrayList<Comparable>());
- return lists.size() - 1;
- }
-
- public List<?> getList(int listId) {
- @SuppressWarnings("rawtypes")
- List<? extends Comparable> list = lists.get(listId);
- if (list == null) {
- throw new RuntimeException("No such list.");
- }
-
- return list;
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/b9f42e91/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/TestStreamEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/TestStreamEnvironment.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/TestStreamEnvironment.java
deleted file mode 100644
index c700102..0000000
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/TestStreamEnvironment.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.util;
-
-import org.apache.flink.api.common.JobExecutionResult;
-import org.apache.flink.runtime.jobgraph.JobGraph;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironmentFactory;
-import org.apache.flink.streaming.api.graph.StreamGraph;
-import org.apache.flink.test.util.ForkableFlinkMiniCluster;
-import org.apache.flink.util.Preconditions;
-
-/**
- * A StreamExecutionEnvironment that executes its jobs on a test cluster.
- */
-public class TestStreamEnvironment extends StreamExecutionEnvironment {
-
- /** The mini cluster in which this environment executes its jobs */
- private ForkableFlinkMiniCluster executor;
-
-
- public TestStreamEnvironment(ForkableFlinkMiniCluster executor, int parallelism) {
- this.executor = Preconditions.checkNotNull(executor);
- setParallelism(parallelism);
- }
-
- @Override
- public JobExecutionResult execute(String jobName) throws Exception {
- final StreamGraph streamGraph = getStreamGraph();
- streamGraph.setJobName(jobName);
- final JobGraph jobGraph = streamGraph.getJobGraph();
- return executor.submitJobAndWait(jobGraph, false);
- }
-
- // ------------------------------------------------------------------------
-
- /**
- * Sets the streaming context environment to a TestStreamEnvironment that runs its programs on
- * the given cluster with the given default parallelism.
- *
- * @param cluster The test cluster to run the test program on.
- * @param parallelism The default parallelism for the test programs.
- */
- public static void setAsContext(final ForkableFlinkMiniCluster cluster, final int parallelism) {
-
- StreamExecutionEnvironmentFactory factory = new StreamExecutionEnvironmentFactory() {
- @Override
- public StreamExecutionEnvironment createExecutionEnvironment() {
- return new TestStreamEnvironment(cluster, parallelism);
- }
- };
-
- initializeContextEnvironment(factory);
- }
-
- /**
- * Resets the streaming context environment to null.
- */
- public static void unsetAsContext() {
- resetContextEnvironment();
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/b9f42e91/flink-streaming-scala/pom.xml
----------------------------------------------------------------------
diff --git a/flink-streaming-scala/pom.xml b/flink-streaming-scala/pom.xml
index ffbcb87..b82faf1 100644
--- a/flink-streaming-scala/pom.xml
+++ b/flink-streaming-scala/pom.xml
@@ -98,15 +98,6 @@ under the License.
<type>test-jar</type>
</dependency>
- <!-- To access streaming test utils -->
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-streaming-java_2.10</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
-
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/flink/blob/b9f42e91/flink-streaming-scala/src/test/java/org/apache/flink/streaming/scala/api/SocketOutputFormatITCase.java
----------------------------------------------------------------------
diff --git a/flink-streaming-scala/src/test/java/org/apache/flink/streaming/scala/api/SocketOutputFormatITCase.java b/flink-streaming-scala/src/test/java/org/apache/flink/streaming/scala/api/SocketOutputFormatITCase.java
deleted file mode 100644
index 7b3ed67..0000000
--- a/flink-streaming-scala/src/test/java/org/apache/flink/streaming/scala/api/SocketOutputFormatITCase.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.scala.api;
-
-import org.apache.flink.streaming.api.scala.OutputFormatTestPrograms;
-import org.apache.flink.streaming.util.SocketOutputTestBase;
-import org.apache.flink.test.testdata.WordCountData;
-import org.junit.Ignore;
-
-@Ignore
-//This test sometimes fails most likely due to the behaviour
-//of the socket. Disabled for now.
-public class SocketOutputFormatITCase extends SocketOutputTestBase {
-
- @Override
- protected void testProgram() throws Exception {
- OutputFormatTestPrograms.wordCountToSocket(WordCountData.TEXT, HOST, port);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/b9f42e91/flink-test-utils-parent/flink-test-utils/pom.xml
----------------------------------------------------------------------
diff --git a/flink-test-utils-parent/flink-test-utils/pom.xml b/flink-test-utils-parent/flink-test-utils/pom.xml
index 3c35cf1..238c2da 100644
--- a/flink-test-utils-parent/flink-test-utils/pom.xml
+++ b/flink-test-utils-parent/flink-test-utils/pom.xml
@@ -58,6 +58,13 @@ under the License.
</dependency>
<dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-streaming-java_2.10</artifactId>
+ <version>${project.version}</version>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>${junit.version}</version>
http://git-wip-us.apache.org/repos/asf/flink/blob/b9f42e91/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/streaming/util/StreamingMultipleProgramsTestBase.java
----------------------------------------------------------------------
diff --git a/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/streaming/util/StreamingMultipleProgramsTestBase.java b/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/streaming/util/StreamingMultipleProgramsTestBase.java
new file mode 100644
index 0000000..c5fbaf0
--- /dev/null
+++ b/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/streaming/util/StreamingMultipleProgramsTestBase.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.util;
+
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.test.util.AbstractTestBase;
+import org.apache.flink.test.util.ForkableFlinkMiniCluster;
+import org.apache.flink.test.util.TestBaseUtils;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+/**
+ * Base class for streaming unit tests that run multiple tests and want to reuse the same
+ * Flink cluster. This saves a significant amount of time, since the startup and
+ * shutdown of the Flink clusters (including actor systems, etc) usually dominates
+ * the execution of the actual tests.
+ *
+ * To write a unit test against this test base, simply extend it and add
+ * one or more regular test methods and retrieve the StreamExecutionEnvironment from
+ * the context:
+ *
+ * <pre>
+ * {@literal @}Test
+ * public void someTest() {
+ * StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+ * // test code
+ * env.execute();
+ * }
+ *
+ * {@literal @}Test
+ * public void anotherTest() {
+ * StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+ * // test code
+ * env.execute();
+ * }
+ *
+ * </pre>
+ */
+public class StreamingMultipleProgramsTestBase extends AbstractTestBase {
+
+ // ------------------------------------------------------------------------
+ // The mini cluster that is shared across tests
+ // ------------------------------------------------------------------------
+
+ protected static final int DEFAULT_PARALLELISM = 4;
+
+ protected static ForkableFlinkMiniCluster cluster;
+
+ public StreamingMultipleProgramsTestBase() {
+ super(new Configuration());
+ }
+
+ // ------------------------------------------------------------------------
+ // Cluster setup & teardown
+ // ------------------------------------------------------------------------
+
+ @BeforeClass
+ public static void setup() throws Exception {
+ cluster = TestBaseUtils.startCluster(1, DEFAULT_PARALLELISM, false, false, true);
+ TestStreamEnvironment.setAsContext(cluster, DEFAULT_PARALLELISM);
+ }
+
+ @AfterClass
+ public static void teardown() throws Exception {
+ TestStreamEnvironment.unsetAsContext();
+ stopCluster(cluster, TestBaseUtils.DEFAULT_TIMEOUT);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/b9f42e91/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/streaming/util/StreamingProgramTestBase.java
----------------------------------------------------------------------
diff --git a/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/streaming/util/StreamingProgramTestBase.java b/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/streaming/util/StreamingProgramTestBase.java
new file mode 100644
index 0000000..50ed1cf
--- /dev/null
+++ b/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/streaming/util/StreamingProgramTestBase.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.util;
+
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.test.util.AbstractTestBase;
+
+import org.junit.Test;
+
+import static org.junit.Assert.fail;
+
+public abstract class StreamingProgramTestBase extends AbstractTestBase {
+
+ protected static final int DEFAULT_PARALLELISM = 4;
+
+ private int parallelism;
+
+
+ public StreamingProgramTestBase() {
+ super(new Configuration());
+ setParallelism(DEFAULT_PARALLELISM);
+ }
+
+
+ public void setParallelism(int parallelism) {
+ this.parallelism = parallelism;
+ setTaskManagerNumSlots(parallelism);
+ }
+
+ public int getParallelism() {
+ return parallelism;
+ }
+
+
+ // --------------------------------------------------------------------------------------------
+ // Methods to create the test program and for pre- and post- test work
+ // --------------------------------------------------------------------------------------------
+
+ protected abstract void testProgram() throws Exception;
+
+ protected void preSubmit() throws Exception {}
+
+ protected void postSubmit() throws Exception {}
+
+ // --------------------------------------------------------------------------------------------
+ // Test entry point
+ // --------------------------------------------------------------------------------------------
+
+ @Test
+ public void testJob() throws Exception {
+ try {
+ // pre-submit
+ try {
+ preSubmit();
+ }
+ catch (Exception e) {
+ System.err.println(e.getMessage());
+ e.printStackTrace();
+ fail("Pre-submit work caused an error: " + e.getMessage());
+ }
+
+ // prepare the test environment
+ startCluster();
+
+ TestStreamEnvironment.setAsContext(this.executor, getParallelism());
+
+ // call the test program
+ try {
+ testProgram();
+ }
+ catch (Exception e) {
+ System.err.println(e.getMessage());
+ e.printStackTrace();
+ fail("Error while calling the test program: " + e.getMessage());
+ }
+ finally {
+ TestStreamEnvironment.unsetAsContext();
+ }
+
+ // post-submit
+ try {
+ postSubmit();
+ }
+ catch (Exception e) {
+ System.err.println(e.getMessage());
+ e.printStackTrace();
+ fail("Post-submit work caused an error: " + e.getMessage());
+ }
+ }
+ finally {
+ stopCluster();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/b9f42e91/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/streaming/util/TestStreamEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/streaming/util/TestStreamEnvironment.java b/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/streaming/util/TestStreamEnvironment.java
new file mode 100644
index 0000000..c700102
--- /dev/null
+++ b/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/streaming/util/TestStreamEnvironment.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.util;
+
+import org.apache.flink.api.common.JobExecutionResult;
+import org.apache.flink.runtime.jobgraph.JobGraph;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironmentFactory;
+import org.apache.flink.streaming.api.graph.StreamGraph;
+import org.apache.flink.test.util.ForkableFlinkMiniCluster;
+import org.apache.flink.util.Preconditions;
+
+/**
+ * A StreamExecutionEnvironment that executes its jobs on a test cluster.
+ */
+public class TestStreamEnvironment extends StreamExecutionEnvironment {
+
+ /** The mini cluster in which this environment executes its jobs */
+ private ForkableFlinkMiniCluster executor;
+
+
+ public TestStreamEnvironment(ForkableFlinkMiniCluster executor, int parallelism) {
+ this.executor = Preconditions.checkNotNull(executor);
+ setParallelism(parallelism);
+ }
+
+ @Override
+ public JobExecutionResult execute(String jobName) throws Exception {
+ final StreamGraph streamGraph = getStreamGraph();
+ streamGraph.setJobName(jobName);
+ final JobGraph jobGraph = streamGraph.getJobGraph();
+ return executor.submitJobAndWait(jobGraph, false);
+ }
+
+ // ------------------------------------------------------------------------
+
+ /**
+ * Sets the streaming context environment to a TestStreamEnvironment that runs its programs on
+ * the given cluster with the given default parallelism.
+ *
+ * @param cluster The test cluster to run the test program on.
+ * @param parallelism The default parallelism for the test programs.
+ */
+ public static void setAsContext(final ForkableFlinkMiniCluster cluster, final int parallelism) {
+
+ StreamExecutionEnvironmentFactory factory = new StreamExecutionEnvironmentFactory() {
+ @Override
+ public StreamExecutionEnvironment createExecutionEnvironment() {
+ return new TestStreamEnvironment(cluster, parallelism);
+ }
+ };
+
+ initializeContextEnvironment(factory);
+ }
+
+ /**
+ * Resets the streaming context environment to null.
+ */
+ public static void unsetAsContext() {
+ resetContextEnvironment();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/b9f42e91/flink-tests/pom.xml
----------------------------------------------------------------------
diff --git a/flink-tests/pom.xml b/flink-tests/pom.xml
index 9fd8c3e..77216e0 100644
--- a/flink-tests/pom.xml
+++ b/flink-tests/pom.xml
@@ -542,7 +542,7 @@ under the License.
</goals>
</pluginExecutionFilter>
<action>
- <ignore></ignore>
+ <ignore/>
</action>
</pluginExecution>
</pluginExecutions>
http://git-wip-us.apache.org/repos/asf/flink/blob/b9f42e91/flink-tests/src/test/java/org/apache/flink/test/streaming/api/StreamingOperatorsITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/streaming/api/StreamingOperatorsITCase.java b/flink-tests/src/test/java/org/apache/flink/test/streaming/api/StreamingOperatorsITCase.java
new file mode 100644
index 0000000..5d99de4
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/test/streaming/api/StreamingOperatorsITCase.java
@@ -0,0 +1,236 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.test.streaming.api;
+
+import org.apache.flink.api.common.functions.FoldFunction;
+import org.apache.flink.api.common.functions.MapFunction;
+import org.apache.flink.api.common.functions.RichMapFunction;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.core.fs.FileSystem;
+import org.apache.flink.util.MathUtils;
+import org.apache.flink.streaming.api.collector.selector.OutputSelector;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.datastream.SplitStream;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.functions.source.SourceFunction;
+import org.apache.flink.streaming.util.StreamingMultipleProgramsTestBase;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class StreamingOperatorsITCase extends StreamingMultipleProgramsTestBase {
+
+ private String resultPath1;
+ private String resultPath2;
+ private String expected1;
+ private String expected2;
+
+ @Rule
+ public TemporaryFolder tempFolder = new TemporaryFolder();
+
+ @Before
+ public void before() throws Exception {
+ resultPath1 = tempFolder.newFile().toURI().toString();
+ resultPath2 = tempFolder.newFile().toURI().toString();
+ expected1 = "";
+ expected2 = "";
+ }
+
+ @After
+ public void after() throws Exception {
+ compareResultsByLinesInMemory(expected1, resultPath1);
+ compareResultsByLinesInMemory(expected2, resultPath2);
+ }
+
+ /**
+ * Tests the proper functioning of the streaming fold operator. For this purpose, a stream
+ * of Tuple2<Integer, Integer> is created. The stream is grouped according to the first tuple
+ * value. Each group is folded where the second tuple value is summed up.
+ *
+ * This test relies on the hash function used by the {@link DataStream#keyBy}, which is
+ * assumed to be {@link MathUtils#murmurHash}.
+ */
+ @Test
+ public void testGroupedFoldOperation() throws Exception {
+ int numElements = 10;
+ final int numKeys = 2;
+
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+ DataStream<Tuple2<Integer, Integer>> sourceStream = env.addSource(new TupleSource(numElements, numKeys));
+
+ SplitStream<Tuple2<Integer, Integer>> splittedResult = sourceStream
+ .keyBy(0)
+ .fold(0, new FoldFunction<Tuple2<Integer, Integer>, Integer>() {
+ @Override
+ public Integer fold(Integer accumulator, Tuple2<Integer, Integer> value) throws Exception {
+ return accumulator + value.f1;
+ }
+ }).map(new RichMapFunction<Integer, Tuple2<Integer, Integer>>() {
+ int key = -1;
+ @Override
+ public Tuple2<Integer, Integer> map(Integer value) throws Exception {
+ if (key == -1){
+ key = MathUtils.murmurHash(value) % numKeys;
+ }
+ return new Tuple2<>(key, value);
+ }
+ }).split(new OutputSelector<Tuple2<Integer, Integer>>() {
+ @Override
+ public Iterable<String> select(Tuple2<Integer, Integer> value) {
+ List<String> output = new ArrayList<>();
+
+ output.add(value.f0 + "");
+ return output;
+ }
+ });
+
+ splittedResult.select("0").map(new MapFunction<Tuple2<Integer,Integer>, Integer>() {
+ @Override
+ public Integer map(Tuple2<Integer, Integer> value) throws Exception {
+ return value.f1;
+ }
+ }).writeAsText(resultPath1, FileSystem.WriteMode.OVERWRITE);
+
+ splittedResult.select("1").map(new MapFunction<Tuple2<Integer, Integer>, Integer>() {
+ @Override
+ public Integer map(Tuple2<Integer, Integer> value) throws Exception {
+ return value.f1;
+ }
+ }).writeAsText(resultPath2, FileSystem.WriteMode.OVERWRITE);
+
+ StringBuilder builder1 = new StringBuilder();
+ StringBuilder builder2 = new StringBuilder();
+ int counter1 = 0;
+ int counter2 = 0;
+
+ for (int i = 0; i < numElements; i++) {
+ if (MathUtils.murmurHash(i) % numKeys == 0) {
+ counter1 += i;
+ builder1.append(counter1 + "\n");
+ } else {
+ counter2 += i;
+ builder2.append(counter2 + "\n");
+ }
+ }
+
+ expected1 = builder1.toString();
+ expected2 = builder2.toString();
+
+ env.execute();
+ }
+
+ /**
+ * Tests whether the fold operation can also be called with non Java serializable types.
+ */
+ @Test
+ public void testFoldOperationWithNonJavaSerializableType() throws Exception {
+ final int numElements = 10;
+
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+ DataStream<Tuple2<Integer, NonSerializable>> input = env.addSource(new NonSerializableTupleSource(numElements));
+
+ input
+ .keyBy(0)
+ .fold(
+ new NonSerializable(42),
+ new FoldFunction<Tuple2<Integer, NonSerializable>, NonSerializable>() {
+ @Override
+ public NonSerializable fold(NonSerializable accumulator, Tuple2<Integer, NonSerializable> value) throws Exception {
+ return new NonSerializable(accumulator.value + value.f1.value);
+ }
+ })
+ .map(new MapFunction<NonSerializable, Integer>() {
+ @Override
+ public Integer map(NonSerializable value) throws Exception {
+ return value.value;
+ }
+ })
+ .writeAsText(resultPath1, FileSystem.WriteMode.OVERWRITE);
+
+ StringBuilder builder = new StringBuilder();
+
+ for (int i = 0; i < numElements; i++) {
+ builder.append(42 + i + "\n");
+ }
+
+ expected1 = builder.toString();
+
+ env.execute();
+ }
+
+ private static class NonSerializable {
+ // This makes the type non-serializable
+ private final Object obj = new Object();
+
+ private final int value;
+
+ public NonSerializable(int value) {
+ this.value = value;
+ }
+ }
+
+ private static class NonSerializableTupleSource implements SourceFunction<Tuple2<Integer, NonSerializable>> {
+ private final int numElements;
+
+ public NonSerializableTupleSource(int numElements) {
+ this.numElements = numElements;
+ }
+
+
+ @Override
+ public void run(SourceContext<Tuple2<Integer, NonSerializable>> ctx) throws Exception {
+ for (int i = 0; i < numElements; i++) {
+ ctx.collect(new Tuple2<>(i, new NonSerializable(i)));
+ }
+ }
+
+ @Override
+ public void cancel() {}
+ }
+
+ private static class TupleSource implements SourceFunction<Tuple2<Integer, Integer>> {
+
+ private final int numElements;
+ private final int numKeys;
+
+ public TupleSource(int numElements, int numKeys) {
+ this.numElements = numElements;
+ this.numKeys = numKeys;
+ }
+
+ @Override
+ public void run(SourceContext<Tuple2<Integer, Integer>> ctx) throws Exception {
+ for (int i = 0; i < numElements; i++) {
+ // keys '1' and '2' hash to different buckets
+ Tuple2<Integer, Integer> result = new Tuple2<>(1 + (MathUtils.murmurHash(i) % numKeys), i);
+ ctx.collect(result);
+ }
+ }
+
+ @Override
+ public void cancel() {
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/b9f42e91/flink-tests/src/test/java/org/apache/flink/test/streaming/api/outputformat/CsvOutputFormatITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/streaming/api/outputformat/CsvOutputFormatITCase.java b/flink-tests/src/test/java/org/apache/flink/test/streaming/api/outputformat/CsvOutputFormatITCase.java
new file mode 100644
index 0000000..c2155ac
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/test/streaming/api/outputformat/CsvOutputFormatITCase.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.test.streaming.api.outputformat;
+
+import org.apache.flink.api.common.functions.FlatMapFunction;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.util.StreamingProgramTestBase;
+import org.apache.flink.test.testdata.WordCountData;
+import org.apache.flink.util.Collector;
+
+public class CsvOutputFormatITCase extends StreamingProgramTestBase {
+
+ protected String resultPath;
+
+ @Override
+ protected void preSubmit() throws Exception {
+ resultPath = getTempDirPath("result");
+ }
+
+ @Override
+ protected void testProgram() throws Exception {
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+
+ DataStream<String> text = env.fromElements(WordCountData.TEXT);
+
+ DataStream<Tuple2<String, Integer>> counts = text
+ .flatMap(new Tokenizer())
+ .keyBy(0).sum(1);
+
+ counts.writeAsCsv(resultPath);
+
+ env.execute("WriteAsCsvTest");
+ }
+
+ @Override
+ protected void postSubmit() throws Exception {
+ //Strip the parentheses from the expected text like output
+ compareResultsByLinesInMemory(WordCountData.STREAMING_COUNTS_AS_TUPLES
+ .replaceAll("[\\\\(\\\\)]", ""), resultPath);
+ }
+
+ public static final class Tokenizer implements FlatMapFunction<String, Tuple2<String, Integer>> {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public void flatMap(String value, Collector<Tuple2<String, Integer>> out)
+ throws Exception {
+ // normalize and split the line
+ String[] tokens = value.toLowerCase().split("\\W+");
+
+ // emit the pairs
+ for (String token : tokens) {
+ if (token.length() > 0) {
+ out.collect(new Tuple2<String, Integer>(token, 1));
+ }
+ }
+ }
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/flink/blob/b9f42e91/flink-tests/src/test/java/org/apache/flink/test/streaming/api/outputformat/TextOutputFormatITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/streaming/api/outputformat/TextOutputFormatITCase.java b/flink-tests/src/test/java/org/apache/flink/test/streaming/api/outputformat/TextOutputFormatITCase.java
new file mode 100644
index 0000000..2940e6d
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/test/streaming/api/outputformat/TextOutputFormatITCase.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.test.streaming.api.outputformat;
+
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.util.StreamingProgramTestBase;
+import org.apache.flink.test.testdata.WordCountData;
+
+public class TextOutputFormatITCase extends StreamingProgramTestBase {
+
+ protected String resultPath;
+
+ @Override
+ protected void preSubmit() throws Exception {
+ resultPath = getTempDirPath("result");
+ }
+
+ @Override
+ protected void testProgram() throws Exception {
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+
+ DataStream<String> text = env.fromElements(WordCountData.TEXT);
+
+ DataStream<Tuple2<String, Integer>> counts = text
+ .flatMap(new CsvOutputFormatITCase.Tokenizer())
+ .keyBy(0).sum(1);
+
+ counts.writeAsText(resultPath);
+
+ env.execute("WriteAsTextTest");
+ }
+
+ @Override
+ protected void postSubmit() throws Exception {
+ compareResultsByLinesInMemory(WordCountData.STREAMING_COUNTS_AS_TUPLES, resultPath);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/b9f42e91/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/ChainedRuntimeContextITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/ChainedRuntimeContextITCase.java b/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/ChainedRuntimeContextITCase.java
new file mode 100644
index 0000000..d21985b
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/ChainedRuntimeContextITCase.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.test.streaming.runtime;
+
+import org.apache.flink.api.common.functions.RichMapFunction;
+import org.apache.flink.api.common.functions.RuntimeContext;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.functions.sink.DiscardingSink;
+import org.apache.flink.streaming.api.functions.source.RichParallelSourceFunction;
+import org.apache.flink.streaming.util.StreamingMultipleProgramsTestBase;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertNotEquals;
+
+@SuppressWarnings("serial")
+public class ChainedRuntimeContextITCase extends StreamingMultipleProgramsTestBase {
+ private static RuntimeContext srcContext;
+ private static RuntimeContext mapContext;
+
+ @Test
+ public void test() throws Exception {
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+ env.setParallelism(1);
+
+ env.addSource(new TestSource()).map(new TestMap()).addSink(new DiscardingSink<Integer>());
+ env.execute();
+
+ assertNotEquals(srcContext, mapContext);
+
+ }
+
+ private static class TestSource extends RichParallelSourceFunction<Integer> {
+
+ @Override
+ public void run(SourceContext<Integer> ctx) throws Exception {
+ }
+
+ @Override
+ public void cancel() {
+ }
+
+ @Override
+ public void open(Configuration c) {
+ srcContext = getRuntimeContext();
+ }
+
+ }
+
+ private static class TestMap extends RichMapFunction<Integer, Integer> {
+
+ @Override
+ public Integer map(Integer value) throws Exception {
+ return value;
+ }
+
+ @Override
+ public void open(Configuration c) {
+ mapContext = getRuntimeContext();
+ }
+
+ }
+
+}
[09/15] flink git commit: [FLINK-3995] [build] Properly structure
test scopes and dependencies
Posted by rm...@apache.org.
[FLINK-3995] [build] Properly structure test scopes and dependencies
Makes the JUnit test utils (TestLogger, retry rules, ...) properly available to
other projects without the 'flink-core' test-jar, via the 'flink-test-utils-junit' project.
Makes the ForkableMiniCluster, TestEnvironment, and other test utilities available in the 'main'
scope of the 'flink-test-utils' project.
Creates a 'flink-test-utils-parent' project that holds the 'flink-test-utils-junit' and
'flink-test-utils' project.
Also moves some tests between projects and inlines some very simple utility functions in
order to simplify some test jar dependencies.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/4b71e0e7
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/4b71e0e7
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/4b71e0e7
Branch: refs/heads/master
Commit: 4b71e0e7a3c452ee515d8514f36ab642d1e6e604
Parents: 5c0c7e3
Author: Stephan Ewen <se...@apache.org>
Authored: Mon May 30 20:56:05 2016 +0200
Committer: Robert Metzger <rm...@apache.org>
Committed: Tue Jul 5 15:49:46 2016 +0200
----------------------------------------------------------------------
flink-batch-connectors/flink-avro/pom.xml | 26 +-
.../api/io/avro/AvroRecordInputFormatTest.java | 37 +-
.../flink-hadoop-compatibility/pom.xml | 11 +-
flink-batch-connectors/flink-hbase/pom.xml | 58 +-
flink-clients/pom.xml | 30 +-
flink-contrib/flink-operator-stats/pom.xml | 21 +-
.../OperatorStatsAccumulatorTest.java | 4 +-
.../flink-statebackend-rocksdb/pom.xml | 22 +-
.../state/RocksDBAsyncKVSnapshotTest.java | 2 +-
flink-contrib/flink-storm-examples/pom.xml | 23 +-
flink-contrib/flink-streaming-contrib/pom.xml | 9 +-
flink-core/pom.xml | 36 +-
.../java/typeutils/InputTypeConfigurable.java | 8 +-
.../apache/flink/util/InstantiationUtil.java | 30 +
.../flink/api/common/ExecutionConfigTest.java | 13 -
.../api/common/io/SequentialFormatTestBase.java | 301 ---------
.../api/common/io/SerializedFormatTest.java | 103 ---
.../flink/configuration/ConfigurationTest.java | 6 +-
.../FilesystemSchemeConfigTest.java | 14 +-
.../configuration/GlobalConfigurationTest.java | 10 +-
.../core/fs/local/LocalFileSystemTest.java | 40 +-
.../flink/core/testutils/CommonTestUtils.java | 159 -----
.../org/apache/flink/util/AbstractIDTest.java | 23 +-
.../flink/util/InstantiationUtilTest.java | 59 +-
.../java/org/apache/flink/util/TestLogger.java | 62 --
flink-examples/flink-examples-streaming/pom.xml | 12 +-
flink-fs-tests/pom.xml | 13 +-
.../flink/hdfstests/FileStateBackendTest.java | 2 +-
flink-java/pom.xml | 7 +-
.../api/common/io/SequentialFormatTestBase.java | 300 +++++++++
.../api/common/io/SerializedFormatTest.java | 102 +++
flink-java8/pom.xml | 22 +-
flink-libraries/flink-cep-scala/pom.xml | 28 +-
flink-libraries/flink-cep/pom.xml | 24 +-
flink-libraries/flink-gelly-examples/pom.xml | 7 +-
flink-libraries/flink-gelly-scala/pom.xml | 5 +-
flink-libraries/flink-gelly/pom.xml | 9 +-
flink-libraries/flink-ml/pom.xml | 20 +-
.../org/apache/flink/ml/MLUtilsSuite.scala | 20 +-
.../flink/ml/classification/SVMITSuite.scala | 2 +-
.../flink/ml/common/FlinkMLToolsSuite.scala | 2 +-
.../org/apache/flink/ml/nn/KNNITSuite.scala | 2 +-
.../org/apache/flink/ml/nn/QuadTreeSuite.scala | 2 +-
.../optimization/GradientDescentITSuite.scala | 2 +-
.../ml/optimization/LossFunctionITSuite.scala | 2 +-
.../PredictionFunctionITSuite.scala | 2 +-
.../flink/ml/pipeline/PipelineITSuite.scala | 2 +-
.../ml/preprocessing/MinMaxScalerITSuite.scala | 2 +-
.../PolynomialFeaturesITSuite.scala | 2 +-
.../preprocessing/StandardScalerITSuite.scala | 2 +-
.../flink/ml/recommendation/ALSITSuite.scala | 2 +-
.../MultipleLinearRegressionITSuite.scala | 2 +-
.../apache/flink/ml/util/FlinkTestBase.scala | 74 ++
flink-libraries/flink-python/pom.xml | 7 +-
.../flink/python/api/PythonPlanBinderTest.java | 13 +-
flink-libraries/flink-table/pom.xml | 66 +-
.../api/scala/stream/TableSinkITCase.scala | 11 +-
flink-optimizer/pom.xml | 20 +-
flink-runtime-web/pom.xml | 101 +--
.../StackTraceSampleCoordinatorTest.java | 1 -
flink-runtime/pom.xml | 42 +-
.../testutils/TestingResourceManager.java | 137 ++++
.../testingUtils/TestingJobManager.scala | 71 ++
.../testingUtils/TestingJobManagerLike.scala | 405 +++++++++++
.../TestingJobManagerMessages.scala | 115 ++++
.../testingUtils/TestingMemoryArchivist.scala | 43 ++
.../runtime/testingUtils/TestingMessages.scala | 40 ++
.../testingUtils/TestingTaskManager.scala | 71 ++
.../testingUtils/TestingTaskManagerLike.scala | 248 +++++++
.../TestingTaskManagerMessages.scala | 94 +++
...ExecutionGraphCheckpointCoordinatorTest.java | 10 +-
.../TaskDeploymentDescriptorTest.java | 4 +-
.../flink/runtime/event/task/TaskEventTest.java | 9 +-
.../ExecutionGraphConstructionTest.java | 46 +-
.../ExecutionGraphDeploymentTest.java | 10 +-
.../ExecutionGraphRestartTest.java | 21 +-
.../ExecutionGraphSignalsTest.java | 6 +-
.../executiongraph/ExecutionGraphTestUtils.java | 13 +-
.../ExecutionStateProgressTest.java | 9 +-
.../executiongraph/LocalInputSplitsTest.java | 7 +-
.../executiongraph/PointwisePatternTest.java | 36 +-
.../TerminalStateDeadlockTest.java | 13 +-
.../VertexLocationConstraintTest.java | 15 +-
.../executiongraph/VertexSlotSharingTest.java | 5 +-
.../instance/InstanceConnectionInfoTest.java | 11 +-
.../runtime/taskmanager/MultiShotLatch.java | 57 --
.../flink/runtime/taskmanager/OneShotLatch.java | 56 --
.../runtime/taskmanager/TaskAsyncCallTest.java | 8 +-
.../TaskManagerConfigurationTest.java | 17 +-
.../runtime/taskmanager/TaskManagerTest.java | 64 +-
.../flink/runtime/taskmanager/TaskTest.java | 22 +-
.../testutils/TestingResourceManager.java | 137 ----
.../TaskManagerLossFailsTasksTest.scala | 5 +-
.../testingUtils/TestingJobManager.scala | 71 --
.../testingUtils/TestingJobManagerLike.scala | 404 -----------
.../TestingJobManagerMessages.scala | 115 ----
.../testingUtils/TestingMemoryArchivist.scala | 43 --
.../runtime/testingUtils/TestingMessages.scala | 40 --
.../testingUtils/TestingTaskManager.scala | 71 --
.../testingUtils/TestingTaskManagerLike.scala | 248 -------
.../TestingTaskManagerMessages.scala | 94 ---
.../runtime/testingUtils/TestingUtils.scala | 1 -
flink-scala-shell/pom.xml | 25 +-
flink-scala/pom.xml | 8 +
.../flink-connector-elasticsearch/pom.xml | 55 +-
.../flink-connector-elasticsearch2/pom.xml | 59 +-
.../flink-connector-filesystem/pom.xml | 21 +-
.../connectors/fs/RollingSinkITCase.java | 37 +-
.../flink-connector-kafka-0.8/pom.xml | 22 +-
.../flink-connector-kafka-0.9/pom.xml | 92 +--
.../flink-connector-kafka-base/pom.xml | 23 +-
flink-streaming-java/pom.xml | 37 +-
.../streaming/api/RestartStrategyTest.java | 20 +-
.../graph/StreamingJobGraphGeneratorTest.java | 6 +-
.../partitioner/RescalePartitionerTest.java | 8 +-
.../tasks/StreamTaskAsyncCheckpointTest.java | 2 +-
.../streaming/runtime/tasks/StreamTaskTest.java | 7 +-
.../streaming/timestamp/TimestampITCase.java | 5 +-
flink-streaming-scala/pom.xml | 20 +-
.../flink-test-utils-junit/pom.xml | 46 ++
.../flink/core/testutils/CommonTestUtils.java | 91 +++
.../flink/core/testutils/MultiShotLatch.java | 55 ++
.../flink/core/testutils/OneShotLatch.java | 55 ++
.../flink/testutils/junit/RetryOnException.java | 60 ++
.../flink/testutils/junit/RetryOnFailure.java | 49 ++
.../apache/flink/testutils/junit/RetryRule.java | 175 +++++
.../java/org/apache/flink/util/TestLogger.java | 83 +++
.../testutils/junit/RetryOnExceptionTest.java | 83 +++
.../testutils/junit/RetryOnFailureTest.java | 78 +++
.../flink-test-utils/pom.xml | 224 ++++++
.../test/testdata/ConnectedComponentsData.java | 128 ++++
.../flink/test/testdata/EnumTriangleData.java | 48 ++
.../apache/flink/test/testdata/KMeansData.java | 257 +++++++
.../flink/test/testdata/PageRankData.java | 57 ++
.../test/testdata/TransitiveClosureData.java | 48 ++
.../flink/test/testdata/WebLogAnalysisData.java | 149 ++++
.../flink/test/testdata/WordCountData.java | 329 +++++++++
.../flink/test/util/AbstractTestBase.java | 152 +++++
.../test/util/CollectionTestEnvironment.java | 63 ++
.../flink/test/util/JavaProgramTestBase.java | 280 ++++++++
.../test/util/MultipleProgramsTestBase.java | 126 ++++
.../apache/flink/test/util/TestBaseUtils.java | 673 +++++++++++++++++++
.../apache/flink/test/util/TestEnvironment.java | 110 +++
.../test/util/ForkableFlinkMiniCluster.scala | 308 +++++++++
flink-test-utils-parent/pom.xml | 43 ++
flink-test-utils/pom.xml | 246 -------
.../test/testdata/ConnectedComponentsData.java | 128 ----
.../flink/test/testdata/EnumTriangleData.java | 48 --
.../apache/flink/test/testdata/KMeansData.java | 257 -------
.../flink/test/testdata/PageRankData.java | 57 --
.../test/testdata/TransitiveClosureData.java | 48 --
.../flink/test/testdata/WebLogAnalysisData.java | 149 ----
.../flink/test/testdata/WordCountData.java | 329 ---------
.../flink/test/util/AbstractTestBase.java | 152 -----
.../test/util/CollectionTestEnvironment.java | 63 --
.../flink/test/util/JavaProgramTestBase.java | 279 --------
.../test/util/MultipleProgramsTestBase.java | 126 ----
.../apache/flink/test/util/TestBaseUtils.java | 666 ------------------
.../apache/flink/test/util/TestEnvironment.java | 110 ---
.../apache/flink/test/util/FlinkTestBase.scala | 73 --
.../test/util/ForkableFlinkMiniCluster.scala | 305 ---------
flink-tests/pom.xml | 21 +-
.../flink/api/scala/io/CsvInputFormatTest.scala | 2 -
flink-yarn-tests/pom.xml | 9 +-
pom.xml | 5 +-
165 files changed, 6441 insertions(+), 5780 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-batch-connectors/flink-avro/pom.xml
----------------------------------------------------------------------
diff --git a/flink-batch-connectors/flink-avro/pom.xml b/flink-batch-connectors/flink-avro/pom.xml
index 0c2d583..fb9d364 100644
--- a/flink-batch-connectors/flink-avro/pom.xml
+++ b/flink-batch-connectors/flink-avro/pom.xml
@@ -18,11 +18,10 @@ specific language governing permissions and limitations
under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
-
+
<parent>
<groupId>org.apache.flink</groupId>
<artifactId>flink-batch-connectors</artifactId>
@@ -36,6 +35,9 @@ under the License.
<packaging>jar</packaging>
<dependencies>
+
+ <!-- core dependencies -->
+
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-java</artifactId>
@@ -49,11 +51,19 @@ under the License.
<!-- version is derived from base module -->
</dependency>
+ <!-- test dependencies -->
+
<dependency>
<groupId>org.apache.flink</groupId>
- <artifactId>flink-core</artifactId>
+ <artifactId>flink-test-utils-junit</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-test-utils_2.10</artifactId>
<version>${project.version}</version>
- <type>test-jar</type>
<scope>test</scope>
</dependency>
@@ -63,13 +73,13 @@ under the License.
<version>${project.version}</version>
<scope>test</scope>
</dependency>
-
+
<dependency>
<groupId>org.apache.flink</groupId>
- <artifactId>flink-test-utils_2.10</artifactId>
+ <artifactId>flink-core</artifactId>
<version>${project.version}</version>
- <type>test-jar</type>
<scope>test</scope>
+ <type>test-jar</type>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-batch-connectors/flink-avro/src/test/java/org/apache/flink/api/io/avro/AvroRecordInputFormatTest.java
----------------------------------------------------------------------
diff --git a/flink-batch-connectors/flink-avro/src/test/java/org/apache/flink/api/io/avro/AvroRecordInputFormatTest.java b/flink-batch-connectors/flink-avro/src/test/java/org/apache/flink/api/io/avro/AvroRecordInputFormatTest.java
index 8439e01..91a9612 100644
--- a/flink-batch-connectors/flink-avro/src/test/java/org/apache/flink/api/io/avro/AvroRecordInputFormatTest.java
+++ b/flink-batch-connectors/flink-avro/src/test/java/org/apache/flink/api/io/avro/AvroRecordInputFormatTest.java
@@ -20,6 +20,8 @@ package org.apache.flink.api.io.avro;
import static org.junit.Assert.*;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.util.*;
@@ -38,7 +40,6 @@ import org.apache.avro.specific.SpecificDatumWriter;
import org.apache.avro.util.Utf8;
import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.api.common.typeutils.ComparatorTestBase;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.api.io.avro.generated.Address;
import org.apache.flink.api.io.avro.generated.Colors;
@@ -51,6 +52,8 @@ import org.apache.flink.api.java.typeutils.runtime.kryo.Serializers;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.core.fs.FileInputSplit;
import org.apache.flink.core.fs.Path;
+import org.apache.flink.core.memory.DataInputViewStreamWrapper;
+import org.apache.flink.core.memory.DataOutputViewStreamWrapper;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -80,7 +83,7 @@ public class AvroRecordInputFormatTest {
final static String TEST_MAP_KEY2 = "KEY 2";
final static long TEST_MAP_VALUE2 = 17554L;
- final static Integer TEST_NUM = 239;
+ final static int TEST_NUM = 239;
final static String TEST_STREET = "Baker Street";
final static String TEST_CITY = "London";
final static String TEST_STATE = "London";
@@ -104,7 +107,7 @@ public class AvroRecordInputFormatTest {
longMap.put(TEST_MAP_KEY2, TEST_MAP_VALUE2);
Address addr = new Address();
- addr.setNum(new Integer(TEST_NUM));
+ addr.setNum(TEST_NUM);
addr.setStreet(TEST_STREET);
addr.setCity(TEST_CITY);
addr.setState(TEST_STATE);
@@ -293,10 +296,18 @@ public class AvroRecordInputFormatTest {
Assert.assertTrue(
ec.getDefaultKryoSerializerClasses().containsKey(Schema.class) &&
ec.getDefaultKryoSerializerClasses().get(Schema.class).equals(Serializers.AvroSchemaSerializer.class));
- ComparatorTestBase.TestOutputView target = new ComparatorTestBase.TestOutputView();
- tser.serialize(rec, target);
- GenericData.Record newRec = tser.deserialize(target.getInputView());
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ try (DataOutputViewStreamWrapper outView = new DataOutputViewStreamWrapper(out)) {
+ tser.serialize(rec, outView);
+ }
+
+ GenericData.Record newRec;
+ try (DataInputViewStreamWrapper inView = new DataInputViewStreamWrapper(
+ new ByteArrayInputStream(out.toByteArray())))
+ {
+ newRec = tser.deserialize(inView);
+ }
// check if it is still the same
assertNotNull(newRec);
@@ -328,10 +339,18 @@ public class AvroRecordInputFormatTest {
Assert.assertEquals(AvroTypeInfo.class, te.getClass());
TypeSerializer<User> tser = te.createSerializer(ec);
- ComparatorTestBase.TestOutputView target = new ComparatorTestBase.TestOutputView();
- tser.serialize(rec, target);
- User newRec = tser.deserialize(target.getInputView());
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ try (DataOutputViewStreamWrapper outView = new DataOutputViewStreamWrapper(out)) {
+ tser.serialize(rec, outView);
+ }
+
+ User newRec;
+ try (DataInputViewStreamWrapper inView = new DataInputViewStreamWrapper(
+ new ByteArrayInputStream(out.toByteArray())))
+ {
+ newRec = tser.deserialize(inView);
+ }
// check if it is still the same
assertNotNull(newRec);
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-batch-connectors/flink-hadoop-compatibility/pom.xml
----------------------------------------------------------------------
diff --git a/flink-batch-connectors/flink-hadoop-compatibility/pom.xml b/flink-batch-connectors/flink-hadoop-compatibility/pom.xml
index 5a5e4a6..7c30f9f 100644
--- a/flink-batch-connectors/flink-hadoop-compatibility/pom.xml
+++ b/flink-batch-connectors/flink-hadoop-compatibility/pom.xml
@@ -35,37 +35,42 @@ under the License.
<packaging>jar</packaging>
<dependencies>
+
+ <!-- core dependencies -->
+
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-java</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
-
+
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-clients_2.10</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
-
+
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>${shading-artifact.name}</artifactId>
<version>${project.version}</version>
</dependency>
+ <!-- test dependencies -->
+
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-tests_2.10</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>
+
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-test-utils_2.10</artifactId>
<version>${project.version}</version>
- <type>test-jar</type>
<scope>test</scope>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-batch-connectors/flink-hbase/pom.xml
----------------------------------------------------------------------
diff --git a/flink-batch-connectors/flink-hbase/pom.xml b/flink-batch-connectors/flink-hbase/pom.xml
index c75782f..cdf25db 100644
--- a/flink-batch-connectors/flink-hbase/pom.xml
+++ b/flink-batch-connectors/flink-hbase/pom.xml
@@ -39,6 +39,9 @@ under the License.
</properties>
<dependencies>
+
+ <!-- core dependencies -->
+
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-core</artifactId>
@@ -66,32 +69,6 @@ under the License.
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-clients_2.10</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-core</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-hadoop-compatibility_2.10</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-shaded-include-yarn_2.10</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
<!-- HBase server needed for TableOutputFormat -->
<!-- TODO implement bulk output format for HBase -->
@@ -157,6 +134,35 @@ under the License.
</exclusion>
</exclusions>
</dependency>
+
+ <!-- test dependencies -->
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-clients_2.10</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-core</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-hadoop-compatibility_2.10</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-shaded-include-yarn_2.10</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
</dependencies>
<profiles>
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-clients/pom.xml
----------------------------------------------------------------------
diff --git a/flink-clients/pom.xml b/flink-clients/pom.xml
index 5643f25..6109f36 100644
--- a/flink-clients/pom.xml
+++ b/flink-clients/pom.xml
@@ -36,18 +36,13 @@ under the License.
<packaging>jar</packaging>
<dependencies>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-core</artifactId>
- <version>${project.version}</version>
- </dependency>
+
+ <!-- core dependencies -->
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-core</artifactId>
<version>${project.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
</dependency>
<dependency>
@@ -73,9 +68,19 @@ under the License.
<artifactId>commons-cli</artifactId>
</dependency>
+ <!-- test dependencies -->
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-test-utils-junit</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+
<dependency>
<groupId>com.typesafe.akka</groupId>
<artifactId>akka-testkit_${scala.binary.version}</artifactId>
+ <scope>test</scope>
</dependency>
</dependencies>
@@ -85,17 +90,6 @@ under the License.
<build>
<plugins>
<plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <executions>
- <execution>
- <goals>
- <goal>test-jar</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
<artifactId>maven-assembly-plugin</artifactId>
<version>2.4</version><!--$NO-MVN-MAN-VER$-->
<executions>
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-contrib/flink-operator-stats/pom.xml
----------------------------------------------------------------------
diff --git a/flink-contrib/flink-operator-stats/pom.xml b/flink-contrib/flink-operator-stats/pom.xml
index 80d4ee243..839f8f5 100644
--- a/flink-contrib/flink-operator-stats/pom.xml
+++ b/flink-contrib/flink-operator-stats/pom.xml
@@ -34,23 +34,20 @@ under the License.
<packaging>jar</packaging>
<dependencies>
+
+ <!-- core dependencies -->
+
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-java</artifactId>
<version>${project.version}</version>
</dependency>
+
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-core</artifactId>
<version>${project.version}</version>
</dependency>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-test-utils_2.10</artifactId>
- <version>${project.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
<dependency>
<groupId>com.clearspring.analytics</groupId>
@@ -63,6 +60,16 @@ under the License.
</exclusion>
</exclusions>
</dependency>
+
+ <!-- test dependencies -->
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-test-utils_2.10</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-contrib/flink-operator-stats/src/test/java/org/apache/flink/contrib/operatorstatistics/OperatorStatsAccumulatorTest.java
----------------------------------------------------------------------
diff --git a/flink-contrib/flink-operator-stats/src/test/java/org/apache/flink/contrib/operatorstatistics/OperatorStatsAccumulatorTest.java b/flink-contrib/flink-operator-stats/src/test/java/org/apache/flink/contrib/operatorstatistics/OperatorStatsAccumulatorTest.java
index 6f488f6..82123e0 100644
--- a/flink-contrib/flink-operator-stats/src/test/java/org/apache/flink/contrib/operatorstatistics/OperatorStatsAccumulatorTest.java
+++ b/flink-contrib/flink-operator-stats/src/test/java/org/apache/flink/contrib/operatorstatistics/OperatorStatsAccumulatorTest.java
@@ -27,8 +27,10 @@ import org.apache.flink.api.java.tuple.Tuple1;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.test.util.AbstractTestBase;
import org.apache.flink.util.Collector;
+
import org.junit.Assert;
import org.junit.Test;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -43,7 +45,7 @@ public class OperatorStatsAccumulatorTest extends AbstractTestBase {
private static final String ACCUMULATOR_NAME = "op-stats";
- public OperatorStatsAccumulatorTest(){
+ public OperatorStatsAccumulatorTest() {
super(new Configuration());
}
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-contrib/flink-statebackend-rocksdb/pom.xml
----------------------------------------------------------------------
diff --git a/flink-contrib/flink-statebackend-rocksdb/pom.xml b/flink-contrib/flink-statebackend-rocksdb/pom.xml
index 115752c..7d551cb 100644
--- a/flink-contrib/flink-statebackend-rocksdb/pom.xml
+++ b/flink-contrib/flink-statebackend-rocksdb/pom.xml
@@ -37,46 +37,50 @@ under the License.
<packaging>jar</packaging>
<dependencies>
+
+ <!-- core dependencies -->
+
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-streaming-java_2.10</artifactId>
<version>${project.version}</version>
</dependency>
+
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-clients_2.10</artifactId>
<version>${project.version}</version>
</dependency>
+
<dependency>
<groupId>org.rocksdb</groupId>
<artifactId>rocksdbjni</artifactId>
<version>4.5.1</version>
</dependency>
+
+ <!-- test dependencies -->
+
<dependency>
<groupId>org.apache.flink</groupId>
- <artifactId>flink-core</artifactId>
+ <artifactId>flink-test-utils-junit</artifactId>
<version>${project.version}</version>
- <type>test-jar</type>
<scope>test</scope>
</dependency>
+
<dependency>
<groupId>org.apache.flink</groupId>
- <artifactId>flink-runtime_2.10</artifactId>
+ <artifactId>flink-streaming-java_2.10</artifactId>
<version>${project.version}</version>
<type>test-jar</type>
<scope>test</scope>
</dependency>
+
<dependency>
<groupId>org.apache.flink</groupId>
- <artifactId>flink-streaming-java_2.10</artifactId>
+ <artifactId>flink-runtime_2.10</artifactId>
<version>${project.version}</version>
<type>test-jar</type>
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>4.11</version>
- </dependency>
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBAsyncKVSnapshotTest.java
----------------------------------------------------------------------
diff --git a/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBAsyncKVSnapshotTest.java b/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBAsyncKVSnapshotTest.java
index efdea30..a58686b 100644
--- a/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBAsyncKVSnapshotTest.java
+++ b/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBAsyncKVSnapshotTest.java
@@ -26,11 +26,11 @@ import org.apache.flink.api.common.typeutils.base.StringSerializer;
import org.apache.flink.api.common.typeutils.base.VoidSerializer;
import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.configuration.ConfigConstants;
+import org.apache.flink.core.testutils.OneShotLatch;
import org.apache.flink.runtime.io.network.api.writer.ResultPartitionWriter;
import org.apache.flink.runtime.operators.testutils.MockInputSplitProvider;
import org.apache.flink.runtime.state.StateHandle;
import org.apache.flink.runtime.state.memory.MemoryStateBackend;
-import org.apache.flink.runtime.taskmanager.OneShotLatch;
import org.apache.flink.streaming.api.graph.StreamConfig;
import org.apache.flink.streaming.api.operators.AbstractStreamOperator;
import org.apache.flink.streaming.api.operators.OneInputStreamOperator;
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-contrib/flink-storm-examples/pom.xml
----------------------------------------------------------------------
diff --git a/flink-contrib/flink-storm-examples/pom.xml b/flink-contrib/flink-storm-examples/pom.xml
index cd8a57f..ec46b94 100644
--- a/flink-contrib/flink-storm-examples/pom.xml
+++ b/flink-contrib/flink-storm-examples/pom.xml
@@ -35,6 +35,9 @@ under the License.
<packaging>jar</packaging>
<dependencies>
+
+ <!-- core dependencies -->
+
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-storm_2.10</artifactId>
@@ -48,28 +51,28 @@ under the License.
</dependency>
<dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-streaming-java_2.10</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
+ <groupId>org.apache.storm</groupId>
+ <artifactId>storm-starter</artifactId>
+ <version>0.9.4</version>
</dependency>
+ <!-- test dependencies -->
+
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-test-utils_2.10</artifactId>
<version>${project.version}</version>
- <type>test-jar</type>
<scope>test</scope>
</dependency>
<dependency>
- <groupId>org.apache.storm</groupId>
- <artifactId>storm-starter</artifactId>
- <version>0.9.4</version>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-streaming-java_2.10</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ <type>test-jar</type>
</dependency>
-
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-contrib/flink-streaming-contrib/pom.xml
----------------------------------------------------------------------
diff --git a/flink-contrib/flink-streaming-contrib/pom.xml b/flink-contrib/flink-streaming-contrib/pom.xml
index 340bac5..4e7e69b 100644
--- a/flink-contrib/flink-streaming-contrib/pom.xml
+++ b/flink-contrib/flink-streaming-contrib/pom.xml
@@ -37,26 +37,33 @@ under the License.
<packaging>jar</packaging>
<dependencies>
+
+ <!-- core dependencies -->
+
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-streaming-java_2.10</artifactId>
<version>${project.version}</version>
</dependency>
+
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-streaming-scala_2.10</artifactId>
<version>${project.version}</version>
</dependency>
+
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-clients_2.10</artifactId>
<version>${project.version}</version>
</dependency>
+
+ <!-- test dependencies -->
+
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-test-utils_2.10</artifactId>
<version>${project.version}</version>
- <type>test-jar</type>
<scope>test</scope>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-core/pom.xml
----------------------------------------------------------------------
diff --git a/flink-core/pom.xml b/flink-core/pom.xml
index 1aadfbd..21d00b3 100644
--- a/flink-core/pom.xml
+++ b/flink-core/pom.xml
@@ -71,8 +71,15 @@ under the License.
<version>${project.version}</version>
</dependency>
- <!-- test depedencies -->
-
+ <!-- test dependencies -->
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-test-utils-junit</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
@@ -95,17 +102,6 @@ under the License.
<build>
<plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <executions>
- <execution>
- <goals>
- <goal>test-jar</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
<!-- Exclude removed CONFIG_KEY from ccheck -->
<plugin>
<groupId>com.github.siom79.japicmp</groupId>
@@ -126,7 +122,21 @@ under the License.
</execution>
</executions>
</plugin>
+
+ <!-- publish some test base classes -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
+
</build>
</project>
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-core/src/main/java/org/apache/flink/api/java/typeutils/InputTypeConfigurable.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/InputTypeConfigurable.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/InputTypeConfigurable.java
index 5958e29..b7fc2e2 100644
--- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/InputTypeConfigurable.java
+++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/InputTypeConfigurable.java
@@ -25,10 +25,8 @@ import org.apache.flink.api.common.typeinfo.TypeInformation;
/**
* {@link org.apache.flink.api.common.io.OutputFormat}s can implement this interface to be configured
- * with the data type they will operate on. The method {@link #setInputType(org.apache.flink.api
- * .common.typeinfo.TypeInformation, org.apache.flink.api.common.ExecutionConfig)} will be
- * called when the output format is used with an output method such as
- * {@link org.apache.flink.api.java.DataSet#output(org.apache.flink.api.common.io.OutputFormat)}.
+ * with the data type they will operate on. The method {@link #setInputType(TypeInformation, ExecutionConfig)
+ * will be called when the output format is used with an output API method.
*/
@Public
public interface InputTypeConfigurable {
@@ -38,7 +36,7 @@ public interface InputTypeConfigurable {
* the DataSet's output method. May be used to configures the output format based on the data type.
*
* @param type The data type of the input.
- * @param executionConfig
+ * @param executionConfig The execution config for this parallel execution.
*/
void setInputType(TypeInformation<?> type, ExecutionConfig executionConfig);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-core/src/main/java/org/apache/flink/util/InstantiationUtil.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/util/InstantiationUtil.java b/flink-core/src/main/java/org/apache/flink/util/InstantiationUtil.java
index bd068f3..bcf71f0 100644
--- a/flink-core/src/main/java/org/apache/flink/util/InstantiationUtil.java
+++ b/flink-core/src/main/java/org/apache/flink/util/InstantiationUtil.java
@@ -21,6 +21,7 @@ package org.apache.flink.util;
import org.apache.flink.annotation.Internal;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.configuration.Configuration;
+import org.apache.flink.core.io.IOReadableWritable;
import org.apache.flink.core.memory.DataInputViewStreamWrapper;
import org.apache.flink.core.memory.DataOutputViewStreamWrapper;
@@ -340,6 +341,35 @@ public final class InstantiationUtil {
return deserializeObject(serializedObject, classLoader);
}
}
+
+ /**
+ * Clones the given writable using the {@link IOReadableWritable serialization}.
+ *
+ * @param original Object to clone
+ * @param <T> Type of the object to clone
+ * @return Cloned object
+ * @throws IOException Thrown is the serialization fails.
+ */
+ public static <T extends IOReadableWritable> T createCopyWritable(T original) throws IOException {
+ if (original == null) {
+ return null;
+ }
+
+ final ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ try (DataOutputViewStreamWrapper out = new DataOutputViewStreamWrapper(baos)) {
+ original.write(out);
+ }
+
+ final ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
+ try (DataInputViewStreamWrapper in = new DataInputViewStreamWrapper(bais)) {
+
+ @SuppressWarnings("unchecked")
+ T copy = (T) instantiate(original.getClass());
+ copy.read(in);
+ return copy;
+ }
+ }
+
// --------------------------------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-core/src/test/java/org/apache/flink/api/common/ExecutionConfigTest.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/api/common/ExecutionConfigTest.java b/flink-core/src/test/java/org/apache/flink/api/common/ExecutionConfigTest.java
index 6dd47d8..1f646a5 100644
--- a/flink-core/src/test/java/org/apache/flink/api/common/ExecutionConfigTest.java
+++ b/flink-core/src/test/java/org/apache/flink/api/common/ExecutionConfigTest.java
@@ -77,17 +77,4 @@ public class ExecutionConfigTest {
throw new RuntimeException("Couldn't create new ExecutionConfig for test.", e);
}
}
-
- /**
- * Deserializes the given ExecutionConfig with the System class loader.
- * @param serializedConfig The serialized ExecutionConfig
- * @return ExecutionConfig
- */
- public static ExecutionConfig deserializeConfig(SerializedValue<ExecutionConfig> serializedConfig) {
- try {
- return serializedConfig.deserializeValue(ExecutionConfigTest.class.getClassLoader());
- } catch (Exception e) {
- throw new RuntimeException("Could not deserialize ExecutionConfig for test.", e);
- }
- }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-core/src/test/java/org/apache/flink/api/common/io/SequentialFormatTestBase.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/api/common/io/SequentialFormatTestBase.java b/flink-core/src/test/java/org/apache/flink/api/common/io/SequentialFormatTestBase.java
deleted file mode 100644
index b00ca95..0000000
--- a/flink-core/src/test/java/org/apache/flink/api/common/io/SequentialFormatTestBase.java
+++ /dev/null
@@ -1,301 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.api.common.io;
-
-import org.apache.flink.api.common.io.statistics.BaseStatistics;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.core.fs.FileInputSplit;
-import org.apache.flink.core.fs.Path;
-import org.apache.flink.core.memory.DataOutputView;
-import org.apache.flink.core.memory.DataOutputViewStreamWrapper;
-import org.apache.flink.util.TestLogger;
-
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runners.Parameterized.Parameters;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Comparator;
-import java.util.List;
-
-/**
- * Test base for {@link org.apache.flink.api.common.io.BinaryInputFormat} and {@link org.apache.flink.api.common.io.BinaryOutputFormat}.
- */
-public abstract class SequentialFormatTestBase<T> extends TestLogger {
-
- public class InputSplitSorter implements Comparator<FileInputSplit> {
- @Override
- public int compare(FileInputSplit o1, FileInputSplit o2) {
- int pathOrder = o1.getPath().getName().compareTo(o2.getPath().getName());
- return pathOrder == 0 ? Long.signum(o1.getStart() - o2.getStart()) : pathOrder;
- }
- }
-
- private int numberOfTuples;
-
- protected long blockSize;
-
- private int parallelism;
-
- private int[] rawDataSizes;
-
- protected File tempFile;
-
- /**
- * Initializes SequentialFormatTest.
- */
- public SequentialFormatTestBase(int numberOfTuples, long blockSize, int parallelism) {
- this.numberOfTuples = numberOfTuples;
- this.blockSize = blockSize;
- this.parallelism = parallelism;
- this.rawDataSizes = new int[parallelism];
- }
-
- /**
- * Count how many bytes would be written if all records were directly serialized
- */
- @Before
- public void calcRawDataSize() throws IOException {
- int recordIndex = 0;
- for (int fileIndex = 0; fileIndex < this.parallelism; fileIndex++) {
- ByteCounter byteCounter = new ByteCounter();
-
- for (int fileCount = 0; fileCount < this.getNumberOfTuplesPerFile(fileIndex); fileCount++, recordIndex++) {
- writeRecord(this.getRecord(recordIndex),
- new DataOutputViewStreamWrapper(byteCounter));
- }
- this.rawDataSizes[fileIndex] = byteCounter.getLength();
- }
- }
-
- /**
- * Checks if the expected input splits were created
- */
- @Test
- public void checkInputSplits() throws IOException {
- FileInputSplit[] inputSplits = this.createInputFormat().createInputSplits(0);
- Arrays.sort(inputSplits, new InputSplitSorter());
-
- int splitIndex = 0;
- for (int fileIndex = 0; fileIndex < this.parallelism; fileIndex++) {
- List<FileInputSplit> sameFileSplits = new ArrayList<FileInputSplit>();
- Path lastPath = inputSplits[splitIndex].getPath();
- for (; splitIndex < inputSplits.length; splitIndex++) {
- if (!inputSplits[splitIndex].getPath().equals(lastPath)) {
- break;
- }
- sameFileSplits.add(inputSplits[splitIndex]);
- }
-
- Assert.assertEquals(this.getExpectedBlockCount(fileIndex), sameFileSplits.size());
-
- long lastBlockLength =
- this.rawDataSizes[fileIndex] % (this.blockSize - getInfoSize()) + getInfoSize();
- for (int index = 0; index < sameFileSplits.size(); index++) {
- Assert.assertEquals(this.blockSize * index, sameFileSplits.get(index).getStart());
- if (index < sameFileSplits.size() - 1) {
- Assert.assertEquals(this.blockSize, sameFileSplits.get(index).getLength());
- }
- }
- Assert.assertEquals(lastBlockLength, sameFileSplits.get(sameFileSplits.size() - 1).getLength());
- }
- }
-
- /**
- * Tests if the expected sequence and amount of data can be read
- */
- @Test
- public void checkRead() throws Exception {
- BinaryInputFormat<T> input = this.createInputFormat();
- FileInputSplit[] inputSplits = input.createInputSplits(0);
- Arrays.sort(inputSplits, new InputSplitSorter());
-
- int readCount = 0;
-
- for (FileInputSplit inputSplit : inputSplits) {
- input.open(inputSplit);
- input.reopen(inputSplit, input.getCurrentState());
-
- T record = createInstance();
-
- while (!input.reachedEnd()) {
- if (input.nextRecord(record) != null) {
- this.checkEquals(this.getRecord(readCount), record);
-
- if (!input.reachedEnd()) {
- Tuple2<Long, Long> state = input.getCurrentState();
-
- input = this.createInputFormat();
- input.reopen(inputSplit, state);
- }
- readCount++;
- }
- }
- }
- Assert.assertEquals(this.numberOfTuples, readCount);
- }
-
- /**
- * Tests the statistics of the given format.
- */
- @Test
- public void checkStatistics() {
- BinaryInputFormat<T> input = this.createInputFormat();
- BaseStatistics statistics = input.getStatistics(null);
- Assert.assertEquals(this.numberOfTuples, statistics.getNumberOfRecords());
- }
-
- @After
- public void cleanup() {
- this.deleteRecursively(this.tempFile);
- }
-
- private void deleteRecursively(File file) {
- if (file.isDirectory()) {
- for (File subFile : file.listFiles()) {
- this.deleteRecursively(subFile);
- }
- } else {
- file.delete();
- }
- }
-
- /**
- * Write out the tuples in a temporary file and return it.
- */
- @Before
- public void writeTuples() throws IOException {
- this.tempFile = File.createTempFile("BinaryInputFormat", null);
- this.tempFile.deleteOnExit();
- Configuration configuration = new Configuration();
- configuration.setLong(BinaryOutputFormat.BLOCK_SIZE_PARAMETER_KEY, this.blockSize);
- if (this.parallelism == 1) {
- BinaryOutputFormat<T> output = createOutputFormat(this.tempFile.toURI().toString(),
- configuration);
- for (int index = 0; index < this.numberOfTuples; index++) {
- output.writeRecord(this.getRecord(index));
- }
- output.close();
- } else {
- this.tempFile.delete();
- this.tempFile.mkdir();
- int recordIndex = 0;
- for (int fileIndex = 0; fileIndex < this.parallelism; fileIndex++) {
- BinaryOutputFormat<T> output = createOutputFormat(this.tempFile.toURI() + "/" +
- (fileIndex+1), configuration);
- for (int fileCount = 0; fileCount < this.getNumberOfTuplesPerFile(fileIndex); fileCount++, recordIndex++) {
- output.writeRecord(this.getRecord(recordIndex));
- }
- output.close();
- }
- }
- }
-
- private int getNumberOfTuplesPerFile(int fileIndex) {
- return this.numberOfTuples / this.parallelism;
- }
-
- /**
- * Tests if the length of the file matches the expected value.
- */
- @Test
- public void checkLength() {
- File[] files = this.tempFile.isDirectory() ? this.tempFile.listFiles() : new File[] { this.tempFile };
- Arrays.sort(files);
- for (int fileIndex = 0; fileIndex < this.parallelism; fileIndex++) {
- long lastBlockLength = this.rawDataSizes[fileIndex] % (this.blockSize - getInfoSize());
- long expectedLength =
- (this.getExpectedBlockCount(fileIndex) - 1) * this.blockSize + getInfoSize() +
- lastBlockLength;
- Assert.assertEquals(expectedLength, files[fileIndex].length());
- }
- }
-
- abstract protected BinaryInputFormat<T> createInputFormat();
-
- abstract protected BinaryOutputFormat<T> createOutputFormat(final String path, final
- Configuration configuration)
- throws IOException;
-
- abstract protected int getInfoSize();
-
- /**
- * Returns the record to write at the given position
- */
- abstract protected T getRecord(int index);
-
- abstract protected T createInstance();
-
- abstract protected void writeRecord(T record, DataOutputView outputView) throws IOException;
-
- /**
- * Checks if both records are equal
- */
- abstract protected void checkEquals(T expected, T actual);
-
- private int getExpectedBlockCount(int fileIndex) {
- int expectedBlockCount =
- (int) Math.ceil((double) this.rawDataSizes[fileIndex] / (this.blockSize -
- getInfoSize()));
- return expectedBlockCount;
- }
-
- @Parameters
- public static List<Object[]> getParameters() {
- ArrayList<Object[]> params = new ArrayList<Object[]>();
- for (int parallelism = 1; parallelism <= 2; parallelism++) {
- // numberOfTuples, blockSize, parallelism
- params.add(new Object[] { 100, BinaryOutputFormat.NATIVE_BLOCK_SIZE, parallelism });
- params.add(new Object[] { 100, 1000, parallelism });
- params.add(new Object[] { 100, 1 << 20, parallelism });
- params.add(new Object[] { 10000, 1000, parallelism });
- params.add(new Object[] { 10000, 1 << 20, parallelism });
- }
- return params;
- }
-
- /**
- * Counts the bytes that would be written.
- *
- */
- private static final class ByteCounter extends OutputStream {
- int length = 0;
-
- /**
- * Returns the length.
- *
- * @return the length
- */
- public int getLength() {
- return this.length;
- }
-
- @Override
- public void write(int b) throws IOException {
- this.length++;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-core/src/test/java/org/apache/flink/api/common/io/SerializedFormatTest.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/api/common/io/SerializedFormatTest.java b/flink-core/src/test/java/org/apache/flink/api/common/io/SerializedFormatTest.java
deleted file mode 100644
index d82623b..0000000
--- a/flink-core/src/test/java/org/apache/flink/api/common/io/SerializedFormatTest.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.api.common.io;
-
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.core.fs.FileSystem;
-import org.apache.flink.core.fs.Path;
-import org.apache.flink.core.memory.DataOutputView;
-import org.apache.flink.types.IntValue;
-import org.apache.flink.types.Record;
-import org.apache.flink.types.StringValue;
-
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-
-import java.io.IOException;
-
-@RunWith(Parameterized.class)
-public class SerializedFormatTest extends SequentialFormatTestBase<Record> {
-
- private BlockInfo info;
-
- public SerializedFormatTest(int numberOfRecords, long blockSize, int parallelism){
- super(numberOfRecords, blockSize, parallelism);
- }
-
- @Before
- public void setup(){
- info = createInputFormat().createBlockInfo();
- }
-
- @Override
- protected BinaryInputFormat<Record> createInputFormat() {
- Configuration configuration = new Configuration();
- configuration.setLong(BinaryInputFormat.BLOCK_SIZE_PARAMETER_KEY, this.blockSize);
-
- final SerializedInputFormat<Record> inputFormat = new SerializedInputFormat<Record>();
- inputFormat.setFilePath(this.tempFile.toURI().toString());
-
- inputFormat.configure(configuration);
- return inputFormat;
- }
-
-
- @Override
- protected BinaryOutputFormat<Record> createOutputFormat(String path, Configuration configuration)
- throws IOException
- {
- final SerializedOutputFormat<Record> outputFormat = new SerializedOutputFormat<Record>();
- outputFormat.setOutputFilePath(new Path(path));
- outputFormat.setWriteMode(FileSystem.WriteMode.OVERWRITE);
-
- configuration = configuration == null ? new Configuration() : configuration;
- outputFormat.configure(configuration);
- outputFormat.open(0, 1);
- return outputFormat;
- }
-
- @Override
- protected int getInfoSize() {
- return info.getInfoSize();
- }
-
- @Override
- protected Record getRecord(int index) {
- return new Record(new IntValue(index), new StringValue(String.valueOf(index)));
- }
-
- @Override
- protected Record createInstance() {
- return new Record();
- }
-
- @Override
- protected void writeRecord(Record record, DataOutputView outputView) throws IOException{
- record.write(outputView);
- }
-
- @Override
- protected void checkEquals(Record expected, Record actual) {
- Assert.assertEquals(expected.getNumFields(), actual.getNumFields());
- Assert.assertEquals(expected.getField(0, IntValue.class), actual.getField(0, IntValue.class));
- Assert.assertEquals(expected.getField(1, StringValue.class), actual.getField(1, StringValue.class));
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-core/src/test/java/org/apache/flink/configuration/ConfigurationTest.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/configuration/ConfigurationTest.java b/flink-core/src/test/java/org/apache/flink/configuration/ConfigurationTest.java
index 5f0947d..cf3c908 100644
--- a/flink-core/src/test/java/org/apache/flink/configuration/ConfigurationTest.java
+++ b/flink-core/src/test/java/org/apache/flink/configuration/ConfigurationTest.java
@@ -23,9 +23,9 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
-import org.apache.flink.core.testutils.CommonTestUtils;
-
+import org.apache.flink.util.InstantiationUtil;
import org.apache.flink.util.TestLogger;
+
import org.junit.Test;
/**
@@ -54,7 +54,7 @@ public class ConfigurationTest extends TestLogger {
orig.setBytes("bytes sequence", new byte[] { 1, 2, 3, 4, 5 } );
orig.setClass("myclass", this.getClass());
- final Configuration copy = CommonTestUtils.createCopyWritable(orig);
+ final Configuration copy = InstantiationUtil.createCopyWritable(orig);
assertEquals("myvalue", copy.getString("mykey", "null"));
assertEquals(100, copy.getInteger("mynumber", 0));
assertEquals(478236947162389746L, copy.getLong("longvalue", 0L));
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-core/src/test/java/org/apache/flink/configuration/FilesystemSchemeConfigTest.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/configuration/FilesystemSchemeConfigTest.java b/flink-core/src/test/java/org/apache/flink/configuration/FilesystemSchemeConfigTest.java
index 8de4a22..3e1a723 100644
--- a/flink-core/src/test/java/org/apache/flink/configuration/FilesystemSchemeConfigTest.java
+++ b/flink-core/src/test/java/org/apache/flink/configuration/FilesystemSchemeConfigTest.java
@@ -31,6 +31,7 @@ import java.io.PrintWriter;
import java.lang.reflect.Field;
import java.net.URI;
import java.net.URISyntaxException;
+import java.util.UUID;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -111,11 +112,11 @@ public class FilesystemSchemeConfigTest {
Field f = FileSystem.class.getDeclaredField("defaultScheme");
f.setAccessible(true);
f.set(null, null);
- } catch (IllegalAccessException e) {
- e.printStackTrace();
- } catch (NoSuchFieldException e) {
+ } catch (IllegalAccessException | NoSuchFieldException e) {
e.printStackTrace();
+ fail("Cannot reset default scheme: " + e.getMessage());
}
+
confFile.delete();
testFile.delete();
tmpDir.delete();
@@ -124,13 +125,14 @@ public class FilesystemSchemeConfigTest {
private File getTmpDir() {
File tmpDir = new File(CommonTestUtils.getTempDir() + File.separator
- + CommonTestUtils.getRandomDirectoryName() + File.separator);
- tmpDir.mkdirs();
+ + UUID.randomUUID().toString() + File.separator);
+
+ assertTrue(tmpDir.mkdirs());
return tmpDir;
}
private File createRandomFile(File path, String suffix) {
- return new File(path.getAbsolutePath() + File.separator + CommonTestUtils.getRandomDirectoryName() + suffix);
+ return new File(path.getAbsolutePath() + File.separator + UUID.randomUUID() + suffix);
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-core/src/test/java/org/apache/flink/configuration/GlobalConfigurationTest.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/configuration/GlobalConfigurationTest.java b/flink-core/src/test/java/org/apache/flink/configuration/GlobalConfigurationTest.java
index 5e1a4d0..ce55d2e 100644
--- a/flink-core/src/test/java/org/apache/flink/configuration/GlobalConfigurationTest.java
+++ b/flink-core/src/test/java/org/apache/flink/configuration/GlobalConfigurationTest.java
@@ -19,12 +19,14 @@
package org.apache.flink.configuration;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.PrintWriter;
import java.lang.reflect.Field;
+import java.util.UUID;
import org.apache.flink.core.testutils.CommonTestUtils;
import org.apache.flink.util.TestLogger;
@@ -211,14 +213,12 @@ public class GlobalConfigurationTest extends TestLogger {
}
private File getTmpDir() {
- File tmpDir = new File(CommonTestUtils.getTempDir() + File.separator
- + CommonTestUtils.getRandomDirectoryName() + File.separator);
- tmpDir.mkdirs();
-
+ File tmpDir = new File(CommonTestUtils.getTempDir(), UUID.randomUUID().toString());
+ assertTrue(tmpDir.mkdirs());
return tmpDir;
}
private File createRandomFile(File path, String suffix) {
- return new File(path.getAbsolutePath() + File.separator + CommonTestUtils.getRandomDirectoryName() + suffix);
+ return new File(path, UUID.randomUUID().toString() + suffix);
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-core/src/test/java/org/apache/flink/core/fs/local/LocalFileSystemTest.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/core/fs/local/LocalFileSystemTest.java b/flink-core/src/test/java/org/apache/flink/core/fs/local/LocalFileSystemTest.java
index d41303b..299524d 100644
--- a/flink-core/src/test/java/org/apache/flink/core/fs/local/LocalFileSystemTest.java
+++ b/flink-core/src/test/java/org/apache/flink/core/fs/local/LocalFileSystemTest.java
@@ -16,9 +16,9 @@
* limitations under the License.
*/
-
package org.apache.flink.core.fs.local;
+import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
@@ -29,13 +29,14 @@ import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.Arrays;
+import java.util.UUID;
import org.apache.flink.core.fs.FSDataInputStream;
import org.apache.flink.core.fs.FSDataOutputStream;
import org.apache.flink.core.fs.FileStatus;
import org.apache.flink.core.fs.Path;
-import org.apache.flink.core.fs.local.LocalFileSystem;
import org.apache.flink.core.testutils.CommonTestUtils;
+
import org.junit.Test;
/**
@@ -49,22 +50,10 @@ public class LocalFileSystemTest {
*/
@Test
public void testLocalFilesystem() {
+ final File tempdir = new File(CommonTestUtils.getTempDir(), UUID.randomUUID().toString());
- File tempdir = new File(CommonTestUtils.getTempDir() + File.separator
- + CommonTestUtils.getRandomDirectoryName() + File.separator);
-
- // we must ensure, that this directory does NOT exist yet
- // probability is extremely low but just to make sure..
- while (tempdir.exists()) {
- tempdir = new File(CommonTestUtils.getTempDir() + File.separator + CommonTestUtils.getRandomDirectoryName()
- + File.separator);
- }
-
- final File testfile1 = new File(tempdir.getAbsolutePath() + File.separator
- + CommonTestUtils.getRandomFilename());
-
- final File testfile2 = new File(tempdir.getAbsolutePath() + File.separator
- + CommonTestUtils.getRandomFilename());
+ final File testfile1 = new File(tempdir, UUID.randomUUID().toString());
+ final File testfile2 = new File(tempdir, UUID.randomUUID().toString());
final Path pathtotestfile1 = new Path(testfile1.toURI().getPath());
final Path pathtotestfile2 = new Path(testfile2.toURI().getPath());
@@ -80,7 +69,7 @@ public class LocalFileSystemTest {
// check that dir is not existent yet
assertFalse(lfs.exists(pathtotmpdir));
- tempdir.mkdirs();
+ assertTrue(tempdir.mkdirs());
// check that local file system recognizes file..
assertTrue(lfs.exists(pathtotmpdir));
@@ -114,7 +103,7 @@ public class LocalFileSystemTest {
// create files.. one ""natively"", one using lfs
final FSDataOutputStream lfsoutput1 = lfs.create(pathtotestfile1, false);
- testfile2.createNewFile();
+ assertTrue(testfile2.createNewFile());
// does lfs create files? does lfs recognize created files?
assertTrue(testfile1.exists());
@@ -125,16 +114,14 @@ public class LocalFileSystemTest {
lfsoutput1.write(testbytes);
lfsoutput1.close();
- assertEquals(testfile1.length(), 5l);
+ assertEquals(testfile1.length(), 5L);
final FileInputStream fisfile1 = new FileInputStream(testfile1);
byte[] testbytestest = new byte[5];
- fisfile1.read(testbytestest);
+ assertEquals(testbytestest.length, fisfile1.read(testbytestest));
fisfile1.close();
-
- // assertEquals is not able to compare arrays properly...
- // assertEquals(testbytes, testbytestest);
- assertTrue(Arrays.equals(testbytes, testbytestest));
+
+ assertArrayEquals(testbytes, testbytestest);
// does lfs see the correct file length?
assertEquals(lfs.getFileStatus(pathtotestfile1).getLen(), testfile1.length());
@@ -171,7 +158,8 @@ public class LocalFileSystemTest {
} catch (IOException e) {
fail(e.getMessage());
- } finally {
+ }
+ finally {
// clean up!
testfile1.delete();
testfile2.delete();
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-core/src/test/java/org/apache/flink/core/testutils/CommonTestUtils.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/core/testutils/CommonTestUtils.java b/flink-core/src/test/java/org/apache/flink/core/testutils/CommonTestUtils.java
deleted file mode 100644
index a330c5b..0000000
--- a/flink-core/src/test/java/org/apache/flink/core/testutils/CommonTestUtils.java
+++ /dev/null
@@ -1,159 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.core.testutils;
-
-import static org.junit.Assert.fail;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.IOException;
-import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
-
-import org.apache.flink.configuration.ConfigConstants;
-import org.apache.flink.configuration.GlobalConfiguration;
-import org.apache.flink.core.io.IOReadableWritable;
-import org.apache.flink.core.memory.DataInputViewStreamWrapper;
-import org.apache.flink.core.memory.DataOutputViewStreamWrapper;
-
-/**
- * This class contains reusable utility methods for unit tests.
- */
-public class CommonTestUtils {
-
- /**
- * Constructs a random filename. The filename is a string of 16 hex characters followed by a <code>.dat</code>
- * prefix.
- *
- * @return the random filename
- */
- public static String getRandomFilename() {
-
- final char[] alphabeth = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f' };
-
- String filename = "";
- for (int i = 0; i < 16; i++) {
- filename += alphabeth[(int) (Math.random() * alphabeth.length)];
- }
-
- return filename + ".dat";
- }
-
- /**
- * Constructs a random directory name. The directory is a string of 16 hex characters
- * prefix.
- *
- * @return the random directory name
- */
- public static String getRandomDirectoryName() {
-
- final char[] alphabeth = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f' };
-
- String filename = "";
- for (int i = 0; i < 16; i++) {
- filename += alphabeth[(int) (Math.random() * alphabeth.length)];
- }
-
- return filename;
- }
-
- /**
- * Reads the path to the directory for temporary files from the configuration and returns it.
- *
- * @return the path to the directory for temporary files
- */
- public static String getTempDir() {
-
- return GlobalConfiguration.getString(ConfigConstants.TASK_MANAGER_TMP_DIR_KEY,
- ConfigConstants.DEFAULT_TASK_MANAGER_TMP_PATH).split(File.pathSeparator)[0];
- }
-
- /**
- * Creates a copy of the given {@link IOReadableWritable} object by an in-memory serialization and subsequent
- * deserialization.
- *
- * @param original
- * the original object to be copied
- * @return the copy of original object created by the original object's serialization/deserialization methods
- * @throws IOException
- * thrown if an error occurs while creating the copy of the object
- */
- @SuppressWarnings("unchecked")
- public static <T extends IOReadableWritable> T createCopyWritable(final T original) throws IOException {
-
- final ByteArrayOutputStream baos = new ByteArrayOutputStream();
- original.write(new DataOutputViewStreamWrapper(baos));
-
- final String className = original.getClass().getName();
- if (className == null) {
- fail("Class name is null");
- }
-
- Class<T> clazz = null;
-
- try {
- clazz = (Class<T>) Class.forName(className);
- } catch (ClassNotFoundException e) {
- fail(e.getMessage());
- }
-
- if (clazz == null) {
- fail("Cannot find class with name " + className);
- }
-
- T copy = null;
- try {
- copy = clazz.newInstance();
- } catch (InstantiationException | IllegalAccessException e) {
- fail(e.getMessage());
- }
-
- if (copy == null) {
- fail("Copy of object of type " + className + " is null");
- }
-
- final ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
- copy.read(new DataInputViewStreamWrapper(bais));
- return copy;
- }
-
- public static <T extends java.io.Serializable> T createCopySerializable(T original) throws IOException {
- if (original == null) {
- throw new IllegalArgumentException();
- }
-
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- ObjectOutputStream oos = new ObjectOutputStream(baos);
- oos.writeObject(original);
- oos.close();
- baos.close();
-
- ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
-
- try (ObjectInputStream ois = new ObjectInputStream(bais)) {
- @SuppressWarnings("unchecked")
- T copy = (T) ois.readObject();
- return copy;
- }
- catch (ClassNotFoundException e) {
- throw new IOException(e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-core/src/test/java/org/apache/flink/util/AbstractIDTest.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/util/AbstractIDTest.java b/flink-core/src/test/java/org/apache/flink/util/AbstractIDTest.java
index ba7d60c..c7fec4a 100644
--- a/flink-core/src/test/java/org/apache/flink/util/AbstractIDTest.java
+++ b/flink-core/src/test/java/org/apache/flink/util/AbstractIDTest.java
@@ -22,7 +22,6 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
-import org.apache.flink.core.testutils.CommonTestUtils;
import org.junit.Test;
/**
@@ -36,7 +35,7 @@ public class AbstractIDTest extends TestLogger {
public void testSerialization() {
final AbstractID origID = new AbstractID();
try {
- final AbstractID copyID = CommonTestUtils.createCopyWritable(origID);
+ final AbstractID copyID = InstantiationUtil.createCopyWritable(origID);
assertEquals(origID.hashCode(), copyID.hashCode());
assertEquals(origID, copyID);
@@ -83,16 +82,16 @@ public class AbstractIDTest extends TestLogger {
AbstractID id10 = new AbstractID(Long.MIN_VALUE, Long.MAX_VALUE);
// test self equality
- assertEquals(0, id1.compareTo(CommonTestUtils.createCopyWritable(id1)));
- assertEquals(0, id2.compareTo(CommonTestUtils.createCopyWritable(id2)));
- assertEquals(0, id3.compareTo(CommonTestUtils.createCopyWritable(id3)));
- assertEquals(0, id4.compareTo(CommonTestUtils.createCopyWritable(id4)));
- assertEquals(0, id5.compareTo(CommonTestUtils.createCopyWritable(id5)));
- assertEquals(0, id6.compareTo(CommonTestUtils.createCopyWritable(id6)));
- assertEquals(0, id7.compareTo(CommonTestUtils.createCopyWritable(id7)));
- assertEquals(0, id8.compareTo(CommonTestUtils.createCopyWritable(id8)));
- assertEquals(0, id9.compareTo(CommonTestUtils.createCopyWritable(id9)));
- assertEquals(0, id10.compareTo(CommonTestUtils.createCopyWritable(id10)));
+ assertEquals(0, id1.compareTo(InstantiationUtil.createCopyWritable(id1)));
+ assertEquals(0, id2.compareTo(InstantiationUtil.createCopyWritable(id2)));
+ assertEquals(0, id3.compareTo(InstantiationUtil.createCopyWritable(id3)));
+ assertEquals(0, id4.compareTo(InstantiationUtil.createCopyWritable(id4)));
+ assertEquals(0, id5.compareTo(InstantiationUtil.createCopyWritable(id5)));
+ assertEquals(0, id6.compareTo(InstantiationUtil.createCopyWritable(id6)));
+ assertEquals(0, id7.compareTo(InstantiationUtil.createCopyWritable(id7)));
+ assertEquals(0, id8.compareTo(InstantiationUtil.createCopyWritable(id8)));
+ assertEquals(0, id9.compareTo(InstantiationUtil.createCopyWritable(id9)));
+ assertEquals(0, id10.compareTo(InstantiationUtil.createCopyWritable(id10)));
// test order
assertCompare(id1, id2, -1);
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-core/src/test/java/org/apache/flink/util/InstantiationUtilTest.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/util/InstantiationUtilTest.java b/flink-core/src/test/java/org/apache/flink/util/InstantiationUtilTest.java
index f9500b0..b369763 100644
--- a/flink-core/src/test/java/org/apache/flink/util/InstantiationUtilTest.java
+++ b/flink-core/src/test/java/org/apache/flink/util/InstantiationUtilTest.java
@@ -20,6 +20,9 @@ package org.apache.flink.util;
import org.apache.flink.api.common.typeutils.base.DoubleValueSerializer;
import org.apache.flink.configuration.Configuration;
+import org.apache.flink.core.io.IOReadableWritable;
+import org.apache.flink.core.memory.DataInputView;
+import org.apache.flink.core.memory.DataOutputView;
import org.apache.flink.types.DoubleValue;
import org.apache.flink.types.StringValue;
import org.apache.flink.types.Value;
@@ -28,6 +31,8 @@ import org.junit.Test;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
+import java.util.Objects;
+import java.util.Random;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
@@ -133,11 +138,20 @@ public class InstantiationUtilTest extends TestLogger {
}
}
+ @Test
+ public void testCopyWritable() throws Exception {
+ WritableType original = new WritableType();
+ WritableType copy = InstantiationUtil.createCopyWritable(original);
+
+ assertTrue(original != copy);
+ assertTrue(original.equals(copy));
+ }
+
// --------------------------------------------------------------------------------------------
private class TestClass {}
- private static class TestException extends IOException{
+ private static class TestException extends IOException {
private static final long serialVersionUID = 1L;
}
@@ -167,4 +181,47 @@ public class InstantiationUtilTest extends TestLogger {
throw new ClassNotFoundException("test exception");
}
}
+
+ private static final class WritableType implements IOReadableWritable {
+
+ private int aInt;
+ private long aLong;
+
+ private WritableType() {
+ Random rnd = new Random();
+ this.aInt = rnd.nextInt();
+ this.aLong = rnd.nextLong();
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(aInt, aLong);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == this) {
+ return true;
+ }
+ else if (obj != null && obj.getClass() == WritableType.class) {
+ WritableType that = (WritableType) obj;
+ return this.aLong == that.aLong && this.aInt == that.aInt;
+ }
+ else {
+ return false;
+ }
+ }
+
+ @Override
+ public void write(DataOutputView out) throws IOException {
+ out.writeInt(aInt);
+ out.writeLong(aLong);
+ }
+
+ @Override
+ public void read(DataInputView in) throws IOException {
+ this.aInt = in.readInt();
+ this.aLong = in.readLong();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-core/src/test/java/org/apache/flink/util/TestLogger.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/util/TestLogger.java b/flink-core/src/test/java/org/apache/flink/util/TestLogger.java
deleted file mode 100644
index bf8d378..0000000
--- a/flink-core/src/test/java/org/apache/flink/util/TestLogger.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.util;
-
-import org.junit.Rule;
-import org.junit.rules.TestRule;
-import org.junit.rules.TestWatcher;
-import org.junit.runner.Description;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Adds automatic test name logging. Every test which wants to log which test is currently
- * executed and why it failed, simply has to extend this class.
- */
-public class TestLogger {
- protected final Logger log = LoggerFactory.getLogger(getClass());
-
- @Rule
- public TestRule watchman = new TestWatcher() {
-
- @Override
- public void starting(Description description) {
- log.info("\n================================================================================"
- + "\nTest {} is running."
- + "\n--------------------------------------------------------------------------------"
- , description);
- }
-
- @Override
- public void succeeded(Description description) {
- log.info("\n--------------------------------------------------------------------------------"
- + "\nTest {} successfully run."
- + "\n================================================================================"
- , description);
- }
-
- @Override
- public void failed(Throwable e, Description description) {
- log.error("\n--------------------------------------------------------------------------------"
- + "\nTest {} failed with:\n{}"
- + "\n================================================================================"
- , description, ExceptionUtils.stringifyException(e));
- }
- };
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-examples/flink-examples-streaming/pom.xml
----------------------------------------------------------------------
diff --git a/flink-examples/flink-examples-streaming/pom.xml b/flink-examples/flink-examples-streaming/pom.xml
index afb0cf5..78760b7 100644
--- a/flink-examples/flink-examples-streaming/pom.xml
+++ b/flink-examples/flink-examples-streaming/pom.xml
@@ -35,6 +35,9 @@ under the License.
<packaging>jar</packaging>
<dependencies>
+
+ <!-- core dependencies -->
+
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-streaming-java_2.10</artifactId>
@@ -65,20 +68,21 @@ under the License.
<version>${project.version}</version>
</dependency>
+ <!-- test dependencies -->
+
<dependency>
<groupId>org.apache.flink</groupId>
- <artifactId>flink-streaming-java_2.10</artifactId>
+ <artifactId>flink-test-utils_2.10</artifactId>
<version>${project.version}</version>
<scope>test</scope>
- <type>test-jar</type>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
- <artifactId>flink-test-utils_2.10</artifactId>
+ <artifactId>flink-streaming-java_2.10</artifactId>
<version>${project.version}</version>
- <type>test-jar</type>
<scope>test</scope>
+ <type>test-jar</type>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-fs-tests/pom.xml
----------------------------------------------------------------------
diff --git a/flink-fs-tests/pom.xml b/flink-fs-tests/pom.xml
index 53cb503..d6ffbb5 100644
--- a/flink-fs-tests/pom.xml
+++ b/flink-fs-tests/pom.xml
@@ -45,13 +45,6 @@ under the License.
<dependency>
<groupId>org.apache.flink</groupId>
- <artifactId>flink-core</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.flink</groupId>
<artifactId>flink-streaming-java_2.10</artifactId>
<version>${project.version}</version>
<scope>test</scope>
@@ -75,16 +68,15 @@ under the License.
<groupId>org.apache.flink</groupId>
<artifactId>flink-test-utils_2.10</artifactId>
<version>${project.version}</version>
- <type>test-jar</type>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
- <artifactId>flink-streaming-java_2.10</artifactId>
+ <artifactId>flink-runtime_2.10</artifactId>
<version>${project.version}</version>
- <type>test-jar</type>
<scope>test</scope>
+ <type>test-jar</type>
</dependency>
<dependency>
@@ -102,6 +94,5 @@ under the License.
<type>test-jar</type>
<version>${hadoop.version}</version><!--$NO-MVN-MAN-VER$-->
</dependency>
-
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-fs-tests/src/test/java/org/apache/flink/hdfstests/FileStateBackendTest.java
----------------------------------------------------------------------
diff --git a/flink-fs-tests/src/test/java/org/apache/flink/hdfstests/FileStateBackendTest.java b/flink-fs-tests/src/test/java/org/apache/flink/hdfstests/FileStateBackendTest.java
index e32c1a6..4ec94f6 100644
--- a/flink-fs-tests/src/test/java/org/apache/flink/hdfstests/FileStateBackendTest.java
+++ b/flink-fs-tests/src/test/java/org/apache/flink/hdfstests/FileStateBackendTest.java
@@ -31,10 +31,10 @@ import org.apache.flink.runtime.state.filesystem.FileStreamStateHandle;
import org.apache.flink.runtime.state.filesystem.FsStateBackend;
import org.apache.flink.runtime.operators.testutils.DummyEnvironment;
import org.apache.flink.api.common.typeutils.base.IntSerializer;
-
import org.apache.flink.runtime.state.AbstractStateBackend;
import org.apache.flink.runtime.state.StreamStateHandle;
import org.apache.flink.runtime.state.memory.ByteStreamStateHandle;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-java/pom.xml
----------------------------------------------------------------------
diff --git a/flink-java/pom.xml b/flink-java/pom.xml
index a3689e7..94021eb 100644
--- a/flink-java/pom.xml
+++ b/flink-java/pom.xml
@@ -65,15 +65,14 @@ under the License.
<version>${guava.version}</version>
<scope>test</scope>
</dependency>
-
+
<dependency>
<groupId>org.apache.flink</groupId>
- <artifactId>flink-core</artifactId>
+ <artifactId>flink-test-utils-junit</artifactId>
<version>${project.version}</version>
- <type>test-jar</type>
<scope>test</scope>
</dependency>
-
+
</dependencies>
<build>
[14/15] flink git commit: [FLINK-3995] [build] flink-test-utils also
contains the streaming test utilities.
Posted by rm...@apache.org.
[FLINK-3995] [build] flink-test-utils also contains the streaming test utilities.
Test utilities include the StreamingMultipleProgramsTestBase and StreamingTestEnvironment.
This moves the ITCases for streaming into 'flink-tests' to achieve that.
This closes #2092
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/b9f42e91
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/b9f42e91
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/b9f42e91
Branch: refs/heads/master
Commit: b9f42e91c9415dd6063079df00c142334b74e636
Parents: 4b71e0e
Author: Stephan Ewen <se...@apache.org>
Authored: Sat Jun 11 02:14:07 2016 +0200
Committer: Robert Metzger <rm...@apache.org>
Committed: Tue Jul 5 15:51:06 2016 +0200
----------------------------------------------------------------------
.../connectors/kafka/Kafka08ITCase.java | 3 +-
.../kafka/testutils/DiscardingSink.java | 33 -
flink-streaming-java/pom.xml | 14 +-
.../api/ChainedRuntimeContextTest.java | 80 --
.../flink/streaming/api/CoStreamITCase.java | 116 ---
.../streaming/api/DataStreamPojoITCase.java | 206 -----
.../flink/streaming/api/DataStreamTest.java | 22 +-
.../apache/flink/streaming/api/IterateTest.java | 700 ---------------
.../flink/streaming/api/OutputSplitterTest.java | 144 ---
.../flink/streaming/api/PartitionerTest.java | 268 ------
.../api/StreamExecutionEnvironmentTest.java | 19 +-
.../streaming/api/StreamingOperatorsITCase.java | 235 -----
.../flink/streaming/api/TypeFillTest.java | 4 +-
.../api/collector/DirectedOutputTest.java | 119 ---
.../api/collector/OutputSelectorTest.java | 1 +
.../api/graph/StreamGraphGeneratorTest.java | 19 +-
.../api/operators/StreamProjectTest.java | 55 +-
.../api/operators/co/CoGroupedReduceTest.java | 125 ---
.../api/operators/co/CoWindowTest.java | 182 ----
.../api/operators/co/SelfConnectionTest.java | 180 ----
.../api/outputformat/CsvOutputFormatITCase.java | 78 --
.../outputformat/SocketOutputFormatITCase.java | 53 --
.../outputformat/TextOutputFormatITCase.java | 55 --
.../api/streamtask/StreamVertexTest.java | 187 ----
.../StreamingJobGraphGeneratorNodeHashTest.java | 14 +-
.../flink/streaming/graph/TranslationTest.java | 3 +-
.../windowing/AllWindowTranslationTest.java | 9 +-
.../operators/windowing/CoGroupJoinITCase.java | 370 --------
.../windowing/TimeWindowTranslationTest.java | 4 +-
.../operators/windowing/WindowFoldITCase.java | 191 ----
.../windowing/WindowTranslationTest.java | 9 +-
.../runtime/state/StateBackendITCase.java | 149 ----
.../runtime/tasks/StreamTaskTimerITCase.java | 338 -------
.../streaming/timestamp/TimestampITCase.java | 875 -------------------
.../apache/flink/streaming/util/NoOpSink.java | 26 -
.../streaming/util/ReceiveCheckNoOpSink.java | 42 -
.../streaming/util/SocketOutputTestBase.java | 106 ---
.../util/StreamingMultipleProgramsTestBase.java | 84 --
.../util/StreamingProgramTestBase.java | 110 ---
.../streaming/util/TestListResultSink.java | 74 --
.../flink/streaming/util/TestListWrapper.java | 64 --
.../streaming/util/TestStreamEnvironment.java | 78 --
flink-streaming-scala/pom.xml | 9 -
.../scala/api/SocketOutputFormatITCase.java | 35 -
.../flink-test-utils/pom.xml | 7 +
.../util/StreamingMultipleProgramsTestBase.java | 85 ++
.../util/StreamingProgramTestBase.java | 110 +++
.../streaming/util/TestStreamEnvironment.java | 78 ++
flink-tests/pom.xml | 2 +-
.../streaming/api/StreamingOperatorsITCase.java | 236 +++++
.../api/outputformat/CsvOutputFormatITCase.java | 78 ++
.../outputformat/TextOutputFormatITCase.java | 55 ++
.../runtime/ChainedRuntimeContextITCase.java | 80 ++
.../streaming/runtime/CoGroupJoinITCase.java | 369 ++++++++
.../test/streaming/runtime/CoStreamITCase.java | 116 +++
.../streaming/runtime/DataStreamPojoITCase.java | 205 +++++
.../streaming/runtime/DirectedOutputITCase.java | 90 ++
.../test/streaming/runtime/IterateITCase.java | 703 +++++++++++++++
.../streaming/runtime/OutputSplitterITCase.java | 144 +++
.../streaming/runtime/PartitionerITCase.java | 269 ++++++
.../streaming/runtime/SelfConnectionITCase.java | 169 ++++
.../streaming/runtime/StateBackendITCase.java | 148 ++++
.../runtime/StreamTaskTimerITCase.java | 339 +++++++
.../test/streaming/runtime/TimestampITCase.java | 875 +++++++++++++++++++
.../streaming/runtime/WindowFoldITCase.java | 190 ++++
.../runtime/util/EvenOddOutputSelector.java | 31 +
.../test/streaming/runtime/util/NoOpIntMap.java | 28 +
.../runtime/util/ReceiveCheckNoOpSink.java | 42 +
.../runtime/util/TestListResultSink.java | 74 ++
.../streaming/runtime/util/TestListWrapper.java | 64 ++
70 files changed, 4637 insertions(+), 5438 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/b9f42e91/flink-streaming-connectors/flink-connector-kafka-0.8/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka08ITCase.java
----------------------------------------------------------------------
diff --git a/flink-streaming-connectors/flink-connector-kafka-0.8/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka08ITCase.java b/flink-streaming-connectors/flink-connector-kafka-0.8/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka08ITCase.java
index b393e5b..36e1151 100644
--- a/flink-streaming-connectors/flink-connector-kafka-0.8/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka08ITCase.java
+++ b/flink-streaming-connectors/flink-connector-kafka-0.8/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka08ITCase.java
@@ -24,10 +24,11 @@ import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.runtime.client.JobCancellationException;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.functions.sink.DiscardingSink;
import org.apache.flink.streaming.connectors.kafka.internals.ZookeeperOffsetHandler;
-import org.apache.flink.streaming.connectors.kafka.testutils.DiscardingSink;
import org.apache.flink.streaming.connectors.kafka.testutils.JobManagerCommunicationUtils;
import org.apache.flink.streaming.util.serialization.SimpleStringSchema;
+
import org.junit.Assert;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/flink/blob/b9f42e91/flink-streaming-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/testutils/DiscardingSink.java
----------------------------------------------------------------------
diff --git a/flink-streaming-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/testutils/DiscardingSink.java b/flink-streaming-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/testutils/DiscardingSink.java
deleted file mode 100644
index 987e6c5..0000000
--- a/flink-streaming-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/testutils/DiscardingSink.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.kafka.testutils;
-
-import org.apache.flink.streaming.api.functions.sink.SinkFunction;
-
-/**
- * Sink function that discards data.
- * @param <T> The type of the function.
- */
-public class DiscardingSink<T> implements SinkFunction<T> {
-
- private static final long serialVersionUID = 2777597566520109843L;
-
- @Override
- public void invoke(T value) {}
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/b9f42e91/flink-streaming-java/pom.xml
----------------------------------------------------------------------
diff --git a/flink-streaming-java/pom.xml b/flink-streaming-java/pom.xml
index 88150b3..790161d 100644
--- a/flink-streaming-java/pom.xml
+++ b/flink-streaming-java/pom.xml
@@ -78,7 +78,7 @@ under the License.
<dependency>
<groupId>org.apache.flink</groupId>
- <artifactId>flink-test-utils_2.10</artifactId>
+ <artifactId>flink-test-utils-junit</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>
@@ -95,18 +95,6 @@ under the License.
<build>
<plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <executions>
- <execution>
- <goals>
- <goal>test-jar</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
-
<!-- disable fork reuse for the streaming project, because of
incorrect declaration of tests -->
<plugin>
http://git-wip-us.apache.org/repos/asf/flink/blob/b9f42e91/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/ChainedRuntimeContextTest.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/ChainedRuntimeContextTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/ChainedRuntimeContextTest.java
deleted file mode 100644
index 68a047c..0000000
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/ChainedRuntimeContextTest.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api;
-
-import static org.junit.Assert.*;
-
-import org.apache.flink.api.common.functions.RichMapFunction;
-import org.apache.flink.api.common.functions.RuntimeContext;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.api.functions.source.RichParallelSourceFunction;
-import org.apache.flink.streaming.util.NoOpSink;
-import org.apache.flink.streaming.util.StreamingMultipleProgramsTestBase;
-
-import org.junit.Test;
-
-@SuppressWarnings("serial")
-public class ChainedRuntimeContextTest extends StreamingMultipleProgramsTestBase {
- private static RuntimeContext srcContext;
- private static RuntimeContext mapContext;
-
- @Test
- public void test() throws Exception {
- StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
- env.setParallelism(1);
-
- env.addSource(new TestSource()).map(new TestMap()).addSink(new NoOpSink<Integer>());
- env.execute();
-
- assertNotEquals(srcContext, mapContext);
-
- }
-
- private static class TestSource extends RichParallelSourceFunction<Integer> {
-
- @Override
- public void run(SourceContext<Integer> ctx) throws Exception {
- }
-
- @Override
- public void cancel() {
- }
-
- @Override
- public void open(Configuration c) {
- srcContext = getRuntimeContext();
- }
-
- }
-
- private static class TestMap extends RichMapFunction<Integer, Integer> {
-
- @Override
- public Integer map(Integer value) throws Exception {
- return value;
- }
-
- @Override
- public void open(Configuration c) {
- mapContext = getRuntimeContext();
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/b9f42e91/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/CoStreamITCase.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/CoStreamITCase.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/CoStreamITCase.java
deleted file mode 100644
index 600e807..0000000
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/CoStreamITCase.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api;
-
-import static org.junit.Assert.assertEquals;
-
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.flink.api.common.functions.FilterFunction;
-import org.apache.flink.api.common.functions.MapFunction;
-import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.api.functions.co.CoFlatMapFunction;
-import org.apache.flink.streaming.util.StreamingMultipleProgramsTestBase;
-import org.apache.flink.streaming.util.TestListResultSink;
-import org.apache.flink.util.Collector;
-
-import org.junit.Test;
-
-@SuppressWarnings("serial")
-public class CoStreamITCase extends StreamingMultipleProgramsTestBase {
-
- @Test
- public void test() throws Exception {
-
- StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
- env.setParallelism(1);
-
- TestListResultSink<String> resultSink = new TestListResultSink<String>();
-
- DataStream<Integer> src = env.fromElements(1, 3, 5);
-
- DataStream<Integer> filter1 = src
- .filter(new FilterFunction<Integer>() {
- @Override
- public boolean filter(Integer value) throws Exception {
- return true;
- }
- })
-
- .keyBy(new KeySelector<Integer, Integer>() {
- @Override
- public Integer getKey(Integer value) throws Exception {
- return value;
- }
- });
-
- DataStream<Tuple2<Integer, Integer>> filter2 = src
- .map(new MapFunction<Integer, Tuple2<Integer, Integer>>() {
-
- @Override
- public Tuple2<Integer, Integer> map(Integer value) throws Exception {
- return new Tuple2<>(value, value + 1);
- }
- })
- .rebalance()
- .filter(new FilterFunction<Tuple2<Integer, Integer>>() {
-
- @Override
- public boolean filter(Tuple2<Integer, Integer> value) throws Exception {
- return true;
- }
- })
- .disableChaining()
- .keyBy(new KeySelector<Tuple2<Integer, Integer>, Integer>() {
-
- @Override
- public Integer getKey(Tuple2<Integer, Integer> value) throws Exception {
- return value.f0;
- }
- });
-
- DataStream<String> connected = filter1.connect(filter2)
- .flatMap(new CoFlatMapFunction<Integer, Tuple2<Integer, Integer>, String>() {
-
- @Override
- public void flatMap1(Integer value, Collector<String> out) throws Exception {
- out.collect(value.toString());
- }
-
- @Override
- public void flatMap2(Tuple2<Integer, Integer> value, Collector<String> out) throws Exception {
- out.collect(value.toString());
- }
- });
-
- connected.addSink(resultSink);
-
-
- env.execute();
-
- List<String> expected = Arrays.asList("(1,2)", "(3,4)", "(5,6)", "1", "3", "5");
- List<String> result = resultSink.getResult();
- Collections.sort(result);
- assertEquals(expected, result);
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/b9f42e91/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/DataStreamPojoITCase.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/DataStreamPojoITCase.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/DataStreamPojoITCase.java
deleted file mode 100644
index 523523b..0000000
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/DataStreamPojoITCase.java
+++ /dev/null
@@ -1,206 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api;
-
-import org.apache.flink.api.common.functions.FlatMapFunction;
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.api.functions.source.SourceFunction;
-import org.apache.flink.streaming.util.StreamingMultipleProgramsTestBase;
-import org.apache.flink.util.Collector;
-import org.junit.Test;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Integration test for streaming programs using POJOs and key selectors
- *
- * See FLINK-3697
- */
-public class DataStreamPojoITCase extends StreamingMultipleProgramsTestBase {
- static List<Data> elements = new ArrayList<>();
- static {
- elements.add(new Data(0,0,0));
- elements.add(new Data(0,0,0));
- elements.add(new Data(1,1,1));
- elements.add(new Data(1,1,1));
- elements.add(new Data(2,2,3));
- elements.add(new Data(2,2,3));
- }
-
- /**
- * Test composite key on the Data POJO (with nested fields)
- */
- @Test
- public void testCompositeKeyOnNestedPojo() throws Exception {
- StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment();
- see.getConfig().disableObjectReuse();
- see.setParallelism(3);
-
- DataStream<Data> dataStream = see.fromCollection(elements);
-
- DataStream<Data> summedStream = dataStream
- .keyBy("aaa", "abc", "wxyz")
- .sum("sum")
- .keyBy("aaa", "abc", "wxyz")
- .flatMap(new FlatMapFunction<Data, Data>() {
- Data[] first = new Data[3];
- @Override
- public void flatMap(Data value, Collector<Data> out) throws Exception {
- if(first[value.aaa] == null) {
- first[value.aaa] = value;
- if(value.sum != 1) {
- throw new RuntimeException("Expected the sum to be one");
- }
- } else {
- if(value.sum != 2) {
- throw new RuntimeException("Expected the sum to be two");
- }
- if(first[value.aaa].aaa != value.aaa) {
- throw new RuntimeException("aaa key wrong");
- }
- if(first[value.aaa].abc != value.abc) {
- throw new RuntimeException("abc key wrong");
- }
- if(first[value.aaa].wxyz != value.wxyz) {
- throw new RuntimeException("wxyz key wrong");
- }
- }
- }
- });
-
- summedStream.print();
-
- see.execute();
- }
-
- /**
- * Test composite & nested key on the Data POJO
- */
- @Test
- public void testNestedKeyOnNestedPojo() throws Exception {
- StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment();
- see.getConfig().disableObjectReuse();
- see.setParallelism(4);
-
- DataStream<Data> dataStream = see.fromCollection(elements);
-
- DataStream<Data> summedStream = dataStream
- .keyBy("aaa", "stats.count")
- .sum("sum")
- .keyBy("aaa", "stats.count")
- .flatMap(new FlatMapFunction<Data, Data>() {
- Data[] first = new Data[3];
- @Override
- public void flatMap(Data value, Collector<Data> out) throws Exception {
- if(value.stats.count != 123) {
- throw new RuntimeException("Wrong value for value.stats.count");
- }
- if(first[value.aaa] == null) {
- first[value.aaa] = value;
- if(value.sum != 1) {
- throw new RuntimeException("Expected the sum to be one");
- }
- } else {
- if(value.sum != 2) {
- throw new RuntimeException("Expected the sum to be two");
- }
- if(first[value.aaa].aaa != value.aaa) {
- throw new RuntimeException("aaa key wrong");
- }
- if(first[value.aaa].abc != value.abc) {
- throw new RuntimeException("abc key wrong");
- }
- if(first[value.aaa].wxyz != value.wxyz) {
- throw new RuntimeException("wxyz key wrong");
- }
- }
- }
- });
-
- summedStream.print();
-
- see.execute();
- }
-
-
- /**
- * As per FLINK-3702 Flink doesn't support nested pojo fields for sum()
- */
- @Test(expected = IllegalArgumentException.class)
- public void testFailOnNestedPojoFieldAccessor() throws Exception {
- StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment();
-
- DataStream<Data> dataStream = see.fromCollection(elements);
- dataStream.keyBy("aaa", "stats.count").sum("stats.count");
- }
-
- public static class Data {
- public int sum; // sum
- public int aaa; // keyBy
- public int abc; //keyBy
- public long wxyz; // keyBy
- public int t1;
- public int t2;
- public Policy policy;
- public Stats stats;
-
- public Data() {
- }
- public Data(int aaa, int abc, int wxyz) {
- this.sum = 1;
- this.aaa = aaa;
- this.abc = abc;
- this.wxyz = wxyz;
- this.stats = new Stats();
- this.stats.count = 123L;
- }
-
- @Override
- public String toString() {
- return "Data{" +
- "sum=" + sum +
- ", aaa=" + aaa +
- ", abc=" + abc +
- ", wxyz=" + wxyz +
- '}';
- }
- }
- public static class Policy {
- public short a;
- public short b;
- public boolean c;
- public boolean d;
-
- public Policy() {}
- }
-
- public static class Stats {
- public long count;
- public float a;
- public float b;
- public float c;
- public float d;
- public float e;
-
- public Stats() {}
- }
-
-}
-
http://git-wip-us.apache.org/repos/asf/flink/blob/b9f42e91/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/DataStreamTest.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/DataStreamTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/DataStreamTest.java
index cf48160..5a86c5c 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/DataStreamTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/DataStreamTest.java
@@ -39,6 +39,7 @@ import org.apache.flink.streaming.api.datastream.SplitStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.co.CoFlatMapFunction;
import org.apache.flink.streaming.api.functions.co.CoMapFunction;
+import org.apache.flink.streaming.api.functions.sink.DiscardingSink;
import org.apache.flink.streaming.api.functions.sink.SinkFunction;
import org.apache.flink.streaming.api.functions.windowing.AllWindowFunction;
import org.apache.flink.streaming.api.graph.StreamEdge;
@@ -57,15 +58,14 @@ import org.apache.flink.streaming.runtime.partitioner.GlobalPartitioner;
import org.apache.flink.streaming.runtime.partitioner.RebalancePartitioner;
import org.apache.flink.streaming.runtime.partitioner.ShufflePartitioner;
import org.apache.flink.streaming.runtime.partitioner.StreamPartitioner;
-import org.apache.flink.streaming.util.NoOpSink;
-import org.apache.flink.streaming.util.StreamingMultipleProgramsTestBase;
import org.apache.flink.util.Collector;
+
import org.junit.Test;
import static org.junit.Assert.*;
@SuppressWarnings("serial")
-public class DataStreamTest extends StreamingMultipleProgramsTestBase {
+public class DataStreamTest {
/**
* Tests union functionality. This ensures that self-unions and unions of streams
@@ -452,7 +452,7 @@ public class DataStreamTest extends StreamingMultipleProgramsTestBase {
}
});
- windowed.addSink(new NoOpSink<Long>());
+ windowed.addSink(new DiscardingSink<Long>());
DataStreamSink<Long> sink = map.addSink(new SinkFunction<Long>() {
private static final long serialVersionUID = 1L;
@@ -486,7 +486,7 @@ public class DataStreamTest extends StreamingMultipleProgramsTestBase {
}
DataStreamSource<Long> parallelSource = env.generateSequence(0, 0);
- parallelSource.addSink(new NoOpSink<Long>());
+ parallelSource.addSink(new DiscardingSink<Long>());
assertEquals(7, env.getStreamGraph().getStreamNode(parallelSource.getId()).getParallelism());
parallelSource.setParallelism(3);
@@ -557,7 +557,7 @@ public class DataStreamTest extends StreamingMultipleProgramsTestBase {
}
};
DataStream<Integer> map = src.map(mapFunction);
- map.addSink(new NoOpSink<Integer>());
+ map.addSink(new DiscardingSink<Integer>());
assertEquals(mapFunction, getFunctionForDataStream(map));
@@ -569,7 +569,7 @@ public class DataStreamTest extends StreamingMultipleProgramsTestBase {
}
};
DataStream<Integer> flatMap = src.flatMap(flatMapFunction);
- flatMap.addSink(new NoOpSink<Integer>());
+ flatMap.addSink(new DiscardingSink<Integer>());
assertEquals(flatMapFunction, getFunctionForDataStream(flatMap));
FilterFunction<Integer> filterFunction = new FilterFunction<Integer>() {
@@ -582,7 +582,7 @@ public class DataStreamTest extends StreamingMultipleProgramsTestBase {
DataStream<Integer> unionFilter = map.union(flatMap)
.filter(filterFunction);
- unionFilter.addSink(new NoOpSink<Integer>());
+ unionFilter.addSink(new DiscardingSink<Integer>());
assertEquals(filterFunction, getFunctionForDataStream(unionFilter));
@@ -606,7 +606,7 @@ public class DataStreamTest extends StreamingMultipleProgramsTestBase {
};
SplitStream<Integer> split = unionFilter.split(outputSelector);
- split.select("dummy").addSink(new NoOpSink<Integer>());
+ split.select("dummy").addSink(new DiscardingSink<Integer>());
List<OutputSelector<?>> outputSelectors = env.getStreamGraph().getStreamNode(unionFilter.getId()).getOutputSelectors();
assertEquals(1, outputSelectors.size());
assertEquals(outputSelector, outputSelectors.get(0));
@@ -632,7 +632,7 @@ public class DataStreamTest extends StreamingMultipleProgramsTestBase {
}
};
DataStream<String> coMap = connect.map(coMapper);
- coMap.addSink(new NoOpSink<String>());
+ coMap.addSink(new DiscardingSink<String>());
assertEquals(coMapper, getFunctionForDataStream(coMap));
try {
@@ -772,7 +772,7 @@ public class DataStreamTest extends StreamingMultipleProgramsTestBase {
return null;
}
});
- coMap.addSink(new NoOpSink());
+ coMap.addSink(new DiscardingSink());
return coMap.getId();
}
http://git-wip-us.apache.org/repos/asf/flink/blob/b9f42e91/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/IterateTest.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/IterateTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/IterateTest.java
deleted file mode 100644
index c6875dd..0000000
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/IterateTest.java
+++ /dev/null
@@ -1,700 +0,0 @@
-/*
-* Licensed to the Apache Software Foundation (ASF) under one or more
-* contributor license agreements. See the NOTICE file distributed with
-* this work for additional information regarding copyright ownership.
-* The ASF licenses this file to You under the Apache License, Version 2.0
-* (the "License"); you may not use this file except in compliance with
-* the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.flink.streaming.api;
-
-import org.apache.flink.api.common.InvalidProgramException;
-import org.apache.flink.api.common.functions.MapFunction;
-import org.apache.flink.api.common.functions.RichFlatMapFunction;
-import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.runtime.jobgraph.JobGraph;
-import org.apache.flink.runtime.jobgraph.JobVertex;
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.datastream.DataStreamSink;
-import org.apache.flink.streaming.api.datastream.IterativeStream;
-import org.apache.flink.streaming.api.datastream.IterativeStream.ConnectedIterativeStreams;
-import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
-import org.apache.flink.streaming.api.datastream.SplitStream;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.api.functions.co.CoMapFunction;
-import org.apache.flink.streaming.api.functions.co.RichCoFlatMapFunction;
-import org.apache.flink.streaming.api.functions.sink.SinkFunction;
-import org.apache.flink.streaming.api.graph.StreamEdge;
-import org.apache.flink.streaming.api.graph.StreamGraph;
-import org.apache.flink.streaming.api.graph.StreamNode;
-import org.apache.flink.streaming.runtime.partitioner.BroadcastPartitioner;
-import org.apache.flink.streaming.runtime.partitioner.ForwardPartitioner;
-import org.apache.flink.streaming.runtime.partitioner.RebalancePartitioner;
-import org.apache.flink.streaming.runtime.partitioner.ShufflePartitioner;
-import org.apache.flink.streaming.util.EvenOddOutputSelector;
-import org.apache.flink.streaming.util.NoOpIntMap;
-import org.apache.flink.streaming.util.ReceiveCheckNoOpSink;
-import org.apache.flink.streaming.util.StreamingMultipleProgramsTestBase;
-import org.apache.flink.util.Collector;
-import org.apache.flink.util.MathUtils;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-@SuppressWarnings({ "unchecked", "unused", "serial" })
-public class IterateTest extends StreamingMultipleProgramsTestBase {
-
- private static final Logger LOG = LoggerFactory.getLogger(IterateTest.class);
-
- private static boolean iterated[];
-
- @Test(expected = UnsupportedOperationException.class)
- public void testIncorrectParallelism() throws Exception {
-
- StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
-
- DataStream<Integer> source = env.fromElements(1, 10);
-
- IterativeStream<Integer> iter1 = source.iterate();
- SingleOutputStreamOperator<Integer> map1 = iter1.map(NoOpIntMap);
- iter1.closeWith(map1).print();
- }
-
- @Test
- public void testDoubleClosing() throws Exception {
-
- StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
-
- // introduce dummy mapper to get to correct parallelism
- DataStream<Integer> source = env.fromElements(1, 10).map(NoOpIntMap);
-
- IterativeStream<Integer> iter1 = source.iterate();
-
- iter1.closeWith(iter1.map(NoOpIntMap));
- iter1.closeWith(iter1.map(NoOpIntMap));
- }
-
-
- @Test(expected = UnsupportedOperationException.class)
- public void testDifferingParallelism() throws Exception {
-
- StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
-
- // introduce dummy mapper to get to correct parallelism
- DataStream<Integer> source = env.fromElements(1, 10)
- .map(NoOpIntMap);
-
- IterativeStream<Integer> iter1 = source.iterate();
-
-
- iter1.closeWith(iter1.map(NoOpIntMap).setParallelism(DEFAULT_PARALLELISM / 2));
-
- }
-
-
- @Test(expected = UnsupportedOperationException.class)
- public void testCoDifferingParallelism() throws Exception {
-
- StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
-
- // introduce dummy mapper to get to correct parallelism
- DataStream<Integer> source = env.fromElements(1, 10).map(NoOpIntMap);
-
- ConnectedIterativeStreams<Integer, Integer> coIter = source.iterate().withFeedbackType(
- Integer.class);
-
-
- coIter.closeWith(coIter.map(NoOpIntCoMap).setParallelism(DEFAULT_PARALLELISM / 2));
-
- }
-
- @Test(expected = UnsupportedOperationException.class)
- public void testClosingFromOutOfLoop() throws Exception {
-
- // this test verifies that we cannot close an iteration with a DataStream that does not
- // have the iteration in its predecessors
-
- StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
-
- // introduce dummy mapper to get to correct parallelism
- DataStream<Integer> source = env.fromElements(1, 10).map(NoOpIntMap);
-
- IterativeStream<Integer> iter1 = source.iterate();
- IterativeStream<Integer> iter2 = source.iterate();
-
-
- iter2.closeWith(iter1.map(NoOpIntMap));
-
- }
-
- @Test(expected = UnsupportedOperationException.class)
- public void testCoIterClosingFromOutOfLoop() throws Exception {
-
- // this test verifies that we cannot close an iteration with a DataStream that does not
- // have the iteration in its predecessors
-
- StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
-
- // introduce dummy mapper to get to correct parallelism
- DataStream<Integer> source = env.fromElements(1, 10).map(NoOpIntMap);
-
- IterativeStream<Integer> iter1 = source.iterate();
- ConnectedIterativeStreams<Integer, Integer> coIter = source.iterate().withFeedbackType(
- Integer.class);
-
-
- coIter.closeWith(iter1.map(NoOpIntMap));
-
- }
-
- @Test(expected = IllegalStateException.class)
- public void testExecutionWithEmptyIteration() throws Exception {
-
- StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
-
- DataStream<Integer> source = env.fromElements(1, 10).map(NoOpIntMap);
-
- IterativeStream<Integer> iter1 = source.iterate();
-
- iter1.map(NoOpIntMap).print();
-
- env.execute();
- }
-
- @Test
- public void testImmutabilityWithCoiteration() {
- StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
-
- DataStream<Integer> source = env.fromElements(1, 10).map(NoOpIntMap); // for rebalance
-
- IterativeStream<Integer> iter1 = source.iterate();
- // Calling withFeedbackType should create a new iteration
- ConnectedIterativeStreams<Integer, String> iter2 = iter1.withFeedbackType(String.class);
-
- iter1.closeWith(iter1.map(NoOpIntMap)).print();
- iter2.closeWith(iter2.map(NoOpCoMap)).print();
-
- StreamGraph graph = env.getStreamGraph();
-
- assertEquals(2, graph.getIterationSourceSinkPairs().size());
-
- for (Tuple2<StreamNode, StreamNode> sourceSinkPair: graph.getIterationSourceSinkPairs()) {
- assertEquals(sourceSinkPair.f0.getOutEdges().get(0).getTargetVertex(), sourceSinkPair.f1.getInEdges().get(0).getSourceVertex());
- }
- }
-
- @Test
- public void testmultipleHeadsTailsSimple() {
- StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
-
- DataStream<Integer> source1 = env.fromElements(1, 2, 3, 4, 5)
- .shuffle()
- .map(NoOpIntMap).name("ParallelizeMapShuffle");
- DataStream<Integer> source2 = env.fromElements(1, 2, 3, 4, 5)
- .map(NoOpIntMap).name("ParallelizeMapRebalance");
-
- IterativeStream<Integer> iter1 = source1.union(source2).iterate();
-
- DataStream<Integer> head1 = iter1.map(NoOpIntMap).name("IterRebalanceMap").setParallelism(DEFAULT_PARALLELISM / 2);
- DataStream<Integer> head2 = iter1.map(NoOpIntMap).name("IterForwardMap");
- DataStreamSink<Integer> head3 = iter1.map(NoOpIntMap).setParallelism(DEFAULT_PARALLELISM / 2).addSink(new ReceiveCheckNoOpSink<Integer>());
- DataStreamSink<Integer> head4 = iter1.map(NoOpIntMap).addSink(new ReceiveCheckNoOpSink<Integer>());
-
- SplitStream<Integer> source3 = env.fromElements(1, 2, 3, 4, 5)
- .map(NoOpIntMap).name("EvenOddSourceMap")
- .split(new EvenOddOutputSelector());
-
- iter1.closeWith(source3.select("even").union(
- head1.rebalance().map(NoOpIntMap).broadcast(), head2.shuffle()));
-
- StreamGraph graph = env.getStreamGraph();
-
- JobGraph jg = graph.getJobGraph();
-
- assertEquals(1, graph.getIterationSourceSinkPairs().size());
-
- Tuple2<StreamNode, StreamNode> sourceSinkPair = graph.getIterationSourceSinkPairs().iterator().next();
- StreamNode itSource = sourceSinkPair.f0;
- StreamNode itSink = sourceSinkPair.f1;
-
- assertEquals(4, itSource.getOutEdges().size());
- assertEquals(3, itSink.getInEdges().size());
-
- assertEquals(itSource.getParallelism(), itSink.getParallelism());
-
- for (StreamEdge edge : itSource.getOutEdges()) {
- if (edge.getTargetVertex().getOperatorName().equals("IterRebalanceMap")) {
- assertTrue(edge.getPartitioner() instanceof RebalancePartitioner);
- } else if (edge.getTargetVertex().getOperatorName().equals("IterForwardMap")) {
- assertTrue(edge.getPartitioner() instanceof ForwardPartitioner);
- }
- }
- for (StreamEdge edge : itSink.getInEdges()) {
- if (graph.getStreamNode(edge.getSourceId()).getOperatorName().equals("ParallelizeMapShuffle")) {
- assertTrue(edge.getPartitioner() instanceof ShufflePartitioner);
- }
-
- if (graph.getStreamNode(edge.getSourceId()).getOperatorName().equals("ParallelizeMapForward")) {
- assertTrue(edge.getPartitioner() instanceof ForwardPartitioner);
- }
-
- if (graph.getStreamNode(edge.getSourceId()).getOperatorName().equals("EvenOddSourceMap")) {
- assertTrue(edge.getPartitioner() instanceof ForwardPartitioner);
- assertTrue(edge.getSelectedNames().contains("even"));
- }
- }
-
- // Test co-location
-
- JobVertex itSource1 = null;
- JobVertex itSink1 = null;
-
- for (JobVertex vertex : jg.getVertices()) {
- if (vertex.getName().contains("IterationSource")) {
- itSource1 = vertex;
- } else if (vertex.getName().contains("IterationSink")) {
-
- itSink1 = vertex;
-
- }
- }
-
- assertTrue(itSource1.getCoLocationGroup() != null);
- assertEquals(itSource1.getCoLocationGroup(), itSink1.getCoLocationGroup());
- }
-
- @Test
- public void testmultipleHeadsTailsWithTailPartitioning() {
- StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
-
- DataStream<Integer> source1 = env.fromElements(1, 2, 3, 4, 5)
- .shuffle()
- .map(NoOpIntMap);
-
- DataStream<Integer> source2 = env.fromElements(1, 2, 3, 4, 5)
- .map(NoOpIntMap);
-
- IterativeStream<Integer> iter1 = source1.union(source2).iterate();
-
- DataStream<Integer> head1 = iter1.map(NoOpIntMap).name("map1");
- DataStream<Integer> head2 = iter1.map(NoOpIntMap)
- .setParallelism(DEFAULT_PARALLELISM / 2)
- .name("shuffle").rebalance();
- DataStreamSink<Integer> head3 = iter1.map(NoOpIntMap).setParallelism(DEFAULT_PARALLELISM / 2)
- .addSink(new ReceiveCheckNoOpSink<Integer>());
- DataStreamSink<Integer> head4 = iter1.map(NoOpIntMap).addSink(new ReceiveCheckNoOpSink<Integer>());
-
- SplitStream<Integer> source3 = env.fromElements(1, 2, 3, 4, 5)
- .map(NoOpIntMap)
- .name("split")
- .split(new EvenOddOutputSelector());
-
- iter1.closeWith(
- source3.select("even").union(
- head1.map(NoOpIntMap).name("bc").broadcast(),
- head2.map(NoOpIntMap).shuffle()));
-
- StreamGraph graph = env.getStreamGraph();
-
- JobGraph jg = graph.getJobGraph();
-
- assertEquals(1, graph.getIterationSourceSinkPairs().size());
-
- Tuple2<StreamNode, StreamNode> sourceSinkPair = graph.getIterationSourceSinkPairs().iterator().next();
- StreamNode itSource = sourceSinkPair.f0;
- StreamNode itSink = sourceSinkPair.f1;
-
- assertEquals(4, itSource.getOutEdges().size());
- assertEquals(3, itSink.getInEdges().size());
-
-
- assertEquals(itSource.getParallelism(), itSink.getParallelism());
-
- for (StreamEdge edge : itSource.getOutEdges()) {
- if (edge.getTargetVertex().getOperatorName().equals("map1")) {
- assertTrue(edge.getPartitioner() instanceof ForwardPartitioner);
- assertEquals(4, edge.getTargetVertex().getParallelism());
- } else if (edge.getTargetVertex().getOperatorName().equals("shuffle")) {
- assertTrue(edge.getPartitioner() instanceof RebalancePartitioner);
- assertEquals(2, edge.getTargetVertex().getParallelism());
- }
- }
- for (StreamEdge edge : itSink.getInEdges()) {
- String tailName = edge.getSourceVertex().getOperatorName();
- if (tailName.equals("split")) {
- assertTrue(edge.getPartitioner() instanceof ForwardPartitioner);
- assertTrue(edge.getSelectedNames().contains("even"));
- } else if (tailName.equals("bc")) {
- assertTrue(edge.getPartitioner() instanceof BroadcastPartitioner);
- } else if (tailName.equals("shuffle")) {
- assertTrue(edge.getPartitioner() instanceof ShufflePartitioner);
- }
- }
-
- // Test co-location
-
- JobVertex itSource1 = null;
- JobVertex itSink1 = null;
-
- for (JobVertex vertex : jg.getVertices()) {
- if (vertex.getName().contains("IterationSource")) {
- itSource1 = vertex;
- } else if (vertex.getName().contains("IterationSink")) {
- itSink1 = vertex;
- }
- }
-
- assertTrue(itSource1.getCoLocationGroup() != null);
- assertTrue(itSink1.getCoLocationGroup() != null);
-
- assertEquals(itSource1.getCoLocationGroup(), itSink1.getCoLocationGroup());
- }
-
- @SuppressWarnings("rawtypes")
- @Test
- public void testSimpleIteration() throws Exception {
- int numRetries = 5;
- int timeoutScale = 1;
-
- for (int numRetry = 0; numRetry < numRetries; numRetry++) {
- try {
- StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
- iterated = new boolean[DEFAULT_PARALLELISM];
-
- DataStream<Boolean> source = env.fromCollection(Collections.nCopies(DEFAULT_PARALLELISM * 2, false))
- .map(NoOpBoolMap).name("ParallelizeMap");
-
- IterativeStream<Boolean> iteration = source.iterate(3000 * timeoutScale);
-
- DataStream<Boolean> increment = iteration.flatMap(new IterationHead()).map(NoOpBoolMap);
-
- iteration.map(NoOpBoolMap).addSink(new ReceiveCheckNoOpSink());
-
- iteration.closeWith(increment).addSink(new ReceiveCheckNoOpSink());
-
- env.execute();
-
- for (boolean iter : iterated) {
- assertTrue(iter);
- }
-
- break; // success
- } catch (Throwable t) {
- LOG.info("Run " + (numRetry + 1) + "/" + numRetries + " failed", t);
-
- if (numRetry >= numRetries - 1) {
- throw t;
- } else {
- timeoutScale *= 2;
- }
- }
- }
- }
-
- @Test
- public void testCoIteration() throws Exception {
- int numRetries = 5;
- int timeoutScale = 1;
-
- for (int numRetry = 0; numRetry < numRetries; numRetry++) {
- try {
- TestSink.collected = new ArrayList<>();
-
- StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
- env.setParallelism(2);
-
- DataStream<String> otherSource = env.fromElements("1000", "2000")
- .map(NoOpStrMap).name("ParallelizeMap");
-
-
- ConnectedIterativeStreams<Integer, String> coIt = env.fromElements(0, 0)
- .map(NoOpIntMap).name("ParallelizeMap")
- .iterate(2000 * timeoutScale)
- .withFeedbackType("String");
-
- try {
- coIt.keyBy(1, 2);
- fail();
- } catch (InvalidProgramException e) {
- // this is expected
- }
-
- DataStream<String> head = coIt
- .flatMap(new RichCoFlatMapFunction<Integer, String, String>() {
-
- private static final long serialVersionUID = 1L;
- boolean seenFromSource = false;
-
- @Override
- public void flatMap1(Integer value, Collector<String> out) throws Exception {
- out.collect(((Integer) (value + 1)).toString());
- }
-
- @Override
- public void flatMap2(String value, Collector<String> out) throws Exception {
- Integer intVal = Integer.valueOf(value);
- if (intVal < 2) {
- out.collect(((Integer) (intVal + 1)).toString());
- }
- if (intVal == 1000 || intVal == 2000) {
- seenFromSource = true;
- }
- }
-
- @Override
- public void close() {
- assertTrue(seenFromSource);
- }
- });
-
- coIt.map(new CoMapFunction<Integer, String, String>() {
-
- @Override
- public String map1(Integer value) throws Exception {
- return value.toString();
- }
-
- @Override
- public String map2(String value) throws Exception {
- return value;
- }
- }).addSink(new ReceiveCheckNoOpSink<String>());
-
- coIt.closeWith(head.broadcast().union(otherSource));
-
- head.addSink(new TestSink()).setParallelism(1);
-
- assertEquals(1, env.getStreamGraph().getIterationSourceSinkPairs().size());
-
- env.execute();
-
- Collections.sort(TestSink.collected);
- assertEquals(Arrays.asList("1", "1", "2", "2", "2", "2"), TestSink.collected);
-
- break; // success
- } catch (Throwable t) {
- LOG.info("Run " + (numRetry + 1) + "/" + numRetries + " failed", t);
-
- if (numRetry >= numRetries - 1) {
- throw t;
- } else {
- timeoutScale *= 2;
- }
- }
- }
- }
-
- /**
- * This test relies on the hash function used by the {@link DataStream#keyBy}, which is
- * assumed to be {@link MathUtils#murmurHash}.
- *
- * For the test to pass all FlatMappers must see at least two records in the iteration,
- * which can only be achieved if the hashed values of the input keys map to a complete
- * congruence system. Given that the test is designed for 3 parallel FlatMapper instances
- * keys chosen from the [1,3] range are a suitable choice.
- */
- @Test
- public void testGroupByFeedback() throws Exception {
- int numRetries = 5;
- int timeoutScale = 1;
-
- for (int numRetry = 0; numRetry < numRetries; numRetry++) {
- try {
- StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
- env.setParallelism(DEFAULT_PARALLELISM - 1);
-
- KeySelector<Integer, Integer> key = new KeySelector<Integer, Integer>() {
-
- @Override
- public Integer getKey(Integer value) throws Exception {
- return value % 3;
- }
- };
-
- DataStream<Integer> source = env.fromElements(1, 2, 3)
- .map(NoOpIntMap).name("ParallelizeMap");
-
- IterativeStream<Integer> it = source.keyBy(key).iterate(3000 * timeoutScale);
-
- DataStream<Integer> head = it.flatMap(new RichFlatMapFunction<Integer, Integer>() {
-
- int received = 0;
- int key = -1;
-
- @Override
- public void flatMap(Integer value, Collector<Integer> out) throws Exception {
- received++;
- if (key == -1) {
- key = MathUtils.murmurHash(value % 3) % 3;
- } else {
- assertEquals(key, MathUtils.murmurHash(value % 3) % 3);
- }
- if (value > 0) {
- out.collect(value - 1);
- }
- }
-
- @Override
- public void close() {
- assertTrue(received > 1);
- }
- });
-
- it.closeWith(head.keyBy(key).union(head.map(NoOpIntMap).keyBy(key))).addSink(new ReceiveCheckNoOpSink<Integer>());
-
- env.execute();
-
- break; // success
- } catch (Throwable t) {
- LOG.info("Run " + (numRetry + 1) + "/" + numRetries + " failed", t);
-
- if (numRetry >= numRetries - 1) {
- throw t;
- } else {
- timeoutScale *= 2;
- }
- }
- }
- }
-
- @SuppressWarnings("deprecation")
- @Test
- public void testWithCheckPointing() throws Exception {
- int numRetries = 5;
- int timeoutScale = 1;
-
- for (int numRetry = 0; numRetry < numRetries; numRetry++) {
- try {
- StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
-
- env.enableCheckpointing();
-
- DataStream<Boolean> source = env.fromCollection(Collections.nCopies(DEFAULT_PARALLELISM * 2, false))
- .map(NoOpBoolMap).name("ParallelizeMap");
-
-
- IterativeStream<Boolean> iteration = source.iterate(3000 * timeoutScale);
-
- iteration.closeWith(iteration.flatMap(new IterationHead())).addSink(new ReceiveCheckNoOpSink<Boolean>());
-
- try {
- env.execute();
-
- // this statement should never be reached
- fail();
- } catch (UnsupportedOperationException e) {
- // expected behaviour
- }
-
- // Test force checkpointing
-
- try {
- env.enableCheckpointing(1, CheckpointingMode.EXACTLY_ONCE, false);
- env.execute();
-
- // this statement should never be reached
- fail();
- } catch (UnsupportedOperationException e) {
- // expected behaviour
- }
-
- env.enableCheckpointing(1, CheckpointingMode.EXACTLY_ONCE, true);
- env.getStreamGraph().getJobGraph();
-
- break; // success
- } catch (Throwable t) {
- LOG.info("Run " + (numRetry + 1) + "/" + numRetries + " failed", t);
-
- if (numRetry >= numRetries - 1) {
- throw t;
- } else {
- timeoutScale *= 2;
- }
- }
- }
- }
-
- public static final class IterationHead extends RichFlatMapFunction<Boolean, Boolean> {
- public void flatMap(Boolean value, Collector<Boolean> out) throws Exception {
- int indx = getRuntimeContext().getIndexOfThisSubtask();
- if (value) {
- iterated[indx] = true;
- } else {
- out.collect(true);
- }
- }
- }
-
- public static CoMapFunction<Integer, String, String> NoOpCoMap = new CoMapFunction<Integer, String, String>() {
-
- public String map1(Integer value) throws Exception {
- return value.toString();
- }
-
- public String map2(String value) throws Exception {
- return value;
- }
- };
-
- public static MapFunction<Integer, Integer> NoOpIntMap = new NoOpIntMap();
-
- public static MapFunction<String, String> NoOpStrMap = new MapFunction<String, String>() {
-
- public String map(String value) throws Exception {
- return value;
- }
-
- };
-
- public static CoMapFunction<Integer, Integer, Integer> NoOpIntCoMap = new CoMapFunction<Integer, Integer, Integer>() {
-
- public Integer map1(Integer value) throws Exception {
- return value;
- }
-
- public Integer map2(Integer value) throws Exception {
- return value;
- }
-
- };
-
- public static MapFunction<Boolean, Boolean> NoOpBoolMap = new MapFunction<Boolean, Boolean>() {
-
- public Boolean map(Boolean value) throws Exception {
- return value;
- }
-
- };
-
- public static class TestSink implements SinkFunction<String> {
-
- private static final long serialVersionUID = 1L;
- public static List<String> collected = new ArrayList<String>();
-
- @Override
- public void invoke(String value) throws Exception {
- collected.add(value);
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/b9f42e91/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/OutputSplitterTest.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/OutputSplitterTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/OutputSplitterTest.java
deleted file mode 100644
index 5126d11..0000000
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/OutputSplitterTest.java
+++ /dev/null
@@ -1,144 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api;
-
-import static org.junit.Assert.assertEquals;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.flink.streaming.api.collector.selector.OutputSelector;
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.util.StreamingMultipleProgramsTestBase;
-import org.apache.flink.streaming.util.TestListResultSink;
-
-import org.junit.Test;
-
-public class OutputSplitterTest extends StreamingMultipleProgramsTestBase {
-
- private static ArrayList<Integer> expectedSplitterResult = new ArrayList<Integer>();
-
- @SuppressWarnings("unchecked")
- @Test
- public void testOnMergedDataStream() throws Exception {
- TestListResultSink<Integer> splitterResultSink1 = new TestListResultSink<Integer>();
- TestListResultSink<Integer> splitterResultSink2 = new TestListResultSink<Integer>();
-
- StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
- env.setParallelism(1);
- env.setBufferTimeout(1);
-
- DataStream<Integer> d1 = env.fromElements(0, 2, 4, 6, 8);
- DataStream<Integer> d2 = env.fromElements(1, 3, 5, 7, 9);
-
- d1 = d1.union(d2);
-
- d1.split(new OutputSelector<Integer>() {
- private static final long serialVersionUID = 8354166915727490130L;
-
- @Override
- public Iterable<String> select(Integer value) {
- List<String> s = new ArrayList<String>();
- if (value > 4) {
- s.add(">");
- } else {
- s.add("<");
- }
- return s;
- }
- }).select(">").addSink(splitterResultSink1);
-
- d1.split(new OutputSelector<Integer>() {
- private static final long serialVersionUID = -6822487543355994807L;
-
- @Override
- public Iterable<String> select(Integer value) {
- List<String> s = new ArrayList<String>();
- if (value % 3 == 0) {
- s.add("yes");
- } else {
- s.add("no");
- }
- return s;
- }
- }).select("yes").addSink(splitterResultSink2);
- env.execute();
-
- expectedSplitterResult.clear();
- expectedSplitterResult.addAll(Arrays.asList(5, 6, 7, 8, 9));
- assertEquals(expectedSplitterResult, splitterResultSink1.getSortedResult());
-
- expectedSplitterResult.clear();
- expectedSplitterResult.addAll(Arrays.asList(0, 3, 6, 9));
- assertEquals(expectedSplitterResult, splitterResultSink2.getSortedResult());
- }
-
- @Test
- public void testOnSingleDataStream() throws Exception {
- TestListResultSink<Integer> splitterResultSink1 = new TestListResultSink<Integer>();
- TestListResultSink<Integer> splitterResultSink2 = new TestListResultSink<Integer>();
-
- StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
- env.setParallelism(1);
- env.setBufferTimeout(1);
-
- DataStream<Integer> ds = env.fromElements(0, 1, 2, 3, 4, 5, 6, 7, 8, 9);
-
- ds.split(new OutputSelector<Integer>() {
- private static final long serialVersionUID = 2524335410904414121L;
-
- @Override
- public Iterable<String> select(Integer value) {
- List<String> s = new ArrayList<String>();
- if (value % 2 == 0) {
- s.add("even");
- } else {
- s.add("odd");
- }
- return s;
- }
- }).select("even").addSink(splitterResultSink1);
-
- ds.split(new OutputSelector<Integer>() {
-
- private static final long serialVersionUID = -511693919586034092L;
-
- @Override
- public Iterable<String> select(Integer value) {
- List<String> s = new ArrayList<String>();
- if (value % 4 == 0) {
- s.add("yes");
- } else {
- s.add("no");
- }
- return s;
- }
- }).select("yes").addSink(splitterResultSink2);
- env.execute();
-
- expectedSplitterResult.clear();
- expectedSplitterResult.addAll(Arrays.asList(0, 2, 4, 6, 8));
- assertEquals(expectedSplitterResult, splitterResultSink1.getSortedResult());
-
- expectedSplitterResult.clear();
- expectedSplitterResult.addAll(Arrays.asList(0, 4, 8));
- assertEquals(expectedSplitterResult, splitterResultSink2.getSortedResult());
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/b9f42e91/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/PartitionerTest.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/PartitionerTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/PartitionerTest.java
deleted file mode 100644
index d408639..0000000
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/PartitionerTest.java
+++ /dev/null
@@ -1,268 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-
-import org.apache.flink.api.common.functions.MapFunction;
-import org.apache.flink.api.common.functions.Partitioner;
-import org.apache.flink.api.common.functions.RichMapFunction;
-import org.apache.flink.api.common.functions.RuntimeContext;
-import org.apache.flink.api.java.tuple.Tuple1;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.util.NoOpIntMap;
-import org.apache.flink.streaming.util.StreamingMultipleProgramsTestBase;
-import org.apache.flink.streaming.util.TestListResultSink;
-
-import org.junit.Test;
-
-/**
- * IT case that tests the different stream partitioning schemes.
- */
-public class PartitionerTest extends StreamingMultipleProgramsTestBase {
-
- @Test(expected = UnsupportedOperationException.class)
- public void testForwardFailsLowToHighParallelism() throws Exception {
- StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
-
- DataStream<Integer> src = env.fromElements(1, 2, 3);
-
- // this doesn't work because it goes from 1 to 3
- src.forward().map(new NoOpIntMap());
-
- env.execute();
- }
-
- @Test(expected = UnsupportedOperationException.class)
- public void testForwardFailsHightToLowParallelism() throws Exception {
- StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
-
- // this does a rebalance that works
- DataStream<Integer> src = env.fromElements(1, 2, 3).map(new NoOpIntMap());
-
- // this doesn't work because it goes from 3 to 1
- src.forward().map(new NoOpIntMap()).setParallelism(1);
-
- env.execute();
- }
-
-
- @Test
- public void partitionerTest() {
-
- TestListResultSink<Tuple2<Integer, String>> hashPartitionResultSink =
- new TestListResultSink<Tuple2<Integer, String>>();
- TestListResultSink<Tuple2<Integer, String>> customPartitionResultSink =
- new TestListResultSink<Tuple2<Integer, String>>();
- TestListResultSink<Tuple2<Integer, String>> broadcastPartitionResultSink =
- new TestListResultSink<Tuple2<Integer, String>>();
- TestListResultSink<Tuple2<Integer, String>> forwardPartitionResultSink =
- new TestListResultSink<Tuple2<Integer, String>>();
- TestListResultSink<Tuple2<Integer, String>> rebalancePartitionResultSink =
- new TestListResultSink<Tuple2<Integer, String>>();
- TestListResultSink<Tuple2<Integer, String>> globalPartitionResultSink =
- new TestListResultSink<Tuple2<Integer, String>>();
-
-
- StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
- env.setParallelism(3);
-
- DataStream<Tuple1<String>> src = env.fromElements(
- new Tuple1<String>("a"),
- new Tuple1<String>("b"),
- new Tuple1<String>("b"),
- new Tuple1<String>("a"),
- new Tuple1<String>("a"),
- new Tuple1<String>("c"),
- new Tuple1<String>("a")
- );
-
- // partition by hash
- src
- .keyBy(0)
- .map(new SubtaskIndexAssigner())
- .addSink(hashPartitionResultSink);
-
- // partition custom
- DataStream<Tuple2<Integer, String>> partitionCustom = src
- .partitionCustom(new Partitioner<String>() {
- @Override
- public int partition(String key, int numPartitions) {
- if (key.equals("c")) {
- return 2;
- } else {
- return 0;
- }
- }
- }, 0)
- .map(new SubtaskIndexAssigner());
-
- partitionCustom.addSink(customPartitionResultSink);
-
- // partition broadcast
- src.broadcast().map(new SubtaskIndexAssigner()).addSink(broadcastPartitionResultSink);
-
- // partition rebalance
- src.rebalance().map(new SubtaskIndexAssigner()).addSink(rebalancePartitionResultSink);
-
- // partition forward
- src.map(new MapFunction<Tuple1<String>, Tuple1<String>>() {
- private static final long serialVersionUID = 1L;
- @Override
- public Tuple1<String> map(Tuple1<String> value) throws Exception {
- return value;
- }
- })
- .forward()
- .map(new SubtaskIndexAssigner())
- .addSink(forwardPartitionResultSink);
-
- // partition global
- src.global().map(new SubtaskIndexAssigner()).addSink(globalPartitionResultSink);
-
- try {
- env.execute();
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
-
- List<Tuple2<Integer, String>> hashPartitionResult = hashPartitionResultSink.getResult();
- List<Tuple2<Integer, String>> customPartitionResult = customPartitionResultSink.getResult();
- List<Tuple2<Integer, String>> broadcastPartitionResult = broadcastPartitionResultSink.getResult();
- List<Tuple2<Integer, String>> forwardPartitionResult = forwardPartitionResultSink.getResult();
- List<Tuple2<Integer, String>> rebalancePartitionResult = rebalancePartitionResultSink.getResult();
- List<Tuple2<Integer, String>> globalPartitionResult = globalPartitionResultSink.getResult();
-
- verifyHashPartitioning(hashPartitionResult);
- verifyCustomPartitioning(customPartitionResult);
- verifyBroadcastPartitioning(broadcastPartitionResult);
- verifyRebalancePartitioning(forwardPartitionResult);
- verifyRebalancePartitioning(rebalancePartitionResult);
- verifyGlobalPartitioning(globalPartitionResult);
- }
-
- private static void verifyHashPartitioning(List<Tuple2<Integer, String>> hashPartitionResult) {
- HashMap<String, Integer> verifier = new HashMap<String, Integer>();
- for (Tuple2<Integer, String> elem : hashPartitionResult) {
- Integer subtaskIndex = verifier.get(elem.f1);
- if (subtaskIndex == null) {
- verifier.put(elem.f1, elem.f0);
- } else if (subtaskIndex != elem.f0) {
- fail();
- }
- }
- }
-
- private static void verifyCustomPartitioning(List<Tuple2<Integer, String>> customPartitionResult) {
- for (Tuple2<Integer, String> stringWithSubtask : customPartitionResult) {
- if (stringWithSubtask.f1.equals("c")) {
- assertEquals(new Integer(2), stringWithSubtask.f0);
- } else {
- assertEquals(new Integer(0), stringWithSubtask.f0);
- }
- }
- }
-
- private static void verifyBroadcastPartitioning(List<Tuple2<Integer, String>> broadcastPartitionResult) {
- List<Tuple2<Integer, String>> expected = Arrays.asList(
- new Tuple2<Integer, String>(0, "a"),
- new Tuple2<Integer, String>(0, "b"),
- new Tuple2<Integer, String>(0, "b"),
- new Tuple2<Integer, String>(0, "a"),
- new Tuple2<Integer, String>(0, "a"),
- new Tuple2<Integer, String>(0, "c"),
- new Tuple2<Integer, String>(0, "a"),
- new Tuple2<Integer, String>(1, "a"),
- new Tuple2<Integer, String>(1, "b"),
- new Tuple2<Integer, String>(1, "b"),
- new Tuple2<Integer, String>(1, "a"),
- new Tuple2<Integer, String>(1, "a"),
- new Tuple2<Integer, String>(1, "c"),
- new Tuple2<Integer, String>(1, "a"),
- new Tuple2<Integer, String>(2, "a"),
- new Tuple2<Integer, String>(2, "b"),
- new Tuple2<Integer, String>(2, "b"),
- new Tuple2<Integer, String>(2, "a"),
- new Tuple2<Integer, String>(2, "a"),
- new Tuple2<Integer, String>(2, "c"),
- new Tuple2<Integer, String>(2, "a"));
-
- assertEquals(
- new HashSet<Tuple2<Integer, String>>(expected),
- new HashSet<Tuple2<Integer, String>>(broadcastPartitionResult));
- }
-
- private static void verifyRebalancePartitioning(List<Tuple2<Integer, String>> rebalancePartitionResult) {
- List<Tuple2<Integer, String>> expected = Arrays.asList(
- new Tuple2<Integer, String>(0, "a"),
- new Tuple2<Integer, String>(1, "b"),
- new Tuple2<Integer, String>(2, "b"),
- new Tuple2<Integer, String>(0, "a"),
- new Tuple2<Integer, String>(1, "a"),
- new Tuple2<Integer, String>(2, "c"),
- new Tuple2<Integer, String>(0, "a"));
-
- assertEquals(
- new HashSet<Tuple2<Integer, String>>(expected),
- new HashSet<Tuple2<Integer, String>>(rebalancePartitionResult));
- }
-
- private static void verifyGlobalPartitioning(List<Tuple2<Integer, String>> globalPartitionResult) {
- List<Tuple2<Integer, String>> expected = Arrays.asList(
- new Tuple2<Integer, String>(0, "a"),
- new Tuple2<Integer, String>(0, "b"),
- new Tuple2<Integer, String>(0, "b"),
- new Tuple2<Integer, String>(0, "a"),
- new Tuple2<Integer, String>(0, "a"),
- new Tuple2<Integer, String>(0, "c"),
- new Tuple2<Integer, String>(0, "a"));
-
- assertEquals(
- new HashSet<Tuple2<Integer, String>>(expected),
- new HashSet<Tuple2<Integer, String>>(globalPartitionResult));
- }
-
- private static class SubtaskIndexAssigner extends RichMapFunction<Tuple1<String>, Tuple2<Integer, String>> {
- private static final long serialVersionUID = 1L;
-
- private int indexOfSubtask;
-
- @Override
- public void open(Configuration parameters) throws Exception {
- super.open(parameters);
- RuntimeContext runtimeContext = getRuntimeContext();
- indexOfSubtask = runtimeContext.getIndexOfThisSubtask();
- }
-
- @Override
- public Tuple2<Integer, String> map(Tuple1<String> value) throws Exception {
- return new Tuple2<Integer, String>(indexOfSubtask, value.f0);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/b9f42e91/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/StreamExecutionEnvironmentTest.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/StreamExecutionEnvironmentTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/StreamExecutionEnvironmentTest.java
index 5e596b9..3fb4513 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/StreamExecutionEnvironmentTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/StreamExecutionEnvironmentTest.java
@@ -31,18 +31,18 @@ import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.functions.sink.DiscardingSink;
import org.apache.flink.streaming.api.functions.source.FromElementsFunction;
import org.apache.flink.streaming.api.functions.source.SourceFunction;
import org.apache.flink.streaming.api.functions.source.StatefulSequenceSource;
import org.apache.flink.streaming.api.graph.StreamGraph;
import org.apache.flink.streaming.api.operators.AbstractUdfStreamOperator;
import org.apache.flink.streaming.api.operators.StreamOperator;
-import org.apache.flink.streaming.util.NoOpSink;
-import org.apache.flink.streaming.util.StreamingMultipleProgramsTestBase;
import org.apache.flink.util.SplittableIterator;
+
import org.junit.Test;
-public class StreamExecutionEnvironmentTest extends StreamingMultipleProgramsTestBase {
+public class StreamExecutionEnvironmentTest {
@Test
public void fromElementsWithBaseTypeTest1() {
@@ -73,18 +73,18 @@ public class StreamExecutionEnvironmentTest extends StreamingMultipleProgramsTes
// expected
}
- dataStream1.addSink(new NoOpSink<Integer>());
+ dataStream1.addSink(new DiscardingSink<Integer>());
DataStreamSource<Integer> dataStream2 = env.fromParallelCollection(new DummySplittableIterator<Integer>(),
typeInfo).setParallelism(4);
- dataStream2.addSink(new NoOpSink<Integer>());
+ dataStream2.addSink(new DiscardingSink<Integer>());
- String plan = env.getExecutionPlan();
+ env.getExecutionPlan();
assertEquals("Parallelism of collection source must be 1.", 1, env.getStreamGraph().getStreamNode(dataStream1.getId()).getParallelism());
assertEquals("Parallelism of parallel collection source must be 4.",
- 4,
+ 4,
env.getStreamGraph().getStreamNode(dataStream2.getId()).getParallelism());
}
catch (Exception e) {
@@ -109,7 +109,7 @@ public class StreamExecutionEnvironmentTest extends StreamingMultipleProgramsTes
}
};
DataStreamSource<Integer> src1 = env.addSource(srcFun);
- src1.addSink(new NoOpSink<Integer>());
+ src1.addSink(new DiscardingSink<Integer>());
assertEquals(srcFun, getFunctionFromDataSource(src1));
List<Long> list = Arrays.asList(0L, 1L, 2L);
@@ -135,8 +135,9 @@ public class StreamExecutionEnvironmentTest extends StreamingMultipleProgramsTes
return streamGraph.getStreamNode(dataStream.getId()).getOperator();
}
+ @SuppressWarnings("unchecked")
private static <T> SourceFunction<T> getFunctionFromDataSource(DataStreamSource<T> dataStreamSource) {
- dataStreamSource.addSink(new NoOpSink<T>());
+ dataStreamSource.addSink(new DiscardingSink<T>());
AbstractUdfStreamOperator<?, ?> operator =
(AbstractUdfStreamOperator<?, ?>) getOperatorFromDataStream(dataStreamSource);
return (SourceFunction<T>) operator.getUserFunction();
http://git-wip-us.apache.org/repos/asf/flink/blob/b9f42e91/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/StreamingOperatorsITCase.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/StreamingOperatorsITCase.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/StreamingOperatorsITCase.java
deleted file mode 100644
index 4f11993..0000000
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/StreamingOperatorsITCase.java
+++ /dev/null
@@ -1,235 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api;
-
-import org.apache.flink.api.common.functions.FoldFunction;
-import org.apache.flink.api.common.functions.MapFunction;
-import org.apache.flink.api.common.functions.RichMapFunction;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.core.fs.FileSystem;
-import org.apache.flink.util.MathUtils;
-import org.apache.flink.streaming.api.collector.selector.OutputSelector;
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.datastream.SplitStream;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.api.functions.source.SourceFunction;
-import org.apache.flink.streaming.util.StreamingMultipleProgramsTestBase;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-
-import java.util.ArrayList;
-import java.util.List;
-
-public class StreamingOperatorsITCase extends StreamingMultipleProgramsTestBase {
-
- private String resultPath1;
- private String resultPath2;
- private String expected1;
- private String expected2;
-
- @Rule
- public TemporaryFolder tempFolder = new TemporaryFolder();
-
- @Before
- public void before() throws Exception {
- resultPath1 = tempFolder.newFile().toURI().toString();
- resultPath2 = tempFolder.newFile().toURI().toString();
- expected1 = "";
- expected2 = "";
- }
-
- @After
- public void after() throws Exception {
- compareResultsByLinesInMemory(expected1, resultPath1);
- compareResultsByLinesInMemory(expected2, resultPath2);
- }
-
- /**
- * Tests the proper functioning of the streaming fold operator. For this purpose, a stream
- * of Tuple2<Integer, Integer> is created. The stream is grouped according to the first tuple
- * value. Each group is folded where the second tuple value is summed up.
- *
- * This test relies on the hash function used by the {@link DataStream#keyBy}, which is
- * assumed to be {@link MathUtils#murmurHash}.
- */
- @Test
- public void testGroupedFoldOperation() throws Exception {
- int numElements = 10;
- final int numKeys = 2;
-
- StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
- DataStream<Tuple2<Integer, Integer>> sourceStream = env.addSource(new TupleSource(numElements, numKeys));
-
- SplitStream<Tuple2<Integer, Integer>> splittedResult = sourceStream
- .keyBy(0)
- .fold(0, new FoldFunction<Tuple2<Integer, Integer>, Integer>() {
- @Override
- public Integer fold(Integer accumulator, Tuple2<Integer, Integer> value) throws Exception {
- return accumulator + value.f1;
- }
- }).map(new RichMapFunction<Integer, Tuple2<Integer, Integer>>() {
- int key = -1;
- @Override
- public Tuple2<Integer, Integer> map(Integer value) throws Exception {
- if (key == -1){
- key = MathUtils.murmurHash(value) % numKeys;
- }
- return new Tuple2<>(key, value);
- }
- }).split(new OutputSelector<Tuple2<Integer, Integer>>() {
- @Override
- public Iterable<String> select(Tuple2<Integer, Integer> value) {
- List<String> output = new ArrayList<>();
-
- output.add(value.f0 + "");
- return output;
- }
- });
-
- splittedResult.select("0").map(new MapFunction<Tuple2<Integer,Integer>, Integer>() {
- @Override
- public Integer map(Tuple2<Integer, Integer> value) throws Exception {
- return value.f1;
- }
- }).writeAsText(resultPath1, FileSystem.WriteMode.OVERWRITE);
-
- splittedResult.select("1").map(new MapFunction<Tuple2<Integer, Integer>, Integer>() {
- @Override
- public Integer map(Tuple2<Integer, Integer> value) throws Exception {
- return value.f1;
- }
- }).writeAsText(resultPath2, FileSystem.WriteMode.OVERWRITE);
-
- StringBuilder builder1 = new StringBuilder();
- StringBuilder builder2 = new StringBuilder();
- int counter1 = 0;
- int counter2 = 0;
-
- for (int i = 0; i < numElements; i++) {
- if (MathUtils.murmurHash(i) % numKeys == 0) {
- counter1 += i;
- builder1.append(counter1 + "\n");
- } else {
- counter2 += i;
- builder2.append(counter2 + "\n");
- }
- }
-
- expected1 = builder1.toString();
- expected2 = builder2.toString();
-
- env.execute();
- }
-
- /**
- * Tests whether the fold operation can also be called with non Java serializable types.
- */
- @Test
- public void testFoldOperationWithNonJavaSerializableType() throws Exception {
- final int numElements = 10;
-
- StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
- DataStream<Tuple2<Integer, NonSerializable>> input = env.addSource(new NonSerializableTupleSource(numElements));
-
- input
- .keyBy(0)
- .fold(
- new NonSerializable(42),
- new FoldFunction<Tuple2<Integer, NonSerializable>, NonSerializable>() {
- @Override
- public NonSerializable fold(NonSerializable accumulator, Tuple2<Integer, NonSerializable> value) throws Exception {
- return new NonSerializable(accumulator.value + value.f1.value);
- }
- })
- .map(new MapFunction<NonSerializable, Integer>() {
- @Override
- public Integer map(NonSerializable value) throws Exception {
- return value.value;
- }
- })
- .writeAsText(resultPath1, FileSystem.WriteMode.OVERWRITE);
-
- StringBuilder builder = new StringBuilder();
-
- for (int i = 0; i < numElements; i++) {
- builder.append(42 + i + "\n");
- }
-
- expected1 = builder.toString();
-
- env.execute();
- }
-
- private static class NonSerializable {
- // This makes the type non-serializable
- private final Object obj = new Object();
-
- private final int value;
-
- public NonSerializable(int value) {
- this.value = value;
- }
- }
-
- private static class NonSerializableTupleSource implements SourceFunction<Tuple2<Integer, NonSerializable>> {
- private final int numElements;
-
- public NonSerializableTupleSource(int numElements) {
- this.numElements = numElements;
- }
-
-
- @Override
- public void run(SourceContext<Tuple2<Integer, NonSerializable>> ctx) throws Exception {
- for (int i = 0; i < numElements; i++) {
- ctx.collect(new Tuple2<>(i, new NonSerializable(i)));
- }
- }
-
- @Override
- public void cancel() {}
- }
-
- private static class TupleSource implements SourceFunction<Tuple2<Integer, Integer>> {
-
- private final int numElements;
- private final int numKeys;
-
- public TupleSource(int numElements, int numKeys) {
- this.numElements = numElements;
- this.numKeys = numKeys;
- }
-
- @Override
- public void run(SourceContext<Tuple2<Integer, Integer>> ctx) throws Exception {
- for (int i = 0; i < numElements; i++) {
- // keys '1' and '2' hash to different buckets
- Tuple2<Integer, Integer> result = new Tuple2<>(1 + (MathUtils.murmurHash(i) % numKeys), i);
- ctx.collect(result);
- }
- }
-
- @Override
- public void cancel() {
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/b9f42e91/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/TypeFillTest.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/TypeFillTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/TypeFillTest.java
index acbb5b4..d931f7b 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/TypeFillTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/TypeFillTest.java
@@ -31,12 +31,12 @@ import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.co.CoFlatMapFunction;
import org.apache.flink.streaming.api.functions.co.CoMapFunction;
import org.apache.flink.streaming.api.functions.source.SourceFunction;
-import org.apache.flink.streaming.util.StreamingMultipleProgramsTestBase;
import org.apache.flink.util.Collector;
+
import org.junit.Test;
@SuppressWarnings("serial")
-public class TypeFillTest extends StreamingMultipleProgramsTestBase {
+public class TypeFillTest {
@Test
public void test() {
http://git-wip-us.apache.org/repos/asf/flink/blob/b9f42e91/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/collector/DirectedOutputTest.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/collector/DirectedOutputTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/collector/DirectedOutputTest.java
deleted file mode 100644
index d2e24c9..0000000
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/collector/DirectedOutputTest.java
+++ /dev/null
@@ -1,119 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.collector;
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.flink.streaming.api.collector.selector.OutputSelector;
-import org.apache.flink.streaming.api.datastream.SplitStream;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.api.functions.sink.SinkFunction;
-import org.apache.flink.streaming.util.StreamingMultipleProgramsTestBase;
-import org.apache.flink.streaming.util.TestListResultSink;
-import org.junit.Test;
-
-public class DirectedOutputTest extends StreamingMultipleProgramsTestBase {
-
- private static final String TEN = "ten";
- private static final String ODD = "odd";
- private static final String EVEN = "even";
- private static final String NON_SELECTED = "nonSelected";
-
- static final class MyOutputSelector implements OutputSelector<Long> {
- private static final long serialVersionUID = 1L;
-
- List<String> outputs = new ArrayList<String>();
-
- @Override
- public Iterable<String> select(Long value) {
- outputs.clear();
- if (value % 2 == 0) {
- outputs.add(EVEN);
- } else {
- outputs.add(ODD);
- }
-
- if (value == 10L) {
- outputs.add(TEN);
- }
-
- if (value == 11L) {
- outputs.add(NON_SELECTED);
- }
- return outputs;
- }
- }
-
- static final class ListSink implements SinkFunction<Long> {
- private static final long serialVersionUID = 1L;
-
- private String name;
- private transient List<Long> list;
-
- public ListSink(String name) {
- this.name = name;
- }
-
- @Override
- public void invoke(Long value) {
- list.add(value);
- }
-
- private void readObject(java.io.ObjectInputStream in) throws IOException,
- ClassNotFoundException {
- in.defaultReadObject();
- outputs.put(name, new ArrayList<Long>());
- this.list = outputs.get(name);
- }
-
- }
-
- private static Map<String, List<Long>> outputs = new HashMap<String, List<Long>>();
-
- @Test
- public void outputSelectorTest() throws Exception {
- StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
- env.setParallelism(3);
-
- TestListResultSink<Long> evenSink = new TestListResultSink<Long>();
- TestListResultSink<Long> oddAndTenSink = new TestListResultSink<Long>();
- TestListResultSink<Long> evenAndOddSink = new TestListResultSink<Long>();
- TestListResultSink<Long> allSink = new TestListResultSink<Long>();
-
- SplitStream<Long> source = env.generateSequence(1, 11).split(new MyOutputSelector());
- source.select(EVEN).addSink(evenSink);
- source.select(ODD, TEN).addSink(oddAndTenSink);
- source.select(EVEN, ODD).addSink(evenAndOddSink);
- source.addSink(allSink);
-
- env.execute();
- assertEquals(Arrays.asList(2L, 4L, 6L, 8L, 10L), evenSink.getSortedResult());
- assertEquals(Arrays.asList(1L, 3L, 5L, 7L, 9L, 10L, 11L), oddAndTenSink.getSortedResult());
- assertEquals(Arrays.asList(1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L, 10L, 11L),
- evenAndOddSink.getSortedResult());
- assertEquals(Arrays.asList(1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L, 10L, 11L),
- allSink.getSortedResult());
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/b9f42e91/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/collector/OutputSelectorTest.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/collector/OutputSelectorTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/collector/OutputSelectorTest.java
index a3d89f2..3194f9e 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/collector/OutputSelectorTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/collector/OutputSelectorTest.java
@@ -25,6 +25,7 @@ import java.util.List;
import org.apache.flink.api.java.tuple.Tuple1;
import org.apache.flink.streaming.api.collector.selector.OutputSelector;
+
import org.junit.Test;
public class OutputSelectorTest {
[06/15] flink git commit: [FLINK-3995] [build] Properly structure
test scopes and dependencies
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskTest.java
index 1762a7a..5394fd1 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskTest.java
@@ -18,10 +18,9 @@
package org.apache.flink.runtime.taskmanager;
-import com.google.common.collect.Maps;
-
-import org.apache.flink.api.common.ExecutionConfigTest;
+import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.configuration.Configuration;
+import org.apache.flink.core.testutils.OneShotLatch;
import org.apache.flink.metrics.groups.TaskMetricGroup;
import org.apache.flink.runtime.blob.BlobKey;
import org.apache.flink.runtime.broadcast.BroadcastVariableManager;
@@ -47,15 +46,18 @@ import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
import org.apache.flink.runtime.memory.MemoryManager;
import org.apache.flink.runtime.messages.TaskMessages;
+import org.apache.flink.util.SerializedValue;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import scala.concurrent.duration.FiniteDuration;
+import java.io.IOException;
import java.lang.reflect.Field;
import java.net.URL;
import java.util.Collections;
+import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
@@ -529,8 +531,7 @@ public class TaskTest {
setInputGate(task, inputGate);
// Expected task state for each partition state
- final Map<ExecutionState, ExecutionState> expected = Maps
- .newHashMapWithExpectedSize(ExecutionState.values().length);
+ final Map<ExecutionState, ExecutionState> expected = new HashMap<>(ExecutionState.values().length);
// Fail the task for unexpected states
for (ExecutionState state : ExecutionState.values()) {
@@ -564,7 +565,7 @@ public class TaskTest {
f.setAccessible(true);
f.set(task, new SingleInputGate[]{inputGate});
- Map<IntermediateDataSetID, SingleInputGate> byId = Maps.newHashMapWithExpectedSize(1);
+ Map<IntermediateDataSetID, SingleInputGate> byId = new HashMap<>(1);
byId.put(inputGate.getConsumedResultId(), inputGate);
f = Task.class.getDeclaredField("inputGatesById");
@@ -628,9 +629,16 @@ public class TaskTest {
}
private TaskDeploymentDescriptor createTaskDeploymentDescriptor(Class<? extends AbstractInvokable> invokable) {
+ SerializedValue<ExecutionConfig> execConfig;
+ try {
+ execConfig = new SerializedValue<>(new ExecutionConfig());
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
return new TaskDeploymentDescriptor(
new JobID(), "Test Job", new JobVertexID(), new ExecutionAttemptID(),
- ExecutionConfigTest.getSerializedConfig(),
+ execConfig,
"Test Task", 0, 1, 0,
new Configuration(), new Configuration(),
invokable.getName(),
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/TestingResourceManager.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/TestingResourceManager.java b/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/TestingResourceManager.java
deleted file mode 100644
index 2422925..0000000
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/TestingResourceManager.java
+++ /dev/null
@@ -1,137 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.testutils;
-
-import akka.actor.ActorRef;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.runtime.clusterframework.ApplicationStatus;
-import org.apache.flink.runtime.clusterframework.messages.RegisterResourceManagerSuccessful;
-import org.apache.flink.runtime.clusterframework.standalone.StandaloneResourceManager;
-import org.apache.flink.runtime.clusterframework.types.ResourceID;
-import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
-import org.apache.flink.runtime.messages.Messages;
-import org.apache.flink.runtime.testingUtils.TestingMessages;
-
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.Set;
-
-
-/**
- * A testing resource manager which may alter the default standalone resource master's behavior.
- */
-public class TestingResourceManager extends StandaloneResourceManager {
-
- /** Set of Actors which want to be informed of a connection to the job manager */
- private Set<ActorRef> waitForResourceManagerConnected = new HashSet<>();
-
- /** Set of Actors which want to be informed of a shutdown */
- private Set<ActorRef> waitForShutdown = new HashSet<>();
-
- /** Flag to signal a connection to the JobManager */
- private boolean isConnected = false;
-
- public TestingResourceManager(Configuration flinkConfig, LeaderRetrievalService leaderRetriever) {
- super(flinkConfig, leaderRetriever);
- }
-
- /**
- * Overwrite messages here if desired
- */
- @Override
- protected void handleMessage(Object message) {
-
- if (message instanceof GetRegisteredResources) {
- sender().tell(new GetRegisteredResourcesReply(getRegisteredTaskManagers()), self());
- } else if (message instanceof FailResource) {
- ResourceID resourceID = ((FailResource) message).resourceID;
- notifyWorkerFailed(resourceID, "Failed for test case.");
-
- } else if (message instanceof NotifyWhenResourceManagerConnected) {
- if (isConnected) {
- sender().tell(
- Messages.getAcknowledge(),
- self());
- } else {
- waitForResourceManagerConnected.add(sender());
- }
- } else if (message instanceof RegisterResourceManagerSuccessful) {
- super.handleMessage(message);
-
- isConnected = true;
-
- for (ActorRef ref : waitForResourceManagerConnected) {
- ref.tell(
- Messages.getAcknowledge(),
- self());
- }
- waitForResourceManagerConnected.clear();
-
- } else if (message instanceof TestingMessages.NotifyOfComponentShutdown$) {
- waitForShutdown.add(sender());
- } else if (message instanceof TestingMessages.Alive$) {
- sender().tell(Messages.getAcknowledge(), self());
- } else {
- super.handleMessage(message);
- }
- }
-
- /**
- * Testing messages
- */
- public static class GetRegisteredResources {}
-
- public static class GetRegisteredResourcesReply {
-
- public Collection<ResourceID> resources;
-
- public GetRegisteredResourcesReply(Collection<ResourceID> resources) {
- this.resources = resources;
- }
-
- }
-
- /**
- * Fails all resources that the resource manager has registered
- */
- public static class FailResource {
-
- public ResourceID resourceID;
-
- public FailResource(ResourceID resourceID) {
- this.resourceID = resourceID;
- }
- }
-
- /**
- * The sender of this message will be informed of a connection to the Job Manager
- */
- public static class NotifyWhenResourceManagerConnected {}
-
- /**
- * Inform registered listeners about a shutdown of the application.
- */
- @Override
- protected void shutdownApplication(ApplicationStatus finalStatus, String optionalDiagnostics) {
- for (ActorRef listener : waitForShutdown) {
- listener.tell(new TestingMessages.ComponentShutdown(self()), self());
- }
- waitForShutdown.clear();
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-runtime/src/test/scala/org/apache/flink/runtime/executiongraph/TaskManagerLossFailsTasksTest.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/scala/org/apache/flink/runtime/executiongraph/TaskManagerLossFailsTasksTest.scala b/flink-runtime/src/test/scala/org/apache/flink/runtime/executiongraph/TaskManagerLossFailsTasksTest.scala
index 1927c39..c30d244 100644
--- a/flink-runtime/src/test/scala/org/apache/flink/runtime/executiongraph/TaskManagerLossFailsTasksTest.scala
+++ b/flink-runtime/src/test/scala/org/apache/flink/runtime/executiongraph/TaskManagerLossFailsTasksTest.scala
@@ -18,7 +18,7 @@
package org.apache.flink.runtime.executiongraph
-import org.apache.flink.api.common.{ExecutionConfig, ExecutionConfigTest, JobID}
+import org.apache.flink.api.common.{ExecutionConfig, JobID}
import org.apache.flink.configuration.Configuration
import org.apache.flink.runtime.akka.AkkaUtils
import org.apache.flink.runtime.executiongraph.ExecutionGraphTestUtils.SimpleActorGateway
@@ -27,6 +27,7 @@ import org.apache.flink.runtime.jobgraph.{JobGraph, JobStatus, JobVertex}
import org.apache.flink.runtime.jobmanager.Tasks
import org.apache.flink.runtime.jobmanager.scheduler.Scheduler
import org.apache.flink.runtime.testingUtils.TestingUtils
+import org.apache.flink.util.SerializedValue
import org.junit.runner.RunWith
import org.scalatest.junit.JUnitRunner
import org.scalatest.{Matchers, WordSpecLike}
@@ -57,7 +58,7 @@ class TaskManagerLossFailsTasksTest extends WordSpecLike with Matchers {
new JobID(),
"test job",
new Configuration(),
- ExecutionConfigTest.getSerializedConfig,
+ new SerializedValue(new ExecutionConfig()),
AkkaUtils.getDefaultTimeout,
new NoRestartStrategy())
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingJobManager.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingJobManager.scala b/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingJobManager.scala
deleted file mode 100644
index 04689c6..0000000
--- a/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingJobManager.scala
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.flink.runtime.testingUtils
-
-import akka.actor.ActorRef
-
-import org.apache.flink.configuration.Configuration
-import org.apache.flink.metrics.MetricRegistry
-import org.apache.flink.runtime.checkpoint.{SavepointStore, CheckpointRecoveryFactory}
-import org.apache.flink.runtime.execution.librarycache.BlobLibraryCacheManager
-import org.apache.flink.runtime.executiongraph.restart.RestartStrategyFactory
-import org.apache.flink.runtime.instance.InstanceManager
-import org.apache.flink.runtime.jobmanager.scheduler.Scheduler
-import org.apache.flink.runtime.jobmanager.{JobManager, SubmittedJobGraphStore}
-import org.apache.flink.runtime.leaderelection.LeaderElectionService
-
-import scala.concurrent.duration._
-import scala.language.postfixOps
-
-import java.util.concurrent.ExecutorService
-
-/** JobManager implementation extended by testing messages
- *
- */
-class TestingJobManager(
- flinkConfiguration: Configuration,
- executorService: ExecutorService,
- instanceManager: InstanceManager,
- scheduler: Scheduler,
- libraryCacheManager: BlobLibraryCacheManager,
- archive: ActorRef,
- restartStrategyFactory: RestartStrategyFactory,
- timeout: FiniteDuration,
- leaderElectionService: LeaderElectionService,
- submittedJobGraphs : SubmittedJobGraphStore,
- checkpointRecoveryFactory : CheckpointRecoveryFactory,
- savepointStore : SavepointStore,
- jobRecoveryTimeout : FiniteDuration,
- metricRegistry : Option[MetricRegistry])
- extends JobManager(
- flinkConfiguration,
- executorService,
- instanceManager,
- scheduler,
- libraryCacheManager,
- archive,
- restartStrategyFactory,
- timeout,
- leaderElectionService,
- submittedJobGraphs,
- checkpointRecoveryFactory,
- savepointStore,
- jobRecoveryTimeout,
- metricRegistry)
- with TestingJobManagerLike {}
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingJobManagerLike.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingJobManagerLike.scala b/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingJobManagerLike.scala
deleted file mode 100644
index 8ea603f..0000000
--- a/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingJobManagerLike.scala
+++ /dev/null
@@ -1,404 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.testingUtils
-
-import akka.actor.{ActorRef, Cancellable, Terminated}
-import akka.pattern.{ask, pipe}
-import org.apache.flink.api.common.JobID
-import org.apache.flink.runtime.FlinkActor
-import org.apache.flink.runtime.clusterframework.messages.RegisterResourceSuccessful
-import org.apache.flink.runtime.execution.ExecutionState
-import org.apache.flink.runtime.jobgraph.JobStatus
-import org.apache.flink.runtime.jobmanager.JobManager
-import org.apache.flink.runtime.messages.ExecutionGraphMessages.JobStatusChanged
-import org.apache.flink.runtime.messages.JobManagerMessages.GrantLeadership
-import org.apache.flink.runtime.messages.Messages.{Acknowledge, Disconnect}
-import org.apache.flink.runtime.messages.RegistrationMessages.RegisterTaskManager
-import org.apache.flink.runtime.messages.TaskManagerMessages.Heartbeat
-import org.apache.flink.runtime.testingUtils.TestingJobManagerMessages._
-import org.apache.flink.runtime.testingUtils.TestingMessages._
-import org.apache.flink.runtime.testingUtils.TestingTaskManagerMessages.AccumulatorsChanged
-
-import scala.collection.mutable
-import scala.concurrent.Future
-import scala.concurrent.duration._
-import language.postfixOps
-
-/** This mixin can be used to decorate a JobManager with messages for testing purpose. */
-trait TestingJobManagerLike extends FlinkActor {
- that: JobManager =>
-
- import scala.collection.JavaConverters._
- import context._
-
- val waitForAllVerticesToBeRunning = scala.collection.mutable.HashMap[JobID, Set[ActorRef]]()
- val waitForTaskManagerToBeTerminated = scala.collection.mutable.HashMap[String, Set[ActorRef]]()
-
- val waitForAllVerticesToBeRunningOrFinished =
- scala.collection.mutable.HashMap[JobID, Set[ActorRef]]()
-
- var periodicCheck: Option[Cancellable] = None
-
- val waitForJobStatus = scala.collection.mutable.HashMap[JobID,
- collection.mutable.HashMap[JobStatus, Set[ActorRef]]]()
-
- val waitForAccumulatorUpdate = scala.collection.mutable.HashMap[JobID, (Boolean, Set[ActorRef])]()
-
- val waitForLeader = scala.collection.mutable.HashSet[ActorRef]()
-
- val waitForNumRegisteredTaskManagers = mutable.PriorityQueue.newBuilder(
- new Ordering[(Int, ActorRef)] {
- override def compare(x: (Int, ActorRef), y: (Int, ActorRef)): Int = y._1 - x._1
- })
-
- val waitForShutdown = scala.collection.mutable.HashSet[ActorRef]()
-
- var disconnectDisabled = false
-
- var postStopEnabled = true
-
- abstract override def postStop(): Unit = {
- if (postStopEnabled) {
- super.postStop()
- } else {
- // only stop leader election service to revoke the leadership of this JM so that a new JM
- // can be elected leader
- leaderElectionService.stop()
- }
- }
-
- abstract override def handleMessage: Receive = {
- handleTestingMessage orElse super.handleMessage
- }
-
- def handleTestingMessage: Receive = {
- case Alive => sender() ! Acknowledge
-
- case RequestExecutionGraph(jobID) =>
- currentJobs.get(jobID) match {
- case Some((executionGraph, jobInfo)) => sender() ! decorateMessage(
- ExecutionGraphFound(
- jobID,
- executionGraph)
- )
-
- case None => archive.tell(decorateMessage(RequestExecutionGraph(jobID)), sender())
- }
-
- case WaitForAllVerticesToBeRunning(jobID) =>
- if(checkIfAllVerticesRunning(jobID)){
- sender() ! decorateMessage(AllVerticesRunning(jobID))
- }else{
- val waiting = waitForAllVerticesToBeRunning.getOrElse(jobID, Set[ActorRef]())
- waitForAllVerticesToBeRunning += jobID -> (waiting + sender())
-
- if(periodicCheck.isEmpty){
- periodicCheck =
- Some(
- context.system.scheduler.schedule(
- 0 seconds,
- 200 millis,
- self,
- decorateMessage(NotifyListeners)
- )
- )
- }
- }
- case WaitForAllVerticesToBeRunningOrFinished(jobID) =>
- if(checkIfAllVerticesRunningOrFinished(jobID)){
- sender() ! decorateMessage(AllVerticesRunning(jobID))
- }else{
- val waiting = waitForAllVerticesToBeRunningOrFinished.getOrElse(jobID, Set[ActorRef]())
- waitForAllVerticesToBeRunningOrFinished += jobID -> (waiting + sender())
-
- if(periodicCheck.isEmpty){
- periodicCheck =
- Some(
- context.system.scheduler.schedule(
- 0 seconds,
- 200 millis,
- self,
- decorateMessage(NotifyListeners)
- )
- )
- }
- }
-
- case NotifyListeners =>
- for(jobID <- currentJobs.keySet){
- notifyListeners(jobID)
- }
-
- if(waitForAllVerticesToBeRunning.isEmpty && waitForAllVerticesToBeRunningOrFinished.isEmpty) {
- periodicCheck foreach { _.cancel() }
- periodicCheck = None
- }
-
-
- case NotifyWhenJobRemoved(jobID) =>
- val gateways = instanceManager.getAllRegisteredInstances.asScala.map(_.getActorGateway)
-
- val responses = gateways.map{
- gateway => gateway.ask(NotifyWhenJobRemoved(jobID), timeout).mapTo[Boolean]
- }
-
- val jobRemovedOnJobManager = (self ? CheckIfJobRemoved(jobID))(timeout).mapTo[Boolean]
-
- val allFutures = responses ++ Seq(jobRemovedOnJobManager)
-
- import context.dispatcher
- Future.fold(allFutures)(true)(_ & _) map(decorateMessage(_)) pipeTo sender()
-
- case CheckIfJobRemoved(jobID) =>
- if(currentJobs.contains(jobID)) {
- context.system.scheduler.scheduleOnce(
- 200 milliseconds,
- self,
- decorateMessage(CheckIfJobRemoved(jobID))
- )(context.dispatcher, sender())
- } else {
- sender() ! decorateMessage(true)
- }
-
- case NotifyWhenTaskManagerTerminated(taskManager) =>
- val waiting = waitForTaskManagerToBeTerminated.getOrElse(taskManager.path.name, Set())
- waitForTaskManagerToBeTerminated += taskManager.path.name -> (waiting + sender)
-
- case msg@Terminated(taskManager) =>
- super.handleMessage(msg)
-
- waitForTaskManagerToBeTerminated.remove(taskManager.path.name) foreach {
- _ foreach {
- listener =>
- listener ! decorateMessage(TaskManagerTerminated(taskManager))
- }
- }
-
- // see shutdown method for reply
- case NotifyOfComponentShutdown =>
- waitForShutdown += sender()
-
- case NotifyWhenAccumulatorChange(jobID) =>
-
- val (updated, registered) = waitForAccumulatorUpdate.
- getOrElse(jobID, (false, Set[ActorRef]()))
- waitForAccumulatorUpdate += jobID -> (updated, registered + sender)
- sender ! true
-
- /**
- * Notification from the task manager that changed accumulator are transferred on next
- * Hearbeat. We need to keep this state to notify the listeners on next Heartbeat report.
- */
- case AccumulatorsChanged(jobID: JobID) =>
- waitForAccumulatorUpdate.get(jobID) match {
- case Some((updated, registered)) =>
- waitForAccumulatorUpdate.put(jobID, (true, registered))
- case None =>
- }
-
- /**
- * Disabled async processing of accumulator values and send accumulators to the listeners if
- * we previously received an [[AccumulatorsChanged]] message.
- */
- case msg : Heartbeat =>
- super.handleMessage(msg)
-
- waitForAccumulatorUpdate foreach {
- case (jobID, (updated, actors)) if updated =>
- currentJobs.get(jobID) match {
- case Some((graph, jobInfo)) =>
- val flinkAccumulators = graph.getFlinkAccumulators
- val userAccumulators = graph.aggregateUserAccumulators
- actors foreach {
- actor => actor ! UpdatedAccumulators(jobID, flinkAccumulators, userAccumulators)
- }
- case None =>
- }
- waitForAccumulatorUpdate.put(jobID, (false, actors))
- case _ =>
- }
-
- case RequestWorkingTaskManager(jobID) =>
- currentJobs.get(jobID) match {
- case Some((eg, _)) =>
- if(eg.getAllExecutionVertices.asScala.isEmpty){
- sender ! decorateMessage(WorkingTaskManager(None))
- } else {
- val resource = eg.getAllExecutionVertices.asScala.head.getCurrentAssignedResource
-
- if(resource == null){
- sender ! decorateMessage(WorkingTaskManager(None))
- } else {
- sender ! decorateMessage(
- WorkingTaskManager(
- Some(resource.getInstance().getActorGateway)
- )
- )
- }
- }
- case None => sender ! decorateMessage(WorkingTaskManager(None))
- }
-
- case NotifyWhenJobStatus(jobID, state) =>
- val jobStatusListener = waitForJobStatus.getOrElseUpdate(jobID,
- scala.collection.mutable.HashMap[JobStatus, Set[ActorRef]]())
-
- val listener = jobStatusListener.getOrElse(state, Set[ActorRef]())
-
- jobStatusListener += state -> (listener + sender)
-
- case msg@JobStatusChanged(jobID, newJobStatus, _, _) =>
- super.handleMessage(msg)
-
- val cleanup = waitForJobStatus.get(jobID) match {
- case Some(stateListener) =>
- stateListener.remove(newJobStatus) match {
- case Some(listeners) =>
- listeners foreach {
- _ ! decorateMessage(JobStatusIs(jobID, newJobStatus))
- }
- case _ =>
- }
- stateListener.isEmpty
-
- case _ => false
- }
-
- if (cleanup) {
- waitForJobStatus.remove(jobID)
- }
-
- case DisableDisconnect =>
- disconnectDisabled = true
-
- case DisablePostStop =>
- postStopEnabled = false
-
- case RequestSavepoint(savepointPath) =>
- try {
- val savepoint = savepointStore.getState(savepointPath)
- sender ! ResponseSavepoint(savepoint)
- }
- catch {
- case e: Exception =>
- sender ! ResponseSavepoint(null)
- }
-
- case msg: Disconnect =>
- if (!disconnectDisabled) {
- super.handleMessage(msg)
-
- val taskManager = sender()
-
- waitForTaskManagerToBeTerminated.remove(taskManager.path.name) foreach {
- _ foreach {
- listener =>
- listener ! decorateMessage(TaskManagerTerminated(taskManager))
- }
- }
- }
-
- case NotifyWhenLeader =>
- if (leaderElectionService.hasLeadership) {
- sender() ! true
- } else {
- waitForLeader += sender()
- }
-
- case msg: GrantLeadership =>
- super.handleMessage(msg)
-
- waitForLeader.foreach(_ ! true)
-
- waitForLeader.clear()
-
- case NotifyWhenAtLeastNumTaskManagerAreRegistered(numRegisteredTaskManager) =>
- if (that.instanceManager.getNumberOfRegisteredTaskManagers >= numRegisteredTaskManager) {
- // there are already at least numRegisteredTaskManager registered --> send Acknowledge
- sender() ! Acknowledge
- } else {
- // wait until we see at least numRegisteredTaskManager being registered at the JobManager
- waitForNumRegisteredTaskManagers += ((numRegisteredTaskManager, sender()))
- }
-
- // TaskManager may be registered on these two messages
- case msg @ (_: RegisterTaskManager | _: RegisterResourceSuccessful) =>
- super.handleMessage(msg)
-
- // dequeue all senders which wait for instanceManager.getNumberOfRegisteredTaskManagers or
- // fewer registered TaskManagers
- while (waitForNumRegisteredTaskManagers.nonEmpty &&
- waitForNumRegisteredTaskManagers.head._1 <=
- instanceManager.getNumberOfRegisteredTaskManagers) {
- val receiver = waitForNumRegisteredTaskManagers.dequeue()._2
- receiver ! Acknowledge
- }
- }
-
- def checkIfAllVerticesRunning(jobID: JobID): Boolean = {
- currentJobs.get(jobID) match {
- case Some((eg, _)) =>
- eg.getAllExecutionVertices.asScala.forall( _.getExecutionState == ExecutionState.RUNNING)
- case None => false
- }
- }
-
- def checkIfAllVerticesRunningOrFinished(jobID: JobID): Boolean = {
- currentJobs.get(jobID) match {
- case Some((eg, _)) =>
- eg.getAllExecutionVertices.asScala.forall {
- case vertex =>
- (vertex.getExecutionState == ExecutionState.RUNNING
- || vertex.getExecutionState == ExecutionState.FINISHED)
- }
- case None => false
- }
- }
-
- def notifyListeners(jobID: JobID): Unit = {
- if(checkIfAllVerticesRunning(jobID)) {
- waitForAllVerticesToBeRunning.remove(jobID) match {
- case Some(listeners) =>
- for (listener <- listeners) {
- listener ! decorateMessage(AllVerticesRunning(jobID))
- }
- case _ =>
- }
- }
-
- if(checkIfAllVerticesRunningOrFinished(jobID)) {
- waitForAllVerticesToBeRunningOrFinished.remove(jobID) match {
- case Some(listeners) =>
- for (listener <- listeners) {
- listener ! decorateMessage(AllVerticesRunning(jobID))
- }
- case _ =>
- }
- }
- }
-
- /**
- * No killing of the VM for testing.
- */
- override protected def shutdown(): Unit = {
- log.info("Shutting down TestingJobManager.")
- waitForShutdown.foreach(_ ! ComponentShutdown(self))
- waitForShutdown.clear()
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingJobManagerMessages.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingJobManagerMessages.scala b/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingJobManagerMessages.scala
deleted file mode 100644
index ec513e3..0000000
--- a/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingJobManagerMessages.scala
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.testingUtils
-
-import java.util.Map
-
-import akka.actor.ActorRef
-import org.apache.flink.api.common.JobID
-import org.apache.flink.api.common.accumulators.Accumulator
-import org.apache.flink.runtime.accumulators.AccumulatorRegistry
-import org.apache.flink.runtime.checkpoint.CompletedCheckpoint
-import org.apache.flink.runtime.executiongraph.{ExecutionAttemptID, ExecutionGraph}
-import org.apache.flink.runtime.instance.ActorGateway
-import org.apache.flink.runtime.jobgraph.JobStatus
-
-object TestingJobManagerMessages {
-
- case class RequestExecutionGraph(jobID: JobID)
-
- sealed trait ResponseExecutionGraph {
- def jobID: JobID
- }
-
- case class ExecutionGraphFound(jobID: JobID, executionGraph: ExecutionGraph) extends
- ResponseExecutionGraph
-
- case class ExecutionGraphNotFound(jobID: JobID) extends ResponseExecutionGraph
-
- case class WaitForAllVerticesToBeRunning(jobID: JobID)
- case class WaitForAllVerticesToBeRunningOrFinished(jobID: JobID)
- case class AllVerticesRunning(jobID: JobID)
-
- case class NotifyWhenJobRemoved(jobID: JobID)
-
- case class RequestWorkingTaskManager(jobID: JobID)
- case class WorkingTaskManager(gatewayOption: Option[ActorGateway])
-
- case class NotifyWhenJobStatus(jobID: JobID, state: JobStatus)
- case class JobStatusIs(jobID: JobID, state: JobStatus)
-
- case object NotifyListeners
-
- case class NotifyWhenTaskManagerTerminated(taskManager: ActorRef)
- case class TaskManagerTerminated(taskManager: ActorRef)
-
- /**
- * Registers a listener to receive a message when accumulators changed.
- * The change must be explicitly triggered by the TestingTaskManager which can receive an
- * [[org.apache.flink.runtime.testingUtils.TestingTaskManagerMessages.AccumulatorsChanged]]
- * message by a task that changed the accumulators. This message is then
- * forwarded to the JobManager which will send the accumulators in the [[UpdatedAccumulators]]
- * message when the next Heartbeat occurs.
- */
- case class NotifyWhenAccumulatorChange(jobID: JobID)
-
- /**
- * Reports updated accumulators back to the listener.
- */
- case class UpdatedAccumulators(jobID: JobID,
- flinkAccumulators: Map[ExecutionAttemptID, Map[AccumulatorRegistry.Metric, Accumulator[_,_]]],
- userAccumulators: Map[String, Accumulator[_,_]])
-
- /** Notifies the sender when the [[TestingJobManager]] has been elected as the leader
- *
- */
- case object NotifyWhenLeader
-
- /**
- * Registers to be notified by an [[org.apache.flink.runtime.messages.Messages.Acknowledge]]
- * message when at least numRegisteredTaskManager have registered at the JobManager.
- *
- * @param numRegisteredTaskManager minimum number of registered TMs before the sender is notified
- */
- case class NotifyWhenAtLeastNumTaskManagerAreRegistered(numRegisteredTaskManager: Int)
-
- /** Disables the post stop method of the [[TestingJobManager]].
- *
- * Only the leaderElectionService is stopped in the postStop method call to revoke the leadership
- */
- case object DisablePostStop
-
- /**
- * Requests a savepoint from the job manager.
- *
- * @param savepointPath The path of the savepoint to request.
- */
- case class RequestSavepoint(savepointPath: String)
-
- /**
- * Response to a savepoint request.
- *
- * @param savepoint The requested savepoint or null if none available.
- */
- case class ResponseSavepoint(savepoint: CompletedCheckpoint)
-
- def getNotifyWhenLeader(): AnyRef = NotifyWhenLeader
- def getDisablePostStop(): AnyRef = DisablePostStop
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingMemoryArchivist.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingMemoryArchivist.scala b/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingMemoryArchivist.scala
deleted file mode 100644
index 71a7e3e..0000000
--- a/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingMemoryArchivist.scala
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.testingUtils
-
-import org.apache.flink.runtime.jobmanager.MemoryArchivist
-import org.apache.flink.runtime.testingUtils.TestingJobManagerMessages.{ExecutionGraphNotFound, ExecutionGraphFound, RequestExecutionGraph}
-
-/** Memory archivist extended by testing messages
- *
- * @param maxEntries number of maximum number of archived jobs
- */
-class TestingMemoryArchivist(maxEntries: Int) extends MemoryArchivist(maxEntries) {
-
- override def handleMessage: Receive = {
- handleTestingMessage orElse super.handleMessage
- }
-
- def handleTestingMessage: Receive = {
- case RequestExecutionGraph(jobID) =>
- val executionGraph = graphs.get(jobID)
-
- executionGraph match {
- case Some(graph) => sender ! decorateMessage(ExecutionGraphFound(jobID, graph))
- case None => sender ! decorateMessage(ExecutionGraphNotFound(jobID))
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingMessages.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingMessages.scala b/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingMessages.scala
deleted file mode 100644
index 91d169a..0000000
--- a/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingMessages.scala
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.testingUtils
-
-import akka.actor.ActorRef
-import org.apache.flink.api.common.JobID
-
-object TestingMessages {
-
- case class CheckIfJobRemoved(jobID: JobID)
-
- case object DisableDisconnect
-
- case object Alive
-
- def getAlive: AnyRef = Alive
-
- def getDisableDisconnect: AnyRef = DisableDisconnect
-
- case object NotifyOfComponentShutdown
- case class ComponentShutdown(ref: ActorRef)
-
- def getNotifyOfComponentShutdown(): AnyRef = NotifyOfComponentShutdown
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingTaskManager.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingTaskManager.scala b/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingTaskManager.scala
deleted file mode 100644
index 2597753..0000000
--- a/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingTaskManager.scala
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.testingUtils
-
-import org.apache.flink.runtime.clusterframework.types.ResourceID
-import org.apache.flink.runtime.instance.InstanceConnectionInfo
-import org.apache.flink.runtime.io.disk.iomanager.IOManager
-import org.apache.flink.runtime.io.network.NetworkEnvironment
-import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService
-import org.apache.flink.runtime.memory.MemoryManager
-import org.apache.flink.runtime.taskmanager.{TaskManager, TaskManagerConfiguration}
-
-import scala.language.postfixOps
-
-/** Subclass of the [[TaskManager]] to support testing messages
- */
-class TestingTaskManager(
- config: TaskManagerConfiguration,
- resourceID: ResourceID,
- connectionInfo: InstanceConnectionInfo,
- memoryManager: MemoryManager,
- ioManager: IOManager,
- network: NetworkEnvironment,
- numberOfSlots: Int,
- leaderRetrievalService: LeaderRetrievalService)
- extends TaskManager(
- config,
- resourceID,
- connectionInfo,
- memoryManager,
- ioManager,
- network,
- numberOfSlots,
- leaderRetrievalService)
- with TestingTaskManagerLike {
-
- def this(
- config: TaskManagerConfiguration,
- connectionInfo: InstanceConnectionInfo,
- memoryManager: MemoryManager,
- ioManager: IOManager,
- network: NetworkEnvironment,
- numberOfSlots: Int,
- leaderRetrievalService: LeaderRetrievalService) {
- this(
- config,
- ResourceID.generate(),
- connectionInfo,
- memoryManager,
- ioManager,
- network,
- numberOfSlots,
- leaderRetrievalService)
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingTaskManagerLike.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingTaskManagerLike.scala b/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingTaskManagerLike.scala
deleted file mode 100644
index f8df89c..0000000
--- a/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingTaskManagerLike.scala
+++ /dev/null
@@ -1,248 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.testingUtils
-
-import akka.actor.{ActorRef, Terminated}
-import org.apache.flink.api.common.JobID
-import org.apache.flink.runtime.FlinkActor
-import org.apache.flink.runtime.execution.ExecutionState
-import org.apache.flink.runtime.executiongraph.ExecutionAttemptID
-import org.apache.flink.runtime.messages.JobManagerMessages.{RequestLeaderSessionID, ResponseLeaderSessionID}
-import org.apache.flink.runtime.messages.Messages.{Acknowledge, Disconnect}
-import org.apache.flink.runtime.messages.RegistrationMessages.{AcknowledgeRegistration, AlreadyRegistered}
-import org.apache.flink.runtime.messages.TaskMessages.{SubmitTask, TaskInFinalState, UpdateTaskExecutionState}
-import org.apache.flink.runtime.taskmanager.TaskManager
-import org.apache.flink.runtime.testingUtils.TestingJobManagerMessages.NotifyWhenJobRemoved
-import org.apache.flink.runtime.testingUtils.TestingMessages._
-import org.apache.flink.runtime.testingUtils.TestingTaskManagerMessages._
-
-import scala.concurrent.duration._
-import language.postfixOps
-
-/** This mixin can be used to decorate a TaskManager with messages for testing purposes. */
-trait TestingTaskManagerLike extends FlinkActor {
- that: TaskManager =>
-
- import scala.collection.JavaConverters._
-
- val waitForRemoval = scala.collection.mutable.HashMap[ExecutionAttemptID, Set[ActorRef]]()
- val waitForJobManagerToBeTerminated = scala.collection.mutable.HashMap[String, Set[ActorRef]]()
- val waitForRegisteredAtResourceManager =
- scala.collection.mutable.HashMap[ActorRef, Set[ActorRef]]()
- val waitForRunning = scala.collection.mutable.HashMap[ExecutionAttemptID, Set[ActorRef]]()
- val unregisteredTasks = scala.collection.mutable.HashSet[ExecutionAttemptID]()
-
- /** Map of registered task submit listeners */
- val registeredSubmitTaskListeners = scala.collection.mutable.HashMap[JobID, ActorRef]()
-
- val waitForShutdown = scala.collection.mutable.HashSet[ActorRef]()
-
- var disconnectDisabled = false
-
- /**
- * Handler for testing related messages
- */
- abstract override def handleMessage: Receive = {
- handleTestingMessage orElse super.handleMessage
- }
-
- def handleTestingMessage: Receive = {
- case Alive => sender() ! Acknowledge
-
- case NotifyWhenTaskIsRunning(executionID) =>
- Option(runningTasks.get(executionID)) match {
- case Some(task) if task.getExecutionState == ExecutionState.RUNNING =>
- sender ! decorateMessage(true)
-
- case _ =>
- val listeners = waitForRunning.getOrElse(executionID, Set())
- waitForRunning += (executionID -> (listeners + sender))
- }
-
- case RequestRunningTasks =>
- sender ! decorateMessage(ResponseRunningTasks(runningTasks.asScala.toMap))
-
- case NotifyWhenTaskRemoved(executionID) =>
- Option(runningTasks.get(executionID)) match {
- case Some(_) =>
- val set = waitForRemoval.getOrElse(executionID, Set())
- waitForRemoval += (executionID -> (set + sender))
- case None =>
- if(unregisteredTasks.contains(executionID)) {
- sender ! decorateMessage(true)
- } else {
- val set = waitForRemoval.getOrElse(executionID, Set())
- waitForRemoval += (executionID -> (set + sender))
- }
- }
-
- case TaskInFinalState(executionID) =>
- super.handleMessage(TaskInFinalState(executionID))
- waitForRemoval.remove(executionID) match {
- case Some(actors) => for(actor <- actors) actor ! decorateMessage(true)
- case None =>
- }
-
- unregisteredTasks += executionID
-
- case RequestBroadcastVariablesWithReferences =>
- sender ! decorateMessage(
- ResponseBroadcastVariablesWithReferences(
- bcVarManager.getNumberOfVariablesWithReferences)
- )
-
- case RequestNumActiveConnections =>
- val numActive = if (network.isAssociated) {
- network.getConnectionManager.getNumberOfActiveConnections
- } else {
- 0
- }
- sender ! decorateMessage(ResponseNumActiveConnections(numActive))
-
- case NotifyWhenJobRemoved(jobID) =>
- if(runningTasks.values.asScala.exists(_.getJobID == jobID)){
- context.system.scheduler.scheduleOnce(
- 200 milliseconds,
- self,
- decorateMessage(CheckIfJobRemoved(jobID)))(
- context.dispatcher,
- sender()
- )
- }else{
- sender ! decorateMessage(true)
- }
-
- case CheckIfJobRemoved(jobID) =>
- if(runningTasks.values.asScala.forall(_.getJobID != jobID)){
- sender ! decorateMessage(true)
- } else {
- context.system.scheduler.scheduleOnce(
- 200 milliseconds,
- self,
- decorateMessage(CheckIfJobRemoved(jobID)))(
- context.dispatcher,
- sender()
- )
- }
-
- case NotifyWhenJobManagerTerminated(jobManager) =>
- val waiting = waitForJobManagerToBeTerminated.getOrElse(jobManager.path.name, Set())
- waitForJobManagerToBeTerminated += jobManager.path.name -> (waiting + sender)
-
- case RegisterSubmitTaskListener(jobId) =>
- registeredSubmitTaskListeners.put(jobId, sender())
-
- case msg@SubmitTask(tdd) =>
- registeredSubmitTaskListeners.get(tdd.getJobID) match {
- case Some(listenerRef) =>
- listenerRef ! ResponseSubmitTaskListener(tdd)
- case None =>
- // Nothing to do
- }
-
- super.handleMessage(msg)
-
- /**
- * Message from task manager that accumulator values changed and need to be reported immediately
- * instead of lazily through the
- * [[org.apache.flink.runtime.messages.TaskManagerMessages.Heartbeat]] message. We forward this
- * message to the job manager that it knows it should report to the listeners.
- */
- case msg: AccumulatorsChanged =>
- currentJobManager match {
- case Some(jobManager) =>
- jobManager.forward(msg)
- sendHeartbeatToJobManager()
- sender ! true
- case None =>
- }
-
- case msg@Terminated(jobManager) =>
- super.handleMessage(msg)
-
- waitForJobManagerToBeTerminated.remove(jobManager.path.name) foreach {
- _ foreach {
- _ ! decorateMessage(JobManagerTerminated(jobManager))
- }
- }
-
- case msg:Disconnect =>
- if (!disconnectDisabled) {
- super.handleMessage(msg)
-
- val jobManager = sender()
-
- waitForJobManagerToBeTerminated.remove(jobManager.path.name) foreach {
- _ foreach {
- _ ! decorateMessage(JobManagerTerminated(jobManager))
- }
- }
- }
-
- case DisableDisconnect =>
- disconnectDisabled = true
-
- case NotifyOfComponentShutdown =>
- waitForShutdown += sender()
-
- case msg @ UpdateTaskExecutionState(taskExecutionState) =>
- super.handleMessage(msg)
-
- if(taskExecutionState.getExecutionState == ExecutionState.RUNNING) {
- waitForRunning.get(taskExecutionState.getID) foreach {
- _ foreach (_ ! decorateMessage(true))
- }
- }
-
- case RequestLeaderSessionID =>
- sender() ! ResponseLeaderSessionID(leaderSessionID.orNull)
-
- case NotifyWhenRegisteredAtJobManager(jobManager: ActorRef) =>
- if(isConnected && jobManager == currentJobManager.get) {
- sender() ! true
- } else {
- val list = waitForRegisteredAtResourceManager.getOrElse(
- jobManager,
- Set[ActorRef]())
-
- waitForRegisteredAtResourceManager += jobManager -> (list + sender())
- }
-
- case msg @ (_: AcknowledgeRegistration | _: AlreadyRegistered) =>
- super.handleMessage(msg)
-
- val jm = sender()
-
- waitForRegisteredAtResourceManager.remove(jm).foreach {
- listeners => listeners.foreach{
- listener =>
- listener ! true
- }
- }
- }
-
- /**
- * No killing of the VM for testing.
- */
- override protected def shutdown(): Unit = {
- log.info("Shutting down TestingJobManager.")
- waitForShutdown.foreach(_ ! ComponentShutdown(self))
- waitForShutdown.clear()
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingTaskManagerMessages.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingTaskManagerMessages.scala b/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingTaskManagerMessages.scala
deleted file mode 100644
index 974e4e8..0000000
--- a/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingTaskManagerMessages.scala
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.testingUtils
-
-import akka.actor.ActorRef
-import org.apache.flink.api.common.JobID
-import org.apache.flink.runtime.deployment.TaskDeploymentDescriptor
-import org.apache.flink.runtime.executiongraph.ExecutionAttemptID
-import org.apache.flink.runtime.taskmanager.Task
-
-/**
- * Additional messages that the [[TestingTaskManager]] understands.
- */
-object TestingTaskManagerMessages {
-
- case class NotifyWhenTaskRemoved(executionID: ExecutionAttemptID)
-
- case class NotifyWhenTaskIsRunning(executionID: ExecutionAttemptID)
-
- case class ResponseRunningTasks(tasks: Map[ExecutionAttemptID, Task]){
- import collection.JavaConverters._
- def asJava: java.util.Map[ExecutionAttemptID, Task] = tasks.asJava
- }
-
- case class ResponseBroadcastVariablesWithReferences(number: Int)
-
- case object RequestNumActiveConnections
- case class ResponseNumActiveConnections(number: Int)
-
- case object RequestRunningTasks
-
- case object RequestBroadcastVariablesWithReferences
-
- case class NotifyWhenJobManagerTerminated(jobManager: ActorRef)
-
- case class JobManagerTerminated(jobManager: ActorRef)
-
- case class NotifyWhenRegisteredAtJobManager(resourceManager: ActorRef)
-
- /**
- * Message to give a hint to the task manager that accumulator values were updated in the task.
- * This message is forwarded to the job manager which knows that it needs to notify listeners
- * of accumulator updates.
- */
- case class AccumulatorsChanged(jobID: JobID)
-
- /**
- * Registers a listener for all [[org.apache.flink.runtime.messages.TaskMessages.SubmitTask]]
- * messages of the given job.
- *
- * If a task is submitted with the given job ID the task deployment
- * descriptor is forwarded to the listener.
- *
- * @param jobId The job ID to listen for.
- */
- case class RegisterSubmitTaskListener(jobId: JobID)
-
- /**
- * A response to a listened job ID containing the submitted task deployment descriptor.
- *
- * @param tdd The submitted task deployment descriptor.
- */
- case class ResponseSubmitTaskListener(tdd: TaskDeploymentDescriptor)
-
- // --------------------------------------------------------------------------
- // Utility methods to allow simpler case object access from Java
- // --------------------------------------------------------------------------
-
- def getRequestRunningTasksMessage: AnyRef = {
- RequestRunningTasks
- }
-
- def getRequestBroadcastVariablesWithReferencesMessage: AnyRef = {
- RequestBroadcastVariablesWithReferences
- }
-
-}
-
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingUtils.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingUtils.scala b/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingUtils.scala
index 2f43d38..02a0fec 100644
--- a/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingUtils.scala
+++ b/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingUtils.scala
@@ -32,7 +32,6 @@ import org.apache.flink.configuration.{ConfigConstants, Configuration}
import org.apache.flink.runtime.client.JobClient
import org.apache.flink.runtime.clusterframework.FlinkResourceManager
import org.apache.flink.runtime.jobgraph.JobGraph
-import org.apache.flink.runtime.clusterframework.standalone.StandaloneResourceManager
import org.apache.flink.runtime.clusterframework.types.ResourceID
import org.apache.flink.runtime.jobmanager.{MemoryArchivist, JobManager}
import org.apache.flink.runtime.testutils.TestingResourceManager
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-scala-shell/pom.xml
----------------------------------------------------------------------
diff --git a/flink-scala-shell/pom.xml b/flink-scala-shell/pom.xml
index 2343485..bd935ee 100644
--- a/flink-scala-shell/pom.xml
+++ b/flink-scala-shell/pom.xml
@@ -34,28 +34,29 @@ under the License.
<dependencies>
+ <!-- core dependencies -->
+
<!-- scala command line parsing -->
<dependency>
<groupId>com.github.scopt</groupId>
<artifactId>scopt_${scala.binary.version}</artifactId>
</dependency>
-
<dependency>
<groupId>org.apache.flink</groupId>
- <artifactId>flink-runtime_2.10</artifactId>
+ <artifactId>flink-clients_2.10</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
- <artifactId>flink-clients_2.10</artifactId>
+ <artifactId>flink-scala_2.10</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
- <artifactId>flink-scala_2.10</artifactId>
+ <artifactId>flink-streaming-scala_2.10</artifactId>
<version>${project.version}</version>
</dependency>
@@ -70,27 +71,19 @@ under the License.
<artifactId>scala-library</artifactId>
<version>${scala.version}</version>
</dependency>
+
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-reflect</artifactId>
<version>${scala.version}</version>
</dependency>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-streaming-java_2.10</artifactId>
- <version>1.1-SNAPSHOT</version>
- </dependency>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-streaming-scala_2.10</artifactId>
- <version>1.1-SNAPSHOT</version>
- </dependency>
- <!-- tests -->
+
+ <!-- test dependencies -->
+
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-test-utils_2.10</artifactId>
<version>${project.version}</version>
- <type>test-jar</type>
<scope>test</scope>
</dependency>
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-scala/pom.xml
----------------------------------------------------------------------
diff --git a/flink-scala/pom.xml b/flink-scala/pom.xml
index 4a6a142..83e0f46 100644
--- a/flink-scala/pom.xml
+++ b/flink-scala/pom.xml
@@ -78,6 +78,14 @@ under the License.
<version>${guava.version}</version>
</dependency>
+ <!-- test dependencies -->
+
+ <dependency>
+ <groupId>org.scalatest</groupId>
+ <artifactId>scalatest_${scala.binary.version}</artifactId>
+ <scope>test</scope>
+ </dependency>
+
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-test-utils_2.10</artifactId>
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-streaming-connectors/flink-connector-elasticsearch/pom.xml
----------------------------------------------------------------------
diff --git a/flink-streaming-connectors/flink-connector-elasticsearch/pom.xml b/flink-streaming-connectors/flink-connector-elasticsearch/pom.xml
index 7ddf7e3..dd528be 100644
--- a/flink-streaming-connectors/flink-connector-elasticsearch/pom.xml
+++ b/flink-streaming-connectors/flink-connector-elasticsearch/pom.xml
@@ -42,6 +42,8 @@ under the License.
<dependencies>
+ <!-- core dependencies -->
+
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-streaming-java_2.10</artifactId>
@@ -49,42 +51,29 @@ under the License.
<scope>provided</scope>
</dependency>
- <dependency>
- <groupId>org.elasticsearch</groupId>
- <artifactId>elasticsearch</artifactId>
- <version>${elasticsearch.version}</version>
- </dependency>
-
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- <version>${guava.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-streaming-java_2.10</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
+ <dependency>
+ <groupId>org.elasticsearch</groupId>
+ <artifactId>elasticsearch</artifactId>
+ <version>${elasticsearch.version}</version>
+ </dependency>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-tests_2.10</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
+ <!-- test dependencies -->
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-test-utils_2.10</artifactId>
- <version>${project.version}</version>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-test-utils_2.10</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-streaming-java_2.10</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
<type>test-jar</type>
- <scope>test</scope>
- </dependency>
-
- </dependencies>
+ </dependency>
+ </dependencies>
<build>
<plugins>
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-streaming-connectors/flink-connector-elasticsearch2/pom.xml
----------------------------------------------------------------------
diff --git a/flink-streaming-connectors/flink-connector-elasticsearch2/pom.xml b/flink-streaming-connectors/flink-connector-elasticsearch2/pom.xml
index 090917b..4089cf6 100644
--- a/flink-streaming-connectors/flink-connector-elasticsearch2/pom.xml
+++ b/flink-streaming-connectors/flink-connector-elasticsearch2/pom.xml
@@ -18,8 +18,8 @@ specific language governing permissions and limitations
under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
@@ -42,6 +42,8 @@ under the License.
<dependencies>
+ <!-- core dependencies -->
+
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-streaming-java_2.10</artifactId>
@@ -49,40 +51,33 @@ under the License.
<scope>provided</scope>
</dependency>
- <dependency>
- <groupId>org.elasticsearch</groupId>
- <artifactId>elasticsearch</artifactId>
- <version>${elasticsearch.version}</version>
- </dependency>
+ <dependency>
+ <groupId>org.elasticsearch</groupId>
+ <artifactId>elasticsearch</artifactId>
+ <version>${elasticsearch.version}</version>
+ </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </dependency>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-streaming-java_2.10</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
+ <!-- core dependencies -->
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-tests_2.10</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-test-utils_2.10</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-test-utils_2.10</artifactId>
- <version>${project.version}</version>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-streaming-java_2.10</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
<type>test-jar</type>
- <scope>test</scope>
- </dependency>
-
- </dependencies>
+ </dependency>
+ </dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-streaming-connectors/flink-connector-filesystem/pom.xml
----------------------------------------------------------------------
diff --git a/flink-streaming-connectors/flink-connector-filesystem/pom.xml b/flink-streaming-connectors/flink-connector-filesystem/pom.xml
index 89ded0b..c5e903f 100644
--- a/flink-streaming-connectors/flink-connector-filesystem/pom.xml
+++ b/flink-streaming-connectors/flink-connector-filesystem/pom.xml
@@ -41,6 +41,8 @@ under the License.
<dependencies>
+ <!-- core dependencies -->
+
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-streaming-java_2.10</artifactId>
@@ -60,37 +62,36 @@ under the License.
<version>${project.version}</version>
</dependency>
+ <!-- test dependencies -->
+
<dependency>
<groupId>org.apache.flink</groupId>
- <artifactId>flink-streaming-java_2.10</artifactId>
+ <artifactId>flink-test-utils-junit</artifactId>
<version>${project.version}</version>
<scope>test</scope>
- <type>test-jar</type>
</dependency>
-
+
<dependency>
<groupId>org.apache.flink</groupId>
- <artifactId>flink-tests_2.10</artifactId>
+ <artifactId>flink-test-utils_2.10</artifactId>
<version>${project.version}</version>
<scope>test</scope>
- <type>test-jar</type>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
- <artifactId>flink-test-utils_2.10</artifactId>
+ <artifactId>flink-streaming-java_2.10</artifactId>
<version>${project.version}</version>
- <type>test-jar</type>
<scope>test</scope>
+ <type>test-jar</type>
</dependency>
-
<dependency>
<groupId>org.apache.flink</groupId>
- <artifactId>flink-runtime_2.10</artifactId>
+ <artifactId>flink-tests_2.10</artifactId>
+ <version>${project.version}</version>
<scope>test</scope>
<type>test-jar</type>
- <version>${project.version}</version>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-streaming-connectors/flink-connector-filesystem/src/test/java/org/apache/flink/streaming/connectors/fs/RollingSinkITCase.java
----------------------------------------------------------------------
diff --git a/flink-streaming-connectors/flink-connector-filesystem/src/test/java/org/apache/flink/streaming/connectors/fs/RollingSinkITCase.java b/flink-streaming-connectors/flink-connector-filesystem/src/test/java/org/apache/flink/streaming/connectors/fs/RollingSinkITCase.java
index b6c5bdb..dd2fc26 100644
--- a/flink-streaming-connectors/flink-connector-filesystem/src/test/java/org/apache/flink/streaming/connectors/fs/RollingSinkITCase.java
+++ b/flink-streaming-connectors/flink-connector-filesystem/src/test/java/org/apache/flink/streaming/connectors/fs/RollingSinkITCase.java
@@ -1,20 +1,21 @@
/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.flink.streaming.connectors.fs;
import org.apache.avro.Schema;
@@ -31,7 +32,7 @@ import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.common.functions.RichFilterFunction;
import org.apache.flink.api.common.functions.RichFlatMapFunction;
import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.runtime.taskmanager.MultiShotLatch;
+import org.apache.flink.core.testutils.MultiShotLatch;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.source.SourceFunction;
@@ -39,6 +40,7 @@ import org.apache.flink.streaming.connectors.fs.AvroKeyValueSinkWriter.AvroKeyVa
import org.apache.flink.streaming.util.StreamingMultipleProgramsTestBase;
import org.apache.flink.util.Collector;
import org.apache.flink.util.NetUtils;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.LocatedFileStatus;
@@ -48,6 +50,7 @@ import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.Text;
+
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-streaming-connectors/flink-connector-kafka-0.8/pom.xml
----------------------------------------------------------------------
diff --git a/flink-streaming-connectors/flink-connector-kafka-0.8/pom.xml b/flink-streaming-connectors/flink-connector-kafka-0.8/pom.xml
index 3557f1c..79fa7bd 100644
--- a/flink-streaming-connectors/flink-connector-kafka-0.8/pom.xml
+++ b/flink-streaming-connectors/flink-connector-kafka-0.8/pom.xml
@@ -41,25 +41,27 @@ under the License.
</properties>
<dependencies>
+
+ <!-- core dependencies -->
+
<dependency>
<groupId>org.apache.flink</groupId>
- <artifactId>flink-connector-kafka-base_2.10</artifactId>
+ <artifactId>flink-streaming-java_2.10</artifactId>
<version>${project.version}</version>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
- <artifactId>flink-connector-kafka-base_2.10</artifactId>
+ <artifactId>flink-shaded-curator-recipes</artifactId>
<version>${project.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
- <artifactId>flink-streaming-java_2.10</artifactId>
+ <artifactId>flink-connector-kafka-base_2.10</artifactId>
<version>${project.version}</version>
- <scope>provided</scope>
</dependency>
<dependency>
@@ -116,6 +118,8 @@ under the License.
</exclusions>
</dependency>
+ <!-- test dependencies -->
+
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-test</artifactId>
@@ -125,16 +129,16 @@ under the License.
<dependency>
<groupId>org.apache.flink</groupId>
- <artifactId>flink-shaded-curator-recipes</artifactId>
+ <artifactId>flink-connector-kafka-base_2.10</artifactId>
<version>${project.version}</version>
- <scope>provided</scope>
+ <type>test-jar</type>
+ <scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-test-utils_2.10</artifactId>
<version>${project.version}</version>
- <type>test-jar</type>
<scope>test</scope>
</dependency>
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-streaming-connectors/flink-connector-kafka-0.9/pom.xml
----------------------------------------------------------------------
diff --git a/flink-streaming-connectors/flink-connector-kafka-0.9/pom.xml b/flink-streaming-connectors/flink-connector-kafka-0.9/pom.xml
index 8feadb5..8ad909a 100644
--- a/flink-streaming-connectors/flink-connector-kafka-0.9/pom.xml
+++ b/flink-streaming-connectors/flink-connector-kafka-0.9/pom.xml
@@ -42,6 +42,15 @@ under the License.
<dependencies>
+ <!-- core dependencies -->
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-streaming-java_2.10</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-connector-kafka-base_2.10</artifactId>
@@ -56,6 +65,24 @@ under the License.
<dependency>
<groupId>org.apache.flink</groupId>
+ <artifactId>flink-table_2.10</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ <!-- Projects depending on this project,
+ won't depend on flink-table. -->
+ <optional>true</optional>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>kafka-clients</artifactId>
+ <version>${kafka.version}</version>
+ </dependency>
+
+ <!-- test dependencies -->
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
<artifactId>flink-connector-kafka-base_2.10</artifactId>
<version>${project.version}</version>
<exclusions>
@@ -70,12 +97,6 @@ under the License.
</dependency>
<dependency>
- <groupId>org.apache.kafka</groupId>
- <artifactId>kafka-clients</artifactId>
- <version>${kafka.version}</version>
- </dependency>
-
- <dependency>
<!-- include 0.9 server for tests -->
<groupId>org.apache.kafka</groupId>
<artifactId>kafka_${scala.binary.version}</artifactId>
@@ -93,31 +114,12 @@ under the License.
<dependency>
<groupId>org.apache.flink</groupId>
- <artifactId>flink-streaming-java_2.10</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-table_2.10</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- <!-- Projects depending on this project,
- won't depend on flink-table. -->
- <optional>true</optional>
- </dependency>
-
- <dependency>
- <groupId>org.apache.flink</groupId>
<artifactId>flink-test-utils_2.10</artifactId>
<version>${project.version}</version>
- <type>test-jar</type>
<scope>test</scope>
</dependency>
</dependencies>
-
<build>
<plugins>
@@ -130,30 +132,30 @@ under the License.
<goal>test-jar</goal>
</goals>
<configuration>
- <includes>
- <include>**/KafkaTestEnvironmentImpl*</include>
- </includes>
- </configuration>
+ <includes>
+ <include>**/KafkaTestEnvironmentImpl*</include>
+ </includes>
+ </configuration>
</execution>
</executions>
</plugin>
<plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-source-plugin</artifactId>
- <executions>
- <execution>
- <id>attach-test-sources</id>
- <goals>
- <goal>test-jar-no-fork</goal>
- </goals>
- <configuration>
- <includes>
- <include>**/KafkaTestEnvironmentImpl*</include>
- </includes>
- </configuration>
- </execution>
- </executions>
- </plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-source-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>attach-test-sources</id>
+ <goals>
+ <goal>test-jar-no-fork</goal>
+ </goals>
+ <configuration>
+ <includes>
+ <include>**/KafkaTestEnvironmentImpl*</include>
+ </includes>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-streaming-connectors/flink-connector-kafka-base/pom.xml
----------------------------------------------------------------------
diff --git a/flink-streaming-connectors/flink-connector-kafka-base/pom.xml b/flink-streaming-connectors/flink-connector-kafka-base/pom.xml
index 1cbb554..258a2cf 100644
--- a/flink-streaming-connectors/flink-connector-kafka-base/pom.xml
+++ b/flink-streaming-connectors/flink-connector-kafka-base/pom.xml
@@ -42,6 +42,8 @@ under the License.
<dependencies>
+ <!-- core dependencies -->
+
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-streaming-java_2.10</artifactId>
@@ -103,14 +105,18 @@ under the License.
</exclusions>
</dependency>
+ <!-- test dependencies -->
+
<!-- force using the latest zkclient -->
<dependency>
<groupId>com.101tec</groupId>
<artifactId>zkclient</artifactId>
<version>0.7</version>
<type>jar</type>
+ <scope>test</scope>
</dependency>
+
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-test</artifactId>
@@ -120,7 +126,7 @@ under the License.
<dependency>
<groupId>org.apache.flink</groupId>
- <artifactId>flink-tests_2.10</artifactId>
+ <artifactId>flink-streaming-java_2.10</artifactId>
<version>${project.version}</version>
<type>test-jar</type>
<scope>test</scope>
@@ -130,6 +136,21 @@ under the License.
<groupId>org.apache.flink</groupId>
<artifactId>flink-test-utils_2.10</artifactId>
<version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-tests_2.10</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-runtime_2.10</artifactId>
+ <version>${project.version}</version>
<type>test-jar</type>
<scope>test</scope>
</dependency>
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-streaming-java/pom.xml
----------------------------------------------------------------------
diff --git a/flink-streaming-java/pom.xml b/flink-streaming-java/pom.xml
index 5ee8001..88150b3 100644
--- a/flink-streaming-java/pom.xml
+++ b/flink-streaming-java/pom.xml
@@ -36,6 +36,8 @@ under the License.
<dependencies>
+ <!-- core dependencies -->
+
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-core</artifactId>
@@ -56,16 +58,8 @@ under the License.
<dependency>
<groupId>org.apache.commons</groupId>
- <artifactId>commons-math</artifactId>
- <version>2.2</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-test-utils_2.10</artifactId>
- <version>${project.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
+ <artifactId>commons-math3</artifactId>
+ <version>3.5</version>
</dependency>
<dependency>
@@ -80,13 +74,24 @@ under the License.
<version>${guava.version}</version>
</dependency>
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-math3</artifactId>
- <version>3.5</version>
- </dependency>
+ <!-- test dependencies -->
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-test-utils_2.10</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-runtime_2.10</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
- </dependencies>
+ </dependencies>
<build>
<plugins>
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/RestartStrategyTest.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/RestartStrategyTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/RestartStrategyTest.java
index f768ace..7bb726d 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/RestartStrategyTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/RestartStrategyTest.java
@@ -18,11 +18,11 @@
package org.apache.flink.streaming.api;
-import org.apache.flink.api.common.ExecutionConfigTest;
import org.apache.flink.api.common.restartstrategy.RestartStrategies;
import org.apache.flink.runtime.jobgraph.JobGraph;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.graph.StreamGraph;
+
import org.junit.Assert;
import org.junit.Test;
@@ -34,9 +34,9 @@ public class RestartStrategyTest {
* strategy has been specified
*/
@Test
- public void testAutomaticRestartingWhenCheckpointing() {
+ public void testAutomaticRestartingWhenCheckpointing() throws Exception {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
- env.enableCheckpointing();
+ env.enableCheckpointing(500);
env.fromElements(1).print();
@@ -44,7 +44,7 @@ public class RestartStrategyTest {
JobGraph jobGraph = graph.getJobGraph();
RestartStrategies.RestartStrategyConfiguration restartStrategy =
- ExecutionConfigTest.deserializeConfig(jobGraph.getSerializedExecutionConfig()).getRestartStrategy();
+ jobGraph.getSerializedExecutionConfig().deserializeValue(getClass().getClassLoader()).getRestartStrategy();
Assert.assertNotNull(restartStrategy);
Assert.assertTrue(restartStrategy instanceof RestartStrategies.FixedDelayRestartStrategyConfiguration);
@@ -56,9 +56,9 @@ public class RestartStrategyTest {
* of execution retries is set to 0, restarting is deactivated
*/
@Test
- public void testNoRestartingWhenCheckpointingAndExplicitExecutionRetriesZero() {
+ public void testNoRestartingWhenCheckpointingAndExplicitExecutionRetriesZero() throws Exception {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
- env.enableCheckpointing();
+ env.enableCheckpointing(500);
env.setNumberOfExecutionRetries(0);
env.fromElements(1).print();
@@ -67,7 +67,7 @@ public class RestartStrategyTest {
JobGraph jobGraph = graph.getJobGraph();
RestartStrategies.RestartStrategyConfiguration restartStrategy =
- ExecutionConfigTest.deserializeConfig(jobGraph.getSerializedExecutionConfig()).getRestartStrategy();
+ jobGraph.getSerializedExecutionConfig().deserializeValue(getClass().getClassLoader()).getRestartStrategy();
Assert.assertNotNull(restartStrategy);
Assert.assertTrue(restartStrategy instanceof RestartStrategies.NoRestartStrategyConfiguration);
@@ -78,9 +78,9 @@ public class RestartStrategyTest {
* of execution retries is set to 42 and the delay to 1337, fixed delay restarting is used.
*/
@Test
- public void testFixedRestartingWhenCheckpointingAndExplicitExecutionRetriesNonZero() {
+ public void testFixedRestartingWhenCheckpointingAndExplicitExecutionRetriesNonZero() throws Exception {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
- env.enableCheckpointing();
+ env.enableCheckpointing(500);
env.setNumberOfExecutionRetries(42);
env.getConfig().setExecutionRetryDelay(1337);
@@ -90,7 +90,7 @@ public class RestartStrategyTest {
JobGraph jobGraph = graph.getJobGraph();
RestartStrategies.RestartStrategyConfiguration restartStrategy =
- ExecutionConfigTest.deserializeConfig(jobGraph.getSerializedExecutionConfig()).getRestartStrategy();
+ jobGraph.getSerializedExecutionConfig().deserializeValue(getClass().getClassLoader()).getRestartStrategy();
Assert.assertNotNull(restartStrategy);
Assert.assertTrue(restartStrategy instanceof RestartStrategies.FixedDelayRestartStrategyConfiguration);
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGeneratorTest.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGeneratorTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGeneratorTest.java
index 0de4325..7f94aa0 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGeneratorTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGeneratorTest.java
@@ -21,7 +21,6 @@ import java.io.IOException;
import java.util.Random;
import org.apache.flink.api.common.ExecutionConfig;
-import org.apache.flink.api.common.ExecutionConfigTest;
import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.runtime.jobgraph.JobGraph;
@@ -29,12 +28,13 @@ import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.sink.SinkFunction;
import org.apache.flink.util.InstantiationUtil;
-
import org.apache.flink.util.SerializedValue;
+
import org.junit.Test;
import static org.junit.Assert.*;
+@SuppressWarnings("serial")
public class StreamingJobGraphGeneratorTest {
@Test
@@ -93,7 +93,7 @@ public class StreamingJobGraphGeneratorTest {
assertNotNull(serializedExecutionConfig);
- ExecutionConfig executionConfig = ExecutionConfigTest.deserializeConfig(serializedExecutionConfig);
+ ExecutionConfig executionConfig = serializedExecutionConfig.deserializeValue(getClass().getClassLoader());
assertEquals(closureCleanerEnabled, executionConfig.isClosureCleanerEnabled());
assertEquals(forceAvroEnabled, executionConfig.isForceAvroEnabled());
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/partitioner/RescalePartitionerTest.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/partitioner/RescalePartitionerTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/partitioner/RescalePartitionerTest.java
index 8484e90..df4efdb 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/partitioner/RescalePartitionerTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/partitioner/RescalePartitionerTest.java
@@ -17,7 +17,7 @@
package org.apache.flink.streaming.runtime.partitioner;
-import org.apache.flink.api.common.ExecutionConfigTest;
+import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.api.common.JobID;
import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.java.tuple.Tuple;
@@ -40,6 +40,7 @@ import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.source.ParallelSourceFunction;
import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
import org.apache.flink.util.Collector;
+import org.apache.flink.util.SerializedValue;
import org.apache.flink.util.TestLogger;
import org.junit.Before;
import org.junit.Test;
@@ -54,6 +55,7 @@ import java.util.Set;
import static org.junit.Assert.*;
+@SuppressWarnings("serial")
public class RescalePartitionerTest extends TestLogger {
private RescalePartitioner<Tuple> distributePartitioner;
@@ -84,7 +86,7 @@ public class RescalePartitionerTest extends TestLogger {
}
@Test
- public void testExecutionGraphGeneration() {
+ public void testExecutionGraphGeneration() throws Exception {
final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(4);
@@ -135,7 +137,7 @@ public class RescalePartitionerTest extends TestLogger {
jobId,
jobName,
cfg,
- ExecutionConfigTest.getSerializedConfig(),
+ new SerializedValue<>(new ExecutionConfig()),
AkkaUtils.getDefaultTimeout(),
new NoRestartStrategy(),
new ArrayList<BlobKey>(),
[04/15] flink git commit: [FLINK-3995] [build] Properly structure
test scopes and dependencies
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/testdata/WebLogAnalysisData.java
----------------------------------------------------------------------
diff --git a/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/testdata/WebLogAnalysisData.java b/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/testdata/WebLogAnalysisData.java
new file mode 100644
index 0000000..ffbf7d1
--- /dev/null
+++ b/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/testdata/WebLogAnalysisData.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.test.testdata;
+
+
+public class WebLogAnalysisData {
+ public static final String DOCS =
+ "url_10|volutpat magna quis consectetuer volutpat ad erat editors exerci oscillations euismod volutpat Lorem convectionullamcorper Lorem volutpat enim tation elit |\n" +
+ "url_11|adipiscing enim diam ex tincidunt tincidunt nonummy volutpat minim euismod volutpat suscipit ex sed laoreet aliquip quis diam tincidunt wisi diam elit sed ut minim ad nonummy amet volutpat nostrud erat |\n" +
+ "url_12|ut nostrud adipiscing adipiscing ipsum nonummy amet volutpat volutpat sit enim Ut amet |\n" +
+ "url_13|euismod sit adipiscing ex suscipit ea veniam tincidunt laoreet nibh editors ullamcorper consectetuer convection commodo sed nostrud ea ex ullamcorper dolor dolore ad diam dolore amet ut tincidunt nonummy euismod enim ullamcorper tincidunt dolor sit volutpat dolor tincidunt aliquam nisl tation ullamcorper sed consectetuer sit sit laoreet ex |\n" +
+ "url_14|ad magna ipsum nonummy aliquip dolore aliquam veniam lobortis nostrud aliquip nibh amet aliquam editors magna aliquam volutpat nonummy sed tation erat adipiscing nostrud magna ut sit dolore volutpat laoreet nisl wisi Ut veniam nibh laoreet ad nostrud ut aliquip |\n" +
+ "url_15|enim ipsum veniam ex editor Lorem elit laoreet exerci ea wisi oscillations convection euismod euismod diam ut euismod ad Lorem ut Ut tation wisi diam suscipit nibh nostrud minim dolor |\n" +
+ "url_16|tation ad sed veniam lobortis editor nonummy Ut ea ipsum aliquip dolore ut laoreet tation ad ut Lorem ipsum minim nostrud quis Lorem enim |\n" +
+ "url_17|veniam editor veniam Lorem aliquam ipsum amet ut convection veniam enim commodo ad ex magna nibh Ut ex sed ut nostrud amet volutpat nibh Ut wisi ea laoreet Lorem amet minim ullamcorper tincidunt veniam laoreet laoreet |\n" +
+ "url_18|veniam diam tincidunt ullamcorper adipiscing oscillations adipiscing aliquip quis nibh suscipit ex sit sit wisi diam aliquam aliquip ea diam euismod ad erat ut ipsum lobortis ea exerci |\n" +
+ "url_19|adipiscing nisl oscillations ea editors tincidunt tation convection tincidunt tincidunt ut wisi tincidunt ut ut Ut nibh aliquam laoreet exerci enim ea nibh erat nonummy |\n" +
+ "url_20|ut diam aliquip ipsum laoreet elit volutpat suscipit nostrud convection tation nisl suscipit nonummy tation tation ut enim dolor nisl magna aliquam enim consectetuer sed tincidunt quis amet elit |\n" +
+ "url_21|tation ad exerci volutpat tincidunt sit ullamcorper oscillations ut minim sed diam Lorem adipiscing |\n" +
+ "url_22|sit exerci ad ut minim convection ad ea dolore Lorem ipsum amet sit wisi sed ullamcorper ipsum enim aliquam tincidunt elit nonummy laoreet laoreet dolore tation ullamcorper commodo Ut elit sit minim suscipit nisl laoreet |\n" +
+ "url_23|Lorem nibh ea ea ex ut tation euismod tincidunt lobortis enim ullamcorper euismod amet magna sit erat enim editor diam dolor volutpat diam nisl ut erat commodo amet veniam consectetuer ex commodo sed magna ea erat ullamcorper dolor sed diam enim euismod |\n" +
+ "url_24|nisl dolor magna editors ad consectetuer veniam commodo Ut dolor wisi dolore amet dolore dolore volutpat convection Ut oscillations |\n" +
+ "url_25|ut euismod quis ad nostrud volutpat dolor wisi oscillations nostrud sed nonummy wisi exerci convection sed erat nostrud quis editors Ut nostrud consectetuer aliquip exerci ut Ut tincidunt aliquam suscipit erat |\n" +
+ "url_26|nisl elit ea minim aliquam dolor consectetuer consectetuer quis Ut convection laoreet sit enim nostrud sed dolor magna dolor elit adipiscing |\n" +
+ "url_27|nisl euismod ipsum ex adipiscing erat euismod diam quis oscillations aliquip nisl ut sit dolor wisi enim tincidunt amet ullamcorper adipiscing nibh Ut volutpat sed nonummy ex ea wisi exerci aliquam elit Ut aliquip nostrud ad nibh ut sit suscipit ut commodo wisi Lorem nibh |\n" +
+ "url_28|laoreet sed editor dolor diam convection sed diam exerci laoreet oscillations consectetuer ullamcorper suscipit ut editors quis commodo editor veniam nibh ea diam ex magna ea elit sit sed nibh lobortis consectetuer erat erat sit minim ea ad ea nisl magna volutpat ut |\n" +
+ "url_29|consectetuer convection amet diam erat euismod erat editor oscillations ipsum tation tation editors minim wisi tation quis adipiscing euismod nonummy erat ut diam suscipit tincidunt ut consectetuer Lorem editor sit quis euismod veniam tincidunt aliquam ut veniam adipiscing magna Ut ut dolore euismod minim veniam |";
+
+ public static final String RANKS =
+ "77|url_10|51|\n" + "7|url_11|24|\n" + "19|url_12|28|\n" + "13|url_13|50|\n" +
+ "81|url_14|41|\n" + "87|url_15|28|\n" + "78|url_16|29|\n" + "70|url_17|7|\n" +
+ "91|url_18|38|\n" + "99|url_19|11|\n" + "37|url_20|46|\n" + "64|url_21|46|\n" +
+ "25|url_22|14|\n" + "6|url_23|12|\n" + "87|url_24|39|\n" + "0|url_25|27|\n" +
+ "97|url_26|27|\n" + "9|url_27|54|\n" + "59|url_28|41|\n" + "40|url_29|11|";
+
+ public static final String VISITS =
+ "112.99.215.248|url_20|2011-5-1|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "189.69.147.166|url_19|2009-8-2|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "83.7.97.153|url_21|2010-3-20|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "215.57.124.59|url_10|2008-1-26|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "174.176.121.10|url_25|2007-12-17|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "66.225.121.11|url_16|2007-10-8|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "57.11.211.126|url_14|2010-12-6|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "129.127.115.129|url_28|2009-2-5|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "137.249.175.115|url_27|2011-5-7|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "101.92.9.112|url_12|2012-1-10|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "113.108.207.12|url_24|2011-3-20|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "179.141.87.220|url_18|2007-10-3|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "46.10.118.101|url_20|2009-7-4|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "91.67.84.155|url_20|2012-7-7|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "102.142.34.164|url_22|2010-7-18|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "203.67.94.61|url_14|2007-7-27|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "229.121.13.143|url_19|2010-4-17|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "209.154.189.12|url_16|2011-4-24|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "57.113.29.197|url_13|2008-9-23|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "132.90.139.12|url_16|2007-10-17|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "101.21.109.146|url_29|2009-2-24|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "171.69.34.255|url_29|2009-2-1|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "11.114.192.13|url_25|2008-9-8|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "37.188.212.141|url_15|2011-1-21|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "102.40.251.47|url_26|2010-7-17|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "136.85.230.245|url_17|2010-8-12|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "68.96.194.82|url_29|2007-10-12|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "62.126.40.210|url_29|2011-2-13|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "12.81.184.191|url_11|2008-1-12|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "12.232.205.52|url_18|2008-1-26|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "227.83.79.30|url_25|2008-6-4|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "88.222.17.152|url_21|2007-10-1|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "55.87.23.63|url_22|2008-8-16|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "54.41.77.25|url_18|2012-12-18|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "203.83.250.233|url_10|2009-4-5|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "42.13.134.196|url_15|2012-9-23|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "224.51.182.133|url_27|2008-11-13|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "180.209.212.98|url_27|2008-6-27|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "29.173.47.29|url_24|2011-9-18|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "251.153.164.73|url_18|2008-3-6|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "195.164.244.58|url_23|2010-12-21|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "96.167.244.51|url_18|2007-2-10|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "131.169.48.126|url_19|2009-6-10|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "171.151.109.83|url_12|2011-6-24|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "150.87.176.48|url_22|2008-10-3|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "56.172.201.54|TPCHQuery3ITCaseurl_25|2012-11-3|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "175.241.83.44|url_13|2008-2-22|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "105.222.1.251|url_18|2012-7-6|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "195.117.130.95|url_10|2012-4-24|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "10.139.236.176|url_14|2011-4-9|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "207.100.168.191|url_22|2007-5-1|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "196.129.126.177|url_25|2008-4-2|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "170.239.116.200|url_27|2010-3-15|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "99.79.118.85|url_13|2008-3-12|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "112.65.132.135|url_15|2010-8-3|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "109.15.235.196|url_14|2012-8-25|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "240.201.229.137|url_21|2008-9-22|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "74.53.245.171|url_18|2007-6-19|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "158.230.101.250|url_17|2011-4-17|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "198.197.213.210|url_11|2012-5-1|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "1.54.145.90|url_11|2009-6-16|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "2.236.14.86|url_13|2007-9-7|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "203.180.186.150|url_16|2010-2-17|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "135.83.247.101|url_22|2007-12-7|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "54.214.72.16|url_12|2007-5-2|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "238.115.186.34|url_10|2007-5-24|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "6.251.55.92|url_26|2007-5-18|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "196.96.11.111|url_16|2007-11-7|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "209.40.86.220|url_24|2009-5-17|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "130.208.66.140|url_19|2011-9-22|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "176.157.170.178|url_27|2008-12-12|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "103.4.119.128|url_15|2010-6-19|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "31.181.160.159|url_12|2009-8-10|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "166.238.60.79|url_23|2009-2-1|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "64.165.114.139|url_19|2007-9-17|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "11.45.238.55|url_26|2011-8-17|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "189.121.92.228|url_18|2008-1-16|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "137.114.181.208|url_14|2007-6-15|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "165.194.234.136|url_18|2012-3-7|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "210.145.85.234|url_10|2007-9-3|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "227.37.135.138|url_21|2010-5-4|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "39.87.120.86|url_25|2007-2-6|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "107.144.159.246|url_11|2010-7-6|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "160.38.166.236|url_15|2009-6-18|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "79.44.39.160|url_10|2008-12-19|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "143.52.62.175|url_19|2011-12-8|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "68.54.137.193|url_19|2007-6-6|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "199.137.128.158|url_27|2008-4-26|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "10.0.80.61|url_10|2011-3-15|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "4.30.48.164|url_29|2007-2-26|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "136.29.50.223|url_12|2008-3-5|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "83.101.200.110|url_23|2009-1-26|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "11.149.83.246|url_29|2010-9-18|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "154.119.20.134|url_14|2011-1-4|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "191.124.167.123|url_15|2009-6-24|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "27.158.167.38|url_11|2011-5-16|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "16.197.156.206|url_16|2009-2-1|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "222.150.206.233|url_20|2012-2-11|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "198.59.29.225|url_21|2007-2-25|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "182.91.244.40|url_18|2010-5-7|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "23.110.45.72|url_28|2008-6-23|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "94.241.8.36|url_11|2007-7-4|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "226.223.202.210|url_17|2007-10-15|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "73.16.194.132|url_14|2012-7-2|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "195.45.98.110|url_13|2012-2-23|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "189.123.80.178|url_29|2011-1-1|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "190.134.174.18|url_23|2011-12-17|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "133.54.252.245|url_13|2011-3-13|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "244.29.167.182|url_18|2012-10-11|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "125.32.251.232|url_26|2010-4-22|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "74.139.30.20|url_16|2012-2-8|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "100.73.180.209|url_29|2010-4-4|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "133.155.158.147|url_16|2011-11-22|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "185.53.212.102|url_15|2010-12-19|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "154.189.125.92|url_27|2007-10-5|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "12.209.166.107|url_26|2007-5-1|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "106.99.152.22|url_25|2011-12-4|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "37.120.32.53|url_22|2008-12-8|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "29.17.125.185|url_12|2011-8-18|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "233.125.12.14|url_19|2011-2-7|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "241.78.212.114|url_25|2007-12-22|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "153.183.247.90|url_18|2011-9-23|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "245.43.17.177|url_15|2012-4-21|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "55.13.54.220|url_24|2008-11-17|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "104.149.82.217|url_10|2011-9-26|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "36.15.167.145|url_24|2008-1-9|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "145.149.81.203|url_20|2008-5-27|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "68.89.4.115|url_18|2007-7-17|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "236.88.134.53|url_15|2010-5-2|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "180.140.58.209|url_29|2012-8-5|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "138.46.241.237|url_17|2010-11-13|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "254.217.79.58|url_29|2007-1-11|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "142.184.197.136|url_10|2007-11-3|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "216.118.94.220|url_12|2007-2-18|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "53.99.54.0|url_14|2008-3-9|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "95.151.221.37|url_28|2009-5-2|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "46.110.43.78|url_27|2007-11-9|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "172.96.203.180|url_11|2012-1-17|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "22.188.231.143|url_21|2007-10-23|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "216.46.80.239|url_22|2009-9-18|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "103.213.7.86|url_26|2007-12-25|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "186.204.3.180|url_27|2012-1-22|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "129.197.116.192|url_10|2012-5-1|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "8.164.181.242|url_24|2008-11-17|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "145.52.12.132|url_28|2012-8-6|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "20.22.62.146|url_11|2007-4-27|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "233.255.32.151|url_19|2008-6-25|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "31.233.226.148|url_24|2011-8-4|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "57.130.120.117|url_25|2008-12-9|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "29.98.249.152|url_21|2011-8-7|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "25.187.66.122|url_20|2008-12-17|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "236.205.44.215|url_26|2008-9-13|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "18.40.99.157|url_23|2012-1-7|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "204.221.88.206|url_24|2012-11-8|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "199.5.94.240|url_24|2011-9-24|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "57.175.34.24|url_14|2009-12-10|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "130.25.116.220|url_10|2007-6-5|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "134.174.12.250|url_22|2011-6-22|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "162.185.114.212|url_13|2011-8-24|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "8.113.137.95|url_14|2011-6-22|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "176.253.5.99|url_12|2010-4-12|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "3.110.24.0|url_12|2009-6-18|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "159.128.152.207|url_17|2012-2-2|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "104.145.150.244|url_10|2012-2-13|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "203.19.224.207|url_12|2012-9-24|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "225.163.125.108|url_20|2010-4-8|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "197.137.29.71|url_21|2007-10-25|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "29.207.144.144|url_12|2012-7-13|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "139.222.222.113|url_14|2010-4-15|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "169.225.20.141|url_29|2012-2-18|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "137.86.141.104|url_26|2012-3-10|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "193.100.2.122|url_24|2011-12-15|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "20.105.209.161|url_13|2010-2-7|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "116.81.52.207|url_11|2007-4-13|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "141.88.224.116|url_19|2009-4-12|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "72.198.58.37|url_26|2012-2-2|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "13.37.92.90|url_22|2011-5-16|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "191.163.173.142|url_24|2012-5-2|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "183.57.175.105|url_27|2007-8-13|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "230.212.34.87|url_10|2010-12-21|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
+ "133.150.217.96|url_24|2012-7-27|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "62.254.96.239|url_24|2011-5-15|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|";
+
+ public static final String EXCEPTED_RESULT = "87|url_24|39\n" + "59|url_28|41\n";
+
+ private WebLogAnalysisData() {}
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/testdata/WordCountData.java
----------------------------------------------------------------------
diff --git a/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/testdata/WordCountData.java b/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/testdata/WordCountData.java
new file mode 100644
index 0000000..23c649b
--- /dev/null
+++ b/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/testdata/WordCountData.java
@@ -0,0 +1,329 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.test.testdata;
+
+
+/**
+ *
+ */
+public class WordCountData {
+
+ public static final String TEXT = "Goethe - Faust: Der Tragoedie erster Teil\n" + "Prolog im Himmel.\n"
+ + "Der Herr. Die himmlischen Heerscharen. Nachher Mephistopheles. Die drei\n" + "Erzengel treten vor.\n"
+ + "RAPHAEL: Die Sonne toent, nach alter Weise, In Brudersphaeren Wettgesang,\n"
+ + "Und ihre vorgeschriebne Reise Vollendet sie mit Donnergang. Ihr Anblick\n"
+ + "gibt den Engeln Staerke, Wenn keiner Sie ergruenden mag; die unbegreiflich\n"
+ + "hohen Werke Sind herrlich wie am ersten Tag.\n"
+ + "GABRIEL: Und schnell und unbegreiflich schnelle Dreht sich umher der Erde\n"
+ + "Pracht; Es wechselt Paradieseshelle Mit tiefer, schauervoller Nacht. Es\n"
+ + "schaeumt das Meer in breiten Fluessen Am tiefen Grund der Felsen auf, Und\n"
+ + "Fels und Meer wird fortgerissen Im ewig schnellem Sphaerenlauf.\n"
+ + "MICHAEL: Und Stuerme brausen um die Wette Vom Meer aufs Land, vom Land\n"
+ + "aufs Meer, und bilden wuetend eine Kette Der tiefsten Wirkung rings umher.\n"
+ + "Da flammt ein blitzendes Verheeren Dem Pfade vor des Donnerschlags. Doch\n"
+ + "deine Boten, Herr, verehren Das sanfte Wandeln deines Tags.\n"
+ + "ZU DREI: Der Anblick gibt den Engeln Staerke, Da keiner dich ergruenden\n"
+ + "mag, Und alle deine hohen Werke Sind herrlich wie am ersten Tag.\n"
+ + "MEPHISTOPHELES: Da du, o Herr, dich einmal wieder nahst Und fragst, wie\n"
+ + "alles sich bei uns befinde, Und du mich sonst gewoehnlich gerne sahst, So\n"
+ + "siehst du mich auch unter dem Gesinde. Verzeih, ich kann nicht hohe Worte\n"
+ + "machen, Und wenn mich auch der ganze Kreis verhoehnt; Mein Pathos braechte\n"
+ + "dich gewiss zum Lachen, Haettst du dir nicht das Lachen abgewoehnt. Von\n"
+ + "Sonn' und Welten weiss ich nichts zu sagen, Ich sehe nur, wie sich die\n"
+ + "Menschen plagen. Der kleine Gott der Welt bleibt stets von gleichem\n"
+ + "Schlag, Und ist so wunderlich als wie am ersten Tag. Ein wenig besser\n"
+ + "wuerd er leben, Haettst du ihm nicht den Schein des Himmelslichts gegeben;\n"
+ + "Er nennt's Vernunft und braucht's allein, Nur tierischer als jedes Tier\n"
+ + "zu sein. Er scheint mir, mit Verlaub von euer Gnaden, Wie eine der\n"
+ + "langbeinigen Zikaden, Die immer fliegt und fliegend springt Und gleich im\n"
+ + "Gras ihr altes Liedchen singt; Und laeg er nur noch immer in dem Grase! In\n"
+ + "jeden Quark begraebt er seine Nase.\n"
+ + "DER HERR: Hast du mir weiter nichts zu sagen? Kommst du nur immer\n"
+ + "anzuklagen? Ist auf der Erde ewig dir nichts recht?\n"
+ + "MEPHISTOPHELES: Nein Herr! ich find es dort, wie immer, herzlich\n"
+ + "schlecht. Die Menschen dauern mich in ihren Jammertagen, Ich mag sogar\n"
+ + "die armen selbst nicht plagen.\n" + "DER HERR: Kennst du den Faust?\n" + "MEPHISTOPHELES: Den Doktor?\n"
+ + "DER HERR: Meinen Knecht!\n"
+ + "MEPHISTOPHELES: Fuerwahr! er dient Euch auf besondre Weise. Nicht irdisch\n"
+ + "ist des Toren Trank noch Speise. Ihn treibt die Gaerung in die Ferne, Er\n"
+ + "ist sich seiner Tollheit halb bewusst; Vom Himmel fordert er die schoensten\n"
+ + "Sterne Und von der Erde jede hoechste Lust, Und alle Naeh und alle Ferne\n"
+ + "Befriedigt nicht die tiefbewegte Brust.\n"
+ + "DER HERR: Wenn er mir auch nur verworren dient, So werd ich ihn bald in\n"
+ + "die Klarheit fuehren. Weiss doch der Gaertner, wenn das Baeumchen gruent, Das\n"
+ + "Bluet und Frucht die kuenft'gen Jahre zieren.\n"
+ + "MEPHISTOPHELES: Was wettet Ihr? den sollt Ihr noch verlieren! Wenn Ihr\n"
+ + "mir die Erlaubnis gebt, Ihn meine Strasse sacht zu fuehren.\n"
+ + "DER HERR: Solang er auf der Erde lebt, So lange sei dir's nicht verboten,\n"
+ + "Es irrt der Mensch so lang er strebt.\n"
+ + "MEPHISTOPHELES: Da dank ich Euch; denn mit den Toten Hab ich mich niemals\n"
+ + "gern befangen. Am meisten lieb ich mir die vollen, frischen Wangen. Fuer\n"
+ + "einem Leichnam bin ich nicht zu Haus; Mir geht es wie der Katze mit der Maus.\n"
+ + "DER HERR: Nun gut, es sei dir ueberlassen! Zieh diesen Geist von seinem\n"
+ + "Urquell ab, Und fuehr ihn, kannst du ihn erfassen, Auf deinem Wege mit\n"
+ + "herab, Und steh beschaemt, wenn du bekennen musst: Ein guter Mensch, in\n"
+ + "seinem dunklen Drange, Ist sich des rechten Weges wohl bewusst.\n"
+ + "MEPHISTOPHELES: Schon gut! nur dauert es nicht lange. Mir ist fuer meine\n"
+ + "Wette gar nicht bange. Wenn ich zu meinem Zweck gelange, Erlaubt Ihr mir\n"
+ + "Triumph aus voller Brust. Staub soll er fressen, und mit Lust, Wie meine\n"
+ + "Muhme, die beruehmte Schlange.\n"
+ + "DER HERR: Du darfst auch da nur frei erscheinen; Ich habe deinesgleichen\n"
+ + "nie gehasst. Von allen Geistern, die verneinen, ist mir der Schalk am\n"
+ + "wenigsten zur Last. Des Menschen Taetigkeit kann allzu leicht erschlaffen,\n"
+ + "er liebt sich bald die unbedingte Ruh; Drum geb ich gern ihm den Gesellen\n"
+ + "zu, Der reizt und wirkt und muss als Teufel schaffen. Doch ihr, die echten\n"
+ + "Goettersoehne, Erfreut euch der lebendig reichen Schoene! Das Werdende, das\n"
+ + "ewig wirkt und lebt, Umfass euch mit der Liebe holden Schranken, Und was\n"
+ + "in schwankender Erscheinung schwebt, Befestigt mit dauernden Gedanken!\n"
+ + "(Der Himmel schliesst, die Erzengel verteilen sich.)\n"
+ + "MEPHISTOPHELES (allein): Von Zeit zu Zeit seh ich den Alten gern, Und\n"
+ + "huete mich, mit ihm zu brechen. Es ist gar huebsch von einem grossen Herrn,\n"
+ + "So menschlich mit dem Teufel selbst zu sprechen.";
+
+ public static final String COUNTS = "machen 1\n" + "zeit 2\n" + "heerscharen 1\n" + "keiner 2\n" + "meine 3\n"
+ + "fuehr 1\n" + "triumph 1\n" + "kommst 1\n" + "frei 1\n" + "schaffen 1\n" + "gesinde 1\n"
+ + "langbeinigen 1\n" + "schalk 1\n" + "besser 1\n" + "solang 1\n" + "meer 4\n" + "fragst 1\n"
+ + "gabriel 1\n" + "selbst 2\n" + "bin 1\n" + "sich 7\n" + "du 11\n" + "sogar 1\n" + "geht 1\n"
+ + "immer 4\n" + "mensch 2\n" + "befestigt 1\n" + "lebt 2\n" + "mag 3\n" + "engeln 2\n" + "breiten 1\n"
+ + "blitzendes 1\n" + "tags 1\n" + "sie 2\n" + "plagen 2\n" + "allzu 1\n" + "meisten 1\n" + "o 1\n"
+ + "pfade 1\n" + "kennst 1\n" + "nichts 3\n" + "gedanken 1\n" + "befriedigt 1\n" + "mich 6\n" + "s 3\n"
+ + "es 8\n" + "verneinen 1\n" + "er 13\n" + "gleich 1\n" + "baeumchen 1\n" + "donnergang 1\n"
+ + "wunderlich 1\n" + "reise 1\n" + "urquell 1\n" + "doch 3\n" + "aufs 2\n" + "toten 1\n" + "niemals 1\n"
+ + "eine 2\n" + "hab 1\n" + "darfst 1\n" + "da 5\n" + "gen 1\n" + "einem 2\n" + "teil 1\n" + "das 7\n"
+ + "speise 1\n" + "wenig 1\n" + "sterne 1\n" + "geb 1\n" + "welten 1\n" + "alle 3\n" + "toent 1\n"
+ + "gras 1\n" + "felsen 1\n" + "kette 1\n" + "ich 14\n" + "fuer 2\n" + "als 3\n" + "mein 1\n"
+ + "schoene 1\n" + "verzeih 1\n" + "schwankender 1\n" + "wie 9\n" + "menschlich 1\n" + "gaertner 1\n"
+ + "taetigkeit 1\n" + "bange 1\n" + "liebe 1\n" + "sei 2\n" + "seh 1\n" + "tollheit 1\n" + "am 6\n"
+ + "michael 1\n" + "geist 1\n" + "ab 1\n" + "nahst 1\n" + "vollendet 1\n" + "liebt 1\n" + "brausen 1\n"
+ + "nase 1\n" + "erlaubt 1\n" + "weiss 2\n" + "schnellem 1\n" + "deinem 1\n" + "gleichem 1\n"
+ + "gaerung 1\n" + "dauernden 1\n" + "deines 1\n" + "vorgeschriebne 1\n" + "irdisch 1\n" + "worte 1\n"
+ + "verehren 1\n" + "hohen 2\n" + "weise 2\n" + "kuenft 1\n" + "werdende 1\n" + "wette 2\n" + "wuetend 1\n"
+ + "erscheinung 1\n" + "gar 2\n" + "verlieren 1\n" + "braucht 1\n" + "weiter 1\n" + "trank 1\n"
+ + "tierischer 1\n" + "wohl 1\n" + "verteilen 1\n" + "verhoehnt 1\n" + "schaeumt 1\n" + "himmelslichts 1\n"
+ + "unbedingte 1\n" + "herzlich 1\n" + "anblick 2\n" + "nennt 1\n" + "gruent 1\n" + "bluet 1\n"
+ + "leichnam 1\n" + "erschlaffen 1\n" + "jammertagen 1\n" + "zieh 1\n" + "ihm 3\n" + "besondre 1\n"
+ + "ihn 5\n" + "grossen 1\n" + "vollen 1\n" + "ihr 7\n" + "boten 1\n" + "voller 1\n" + "singt 1\n"
+ + "muhme 1\n" + "schon 1\n" + "last 1\n" + "kleine 1\n" + "paradieseshelle 1\n" + "nein 1\n" + "echten 1\n"
+ + "unter 1\n" + "bei 1\n" + "herr 11\n" + "gern 3\n" + "sphaerenlauf 1\n" + "stets 1\n" + "ganze 1\n"
+ + "braechte 1\n" + "fordert 1\n" + "schoensten 1\n" + "herrlich 2\n" + "gegeben 1\n" + "allein 2\n"
+ + "reichen 1\n" + "schauervoller 1\n" + "musst 1\n" + "recht 1\n" + "bleibt 1\n" + "pracht 1\n"
+ + "treibt 1\n" + "befangen 1\n" + "was 2\n" + "menschen 3\n" + "jede 1\n" + "hohe 1\n" + "tiefsten 1\n"
+ + "bilden 1\n" + "drum 1\n" + "gibt 2\n" + "guter 1\n" + "fuerwahr 1\n" + "im 3\n" + "grund 1\n" + "in 9\n"
+ + "hoechste 1\n" + "schliesst 1\n" + "fels 1\n" + "steh 1\n" + "euer 1\n" + "erster 1\n" + "ersten 3\n"
+ + "goettersoehne 1\n" + "brechen 1\n" + "tiefen 1\n" + "frucht 1\n" + "kreis 1\n" + "siehst 1\n"
+ + "wege 1\n" + "ist 8\n" + "zikaden 1\n" + "frischen 1\n" + "ruh 1\n" + "deine 2\n" + "maus 1\n"
+ + "brudersphaeren 1\n" + "nachher 1\n" + "euch 4\n" + "gnaden 1\n" + "anzuklagen 1\n" + "schlange 1\n"
+ + "staerke 2\n" + "erde 4\n" + "verlaub 1\n" + "sanfte 1\n" + "holden 1\n" + "sonst 1\n" + "treten 1\n"
+ + "sahst 1\n" + "alten 1\n" + "um 1\n" + "wieder 1\n" + "alter 1\n" + "altes 1\n" + "nun 1\n" + "lieb 1\n"
+ + "gesellen 1\n" + "erscheinen 1\n" + "wirkt 2\n" + "haettst 2\n" + "nur 7\n" + "tiefbewegte 1\n"
+ + "lachen 2\n" + "drange 1\n" + "schlag 1\n" + "schein 1\n" + "muss 1\n" + "verworren 1\n" + "weges 1\n"
+ + "allen 1\n" + "gewoehnlich 1\n" + "alles 1\n" + "halb 1\n" + "stuerme 1\n" + "springt 1\n" + "sollt 1\n"
+ + "klarheit 1\n" + "so 6\n" + "erfassen 1\n" + "liedchen 1\n" + "prolog 1\n" + "zur 1\n" + "fressen 1\n"
+ + "zum 1\n" + "faust 2\n" + "erzengel 2\n" + "jahre 1\n" + "sonn 1\n" + "raphael 1\n" + "land 2\n"
+ + "lang 1\n" + "gelange 1\n" + "lust 2\n" + "welt 1\n" + "sehe 1\n" + "ihre 1\n" + "jedes 1\n"
+ + "erfreut 1\n" + "seiner 1\n" + "denn 1\n" + "wandeln 1\n" + "wechselt 1\n" + "jeden 1\n" + "dort 1\n"
+ + "schlecht 1\n" + "wenigsten 1\n" + "wuerd 1\n" + "schranken 1\n" + "bewusst 2\n" + "seinem 2\n"
+ + "gehasst 1\n" + "sein 1\n" + "meinem 1\n" + "meinen 1\n" + "pathos 1\n" + "herrn 1\n" + "lange 2\n"
+ + "herab 1\n" + "diesen 1\n" + "ihren 1\n" + "beruehmte 1\n" + "goethe 1\n" + "tag 3\n" + "tier 1\n"
+ + "quark 1\n" + "dank 1\n" + "seine 1\n" + "teufel 2\n" + "zweck 1\n" + "wenn 7\n" + "soll 1\n"
+ + "wirkung 1\n" + "erlaubnis 1\n" + "lebendig 1\n" + "uns 1\n" + "leicht 1\n" + "gewiss 1\n"
+ + "schnell 1\n" + "und 29\n" + "gerne 1\n" + "rechten 1\n" + "umher 2\n" + "vernunft 1\n" + "grase 1\n"
+ + "nach 1\n" + "leben 1\n" + "gott 1\n" + "der 29\n" + "des 5\n" + "doktor 1\n" + "beschaemt 1\n"
+ + "dreht 1\n" + "habe 1\n" + "sagen 2\n" + "bekennen 1\n" + "dunklen 1\n" + "wettet 1\n" + "den 9\n"
+ + "mephistopheles 9\n" + "dem 4\n" + "auch 4\n" + "kann 2\n" + "armen 1\n" + "mir 9\n" + "strebt 1\n"
+ + "gut 2\n" + "mit 11\n" + "bald 2\n" + "himmlischen 1\n" + "himmel 3\n" + "noch 3\n" + "kannst 1\n"
+ + "deinesgleichen 1\n" + "flammt 1\n" + "ergruenden 2\n" + "nacht 1\n" + "scheint 1\n" + "ferne 2\n"
+ + "tragoedie 1\n" + "abgewoehnt 1\n" + "reizt 1\n" + "geistern 1\n" + "nicht 10\n" + "sacht 1\n"
+ + "unbegreiflich 2\n" + "schnelle 1\n" + "einmal 1\n" + "werd 1\n" + "werke 2\n" + "begraebt 1\n"
+ + "knecht 1\n" + "rings 1\n" + "wird 1\n" + "katze 1\n" + "huete 1\n" + "fortgerissen 1\n" + "gebt 1\n"
+ + "huebsch 1\n" + "hast 1\n" + "irrt 1\n" + "befinde 1\n" + "sind 2\n" + "fuehren 2\n" + "fliegt 1\n"
+ + "ewig 3\n" + "brust 2\n" + "sonne 1\n" + "sprechen 1\n" + "ein 3\n" + "strasse 1\n" + "von 8\n"
+ + "ueberlassen 1\n" + "dir 4\n" + "vom 3\n" + "zu 11\n" + "schwebt 1\n" + "die 22\n" + "vor 2\n"
+ + "wangen 1\n" + "wettgesang 1\n" + "donnerschlags 1\n" + "find 1\n" + "dich 3\n" + "umfass 1\n"
+ + "verboten 1\n" + "laeg 1\n" + "nie 1\n" + "drei 2\n" + "dauern 1\n" + "toren 1\n" + "dauert 1\n"
+ + "verheeren 1\n" + "fliegend 1\n" + "aus 1\n" + "staub 1\n" + "fluessen 1\n" + "haus 1\n" + "auf 5\n"
+ + "dient 2\n" + "tiefer 1\n" + "naeh 1\n" + "zieren 1\n";
+
+ public static final String STREAMING_COUNTS_AS_TUPLES = "(machen,1)\n" + "(zeit,1)\n" + "(zeit,2)\n" + "(heerscharen,1)\n" + "(keiner,1)\n"
+ + "(keiner,2)\n" + "(meine,1)\n" + "(meine,2)\n" + "(meine,3)\n" + "(fuehr,1)\n" + "(triumph,1)\n" + "(kommst,1)\n"
+ + "(frei,1)\n" + "(schaffen,1)\n" + "(gesinde,1)\n" + "(langbeinigen,1)\n" + "(schalk,1)\n" + "(besser,1)\n" + "(solang,1)\n"
+ + "(meer,1)\n" + "(meer,2)\n" + "(meer,3)\n" + "(meer,4)\n" + "(fragst,1)\n" + "(gabriel,1)\n" + "(selbst,1)\n" + "(selbst,2)\n"
+ + "(bin,1)\n" + "(sich,1)\n" + "(sich,2)\n" + "(sich,3)\n" + "(sich,4)\n" + "(sich,5)\n" + "(sich,6)\n" + "(sich,7)\n" + "(du,1)\n"
+ + "(du,2)\n" + "(du,3)\n" + "(du,4)\n" + "(du,5)\n" + "(du,6)\n" + "(du,7)\n" + "(du,8)\n" + "(du,9)\n" + "(du,10)\n" + "(du,11)\n"
+ + "(sogar,1)\n" + "(geht,1)\n" + "(immer,1)\n" + "(immer,2)\n" + "(immer,3)\n" + "(immer,4)\n" + "(mensch,1)\n" + "(mensch,2)\n"
+ + "(befestigt,1)\n" + "(lebt,1)\n" + "(lebt,2)\n" + "(mag,1)\n" + "(mag,2)\n" + "(mag,3)\n" + "(engeln,1)\n" + "(engeln,2)\n"
+ + "(breiten,1)\n" + "(blitzendes,1)\n" + "(tags,1)\n" + "(sie,1)\n" + "(sie,2)\n" + "(plagen,1)\n" + "(plagen,2)\n" + "(allzu,1)\n"
+ + "(meisten,1)\n" + "(o,1)\n" + "(pfade,1)\n" + "(kennst,1)\n" + "(nichts,1)\n" + "(nichts,2)\n" + "(nichts,3)\n" + "(gedanken,1)\n"
+ + "(befriedigt,1)\n" + "(mich,1)\n" + "(mich,2)\n" + "(mich,3)\n" + "(mich,4)\n" + "(mich,5)\n" + "(mich,6)\n" + "(s,1)\n" + "(s,2)\n"
+ + "(s,3)\n" + "(es,1)\n" + "(es,2)\n" + "(es,3)\n" + "(es,4)\n" + "(es,5)\n" + "(es,6)\n" + "(es,7)\n" + "(es,8)\n" + "(verneinen,1)\n"
+ + "(er,1)\n" + "(er,2)\n" + "(er,3)\n" + "(er,4)\n" + "(er,5)\n" + "(er,6)\n" + "(er,7)\n" + "(er,8)\n" + "(er,9)\n" + "(er,10)\n"
+ + "(er,11)\n" + "(er,12)\n" + "(er,13)\n" + "(gleich,1)\n" + "(baeumchen,1)\n" + "(donnergang,1)\n" + "(wunderlich,1)\n"
+ + "(reise,1)\n" + "(urquell,1)\n" + "(doch,1)\n" + "(doch,2)\n" + "(doch,3)\n" + "(aufs,1)\n" + "(aufs,2)\n" + "(toten,1)\n"
+ + "(niemals,1)\n" + "(eine,1)\n" + "(eine,2)\n" + "(hab,1)\n" + "(darfst,1)\n" + "(da,1)\n" + "(da,2)\n" + "(da,3)\n" + "(da,4)\n"
+ + "(da,5)\n" + "(gen,1)\n" + "(einem,1)\n" + "(einem,2)\n" + "(teil,1)\n" + "(das,1)\n" + "(das,2)\n" + "(das,3)\n" + "(das,4)\n"
+ + "(das,5)\n" + "(das,6)\n" + "(das,7)\n" + "(speise,1)\n" + "(wenig,1)\n" + "(sterne,1)\n" + "(geb,1)\n" + "(welten,1)\n"
+ + "(alle,1)\n" + "(alle,2)\n" + "(alle,3)\n" + "(toent,1)\n" + "(gras,1)\n" + "(felsen,1)\n" + "(kette,1)\n" + "(ich,1)\n"
+ + "(ich,2)\n" + "(ich,3)\n" + "(ich,4)\n" + "(ich,5)\n" + "(ich,6)\n" + "(ich,7)\n" + "(ich,8)\n" + "(ich,9)\n" + "(ich,10)\n"
+ + "(ich,11)\n" + "(ich,12)\n" + "(ich,13)\n" + "(ich,14)\n" + "(fuer,1)\n" + "(fuer,2)\n" + "(als,1)\n" + "(als,2)\n" + "(als,3)\n"
+ + "(mein,1)\n" + "(schoene,1)\n" + "(verzeih,1)\n" + "(schwankender,1)\n" + "(wie,1)\n" + "(wie,2)\n" + "(wie,3)\n" + "(wie,4)\n"
+ + "(wie,5)\n" + "(wie,6)\n" + "(wie,7)\n" + "(wie,8)\n" + "(wie,9)\n" + "(menschlich,1)\n" + "(gaertner,1)\n" + "(taetigkeit,1)\n"
+ + "(bange,1)\n" + "(liebe,1)\n" + "(sei,1)\n" + "(sei,2)\n" + "(seh,1)\n" + "(tollheit,1)\n" + "(am,1)\n" + "(am,2)\n" + "(am,3)\n"
+ + "(am,4)\n" + "(am,5)\n" + "(am,6)\n" + "(michael,1)\n" + "(geist,1)\n" + "(ab,1)\n" + "(nahst,1)\n" + "(vollendet,1)\n"
+ + "(liebt,1)\n" + "(brausen,1)\n" + "(nase,1)\n" + "(erlaubt,1)\n" + "(weiss,1)\n" + "(weiss,2)\n" + "(schnellem,1)\n"
+ + "(deinem,1)\n" + "(gleichem,1)\n" + "(gaerung,1)\n" + "(dauernden,1)\n" + "(deines,1)\n" + "(vorgeschriebne,1)\n"
+ + "(irdisch,1)\n" + "(worte,1)\n" + "(verehren,1)\n" + "(hohen,1)\n" + "(hohen,2)\n" + "(weise,1)\n" + "(weise,2)\n"
+ + "(kuenft,1)\n" + "(werdende,1)\n" + "(wette,1)\n" + "(wette,2)\n" + "(wuetend,1)\n" + "(erscheinung,1)\n" + "(gar,1)\n"
+ + "(gar,2)\n" + "(verlieren,1)\n" + "(braucht,1)\n" + "(weiter,1)\n" + "(trank,1)\n" + "(tierischer,1)\n" + "(wohl,1)\n"
+ + "(verteilen,1)\n" + "(verhoehnt,1)\n" + "(schaeumt,1)\n" + "(himmelslichts,1)\n" + "(unbedingte,1)\n" + "(herzlich,1)\n"
+ + "(anblick,1)\n" + "(anblick,2)\n" + "(nennt,1)\n" + "(gruent,1)\n" + "(bluet,1)\n" + "(leichnam,1)\n" + "(erschlaffen,1)\n"
+ + "(jammertagen,1)\n" + "(zieh,1)\n" + "(ihm,1)\n" + "(ihm,2)\n" + "(ihm,3)\n" + "(besondre,1)\n" + "(ihn,1)\n" + "(ihn,2)\n"
+ + "(ihn,3)\n" + "(ihn,4)\n" + "(ihn,5)\n" + "(grossen,1)\n" + "(vollen,1)\n" + "(ihr,1)\n" + "(ihr,2)\n" + "(ihr,3)\n"
+ + "(ihr,4)\n" + "(ihr,5)\n" + "(ihr,6)\n" + "(ihr,7)\n" + "(boten,1)\n" + "(voller,1)\n" + "(singt,1)\n" + "(muhme,1)\n"
+ + "(schon,1)\n" + "(last,1)\n" + "(kleine,1)\n" + "(paradieseshelle,1)\n" + "(nein,1)\n" + "(echten,1)\n" + "(unter,1)\n"
+ + "(bei,1)\n" + "(herr,1)\n" + "(herr,2)\n" + "(herr,3)\n" + "(herr,4)\n" + "(herr,5)\n" + "(herr,6)\n" + "(herr,7)\n"
+ + "(herr,8)\n" + "(herr,9)\n" + "(herr,10)\n" + "(herr,11)\n" + "(gern,1)\n" + "(gern,2)\n" + "(gern,3)\n" + "(sphaerenlauf,1)\n"
+ + "(stets,1)\n" + "(ganze,1)\n" + "(braechte,1)\n" + "(fordert,1)\n" + "(schoensten,1)\n" + "(herrlich,1)\n" + "(herrlich,2)\n"
+ + "(gegeben,1)\n" + "(allein,1)\n" + "(allein,2)\n" + "(reichen,1)\n" + "(schauervoller,1)\n" + "(musst,1)\n" + "(recht,1)\n"
+ + "(bleibt,1)\n" + "(pracht,1)\n" + "(treibt,1)\n" + "(befangen,1)\n" + "(was,1)\n" + "(was,2)\n" + "(menschen,1)\n"
+ + "(menschen,2)\n" + "(menschen,3)\n" + "(jede,1)\n" + "(hohe,1)\n" + "(tiefsten,1)\n" + "(bilden,1)\n" + "(drum,1)\n"
+ + "(gibt,1)\n" + "(gibt,2)\n" + "(guter,1)\n" + "(fuerwahr,1)\n" + "(im,1)\n" + "(im,2)\n" + "(im,3)\n" + "(grund,1)\n"
+ + "(in,1)\n" + "(in,2)\n" + "(in,3)\n" + "(in,4)\n" + "(in,5)\n" + "(in,6)\n" + "(in,7)\n" + "(in,8)\n" + "(in,9)\n" + "(hoechste,1)\n"
+ + "(schliesst,1)\n" + "(fels,1)\n" + "(steh,1)\n" + "(euer,1)\n" + "(erster,1)\n" + "(ersten,1)\n" + "(ersten,2)\n" + "(ersten,3)\n"
+ + "(goettersoehne,1)\n" + "(brechen,1)\n" + "(tiefen,1)\n" + "(frucht,1)\n" + "(kreis,1)\n" + "(siehst,1)\n" + "(wege,1)\n"
+ + "(ist,1)\n" + "(ist,2)\n" + "(ist,3)\n" + "(ist,4)\n" + "(ist,5)\n" + "(ist,6)\n" + "(ist,7)\n" + "(ist,8)\n" + "(zikaden,1)\n"
+ + "(frischen,1)\n" + "(ruh,1)\n" + "(deine,1)\n" + "(deine,2)\n" + "(maus,1)\n" + "(brudersphaeren,1)\n" + "(nachher,1)\n"
+ + "(euch,1)\n" + "(euch,2)\n" + "(euch,3)\n" + "(euch,4)\n" + "(gnaden,1)\n" + "(anzuklagen,1)\n" + "(schlange,1)\n" + "(staerke,1)\n"
+ + "(staerke,2)\n" + "(erde,1)\n" + "(erde,2)\n" + "(erde,3)\n" + "(erde,4)\n" + "(verlaub,1)\n" + "(sanfte,1)\n" + "(holden,1)\n"
+ + "(sonst,1)\n" + "(treten,1)\n" + "(sahst,1)\n" + "(alten,1)\n" + "(um,1)\n" + "(wieder,1)\n" + "(alter,1)\n" + "(altes,1)\n"
+ + "(nun,1)\n" + "(lieb,1)\n" + "(gesellen,1)\n" + "(erscheinen,1)\n" + "(wirkt,1)\n" + "(wirkt,2)\n" + "(haettst,1)\n" + "(haettst,2)\n"
+ + "(nur,1)\n" + "(nur,2)\n" + "(nur,3)\n" + "(nur,4)\n" + "(nur,5)\n" + "(nur,6)\n" + "(nur,7)\n" + "(tiefbewegte,1)\n" + "(lachen,1)\n"
+ + "(lachen,2)\n" + "(drange,1)\n" + "(schlag,1)\n" + "(schein,1)\n" + "(muss,1)\n" + "(verworren,1)\n" + "(weges,1)\n" + "(allen,1)\n"
+ + "(gewoehnlich,1)\n" + "(alles,1)\n" + "(halb,1)\n" + "(stuerme,1)\n" + "(springt,1)\n" + "(sollt,1)\n" + "(klarheit,1)\n"
+ + "(so,1)\n" + "(so,2)\n" + "(so,3)\n" + "(so,4)\n" + "(so,5)\n" + "(so,6)\n" + "(erfassen,1)\n" + "(liedchen,1)\n" + "(prolog,1)\n"
+ + "(zur,1)\n" + "(fressen,1)\n" + "(zum,1)\n" + "(faust,1)\n" + "(faust,2)\n" + "(erzengel,1)\n" + "(erzengel,2)\n" + "(jahre,1)\n"
+ + "(sonn,1)\n" + "(raphael,1)\n" + "(land,1)\n" + "(land,2)\n" + "(lang,1)\n" + "(gelange,1)\n" + "(lust,1)\n" + "(lust,2)\n"
+ + "(welt,1)\n" + "(sehe,1)\n" + "(ihre,1)\n" + "(jedes,1)\n" + "(erfreut,1)\n" + "(seiner,1)\n" + "(denn,1)\n" + "(wandeln,1)\n"
+ + "(wechselt,1)\n" + "(jeden,1)\n" + "(dort,1)\n" + "(schlecht,1)\n" + "(wenigsten,1)\n" + "(wuerd,1)\n" + "(schranken,1)\n"
+ + "(bewusst,1)\n" + "(bewusst,2)\n" + "(seinem,1)\n" + "(seinem,2)\n" + "(gehasst,1)\n" + "(sein,1)\n" + "(meinem,1)\n"
+ + "(meinen,1)\n" + "(pathos,1)\n" + "(herrn,1)\n" + "(lange,1)\n" + "(lange,2)\n" + "(herab,1)\n" + "(diesen,1)\n" + "(ihren,1)\n"
+ + "(beruehmte,1)\n" + "(goethe,1)\n" + "(tag,1)\n" + "(tag,2)\n" + "(tag,3)\n" + "(tier,1)\n" + "(quark,1)\n" + "(dank,1)\n"
+ + "(seine,1)\n" + "(teufel,1)\n" + "(teufel,2)\n" + "(zweck,1)\n" + "(wenn,1)\n" + "(wenn,2)\n" + "(wenn,3)\n" + "(wenn,4)\n"
+ + "(wenn,5)\n" + "(wenn,6)\n" + "(wenn,7)\n" + "(soll,1)\n" + "(wirkung,1)\n" + "(erlaubnis,1)\n" + "(lebendig,1)\n" + "(uns,1)\n"
+ + "(leicht,1)\n" + "(gewiss,1)\n" + "(schnell,1)\n" + "(und,1)\n" + "(und,2)\n" + "(und,3)\n" + "(und,4)\n" + "(und,5)\n" + "(und,6)\n"
+ + "(und,7)\n" + "(und,8)\n" + "(und,9)\n" + "(und,10)\n" + "(und,11)\n" + "(und,12)\n" + "(und,13)\n" + "(und,14)\n" + "(und,15)\n"
+ + "(und,16)\n" + "(und,17)\n" + "(und,18)\n" + "(und,19)\n" + "(und,20)\n" + "(und,21)\n" + "(und,22)\n" + "(und,23)\n" + "(und,24)\n"
+ + "(und,25)\n" + "(und,26)\n" + "(und,27)\n" + "(und,28)\n" + "(und,29)\n" + "(gerne,1)\n" + "(rechten,1)\n" + "(umher,1)\n" + "(umher,2)\n"
+ + "(vernunft,1)\n" + "(grase,1)\n" + "(nach,1)\n" + "(leben,1)\n" + "(gott,1)\n" + "(der,1)\n" + "(der,2)\n" + "(der,3)\n" + "(der,4)\n"
+ + "(der,5)\n" + "(der,6)\n" + "(der,7)\n" + "(der,8)\n" + "(der,9)\n" + "(der,10)\n" + "(der,11)\n" + "(der,12)\n" + "(der,13)\n"
+ + "(der,14)\n" + "(der,15)\n" + "(der,16)\n" + "(der,17)\n" + "(der,18)\n" + "(der,19)\n" + "(der,20)\n" + "(der,21)\n" + "(der,22)\n"
+ + "(der,23)\n" + "(der,24)\n" + "(der,25)\n" + "(der,26)\n" + "(der,27)\n" + "(der,28)\n" + "(der,29)\n" + "(des,1)\n" + "(des,2)\n"
+ + "(des,3)\n" + "(des,4)\n" + "(des,5)\n" + "(doktor,1)\n" + "(beschaemt,1)\n" + "(dreht,1)\n" + "(habe,1)\n" + "(sagen,1)\n" + "(sagen,2)\n"
+ + "(bekennen,1)\n" + "(dunklen,1)\n" + "(wettet,1)\n" + "(den,1)\n" + "(den,2)\n" + "(den,3)\n" + "(den,4)\n" + "(den,5)\n" + "(den,6)\n"
+ + "(den,7)\n" + "(den,8)\n" + "(den,9)\n" + "(mephistopheles,1)\n" + "(mephistopheles,2)\n" + "(mephistopheles,3)\n"
+ + "(mephistopheles,4)\n" + "(mephistopheles,5)\n" + "(mephistopheles,6)\n" + "(mephistopheles,7)\n" + "(mephistopheles,8)\n"
+ + "(mephistopheles,9)\n" + "(dem,1)\n" + "(dem,2)\n" + "(dem,3)\n" + "(dem,4)\n" + "(auch,1)\n" + "(auch,2)\n" + "(auch,3)\n" + "(auch,4)\n"
+ + "(kann,1)\n" + "(kann,2)\n" + "(armen,1)\n" + "(mir,1)\n" + "(mir,2)\n" + "(mir,3)\n" + "(mir,4)\n" + "(mir,5)\n" + "(mir,6)\n" + "(mir,7)\n"
+ + "(mir,8)\n" + "(mir,9)\n" + "(strebt,1)\n" + "(gut,1)\n" + "(gut,2)\n" + "(mit,1)\n" + "(mit,2)\n" + "(mit,3)\n" + "(mit,4)\n" + "(mit,5)\n"
+ + "(mit,6)\n" + "(mit,7)\n" + "(mit,8)\n" + "(mit,9)\n" + "(mit,10)\n" + "(mit,11)\n" + "(bald,1)\n" + "(bald,2)\n" + "(himmlischen,1)\n"
+ + "(himmel,1)\n" + "(himmel,2)\n" + "(himmel,3)\n" + "(noch,1)\n" + "(noch,2)\n" + "(noch,3)\n" + "(kannst,1)\n" + "(deinesgleichen,1)\n"
+ + "(flammt,1)\n" + "(ergruenden,1)\n" + "(ergruenden,2)\n" + "(nacht,1)\n" + "(scheint,1)\n" + "(ferne,1)\n" + "(ferne,2)\n"
+ + "(tragoedie,1)\n" + "(abgewoehnt,1)\n" + "(reizt,1)\n" + "(geistern,1)\n" + "(nicht,1)\n" + "(nicht,2)\n" + "(nicht,3)\n" + "(nicht,4)\n"
+ + "(nicht,5)\n" + "(nicht,6)\n" + "(nicht,7)\n" + "(nicht,8)\n" + "(nicht,9)\n" + "(nicht,10)\n" + "(sacht,1)\n" + "(unbegreiflich,1)\n"
+ + "(unbegreiflich,2)\n" + "(schnelle,1)\n" + "(einmal,1)\n" + "(werd,1)\n" + "(werke,1)\n" + "(werke,2)\n" + "(begraebt,1)\n"
+ + "(knecht,1)\n" + "(rings,1)\n" + "(wird,1)\n" + "(katze,1)\n" + "(huete,1)\n" + "(fortgerissen,1)\n" + "(gebt,1)\n" + "(huebsch,1)\n"
+ + "(hast,1)\n" + "(irrt,1)\n" + "(befinde,1)\n" + "(sind,1)\n" + "(sind,2)\n" + "(fuehren,1)\n" + "(fuehren,2)\n" + "(fliegt,1)\n"
+ + "(ewig,1)\n" + "(ewig,2)\n" + "(ewig,3)\n" + "(brust,1)\n" + "(brust,2)\n" + "(sonne,1)\n" + "(sprechen,1)\n" + "(ein,1)\n" + "(ein,2)\n"
+ + "(ein,3)\n" + "(strasse,1)\n" + "(von,1)\n" + "(von,2)\n" + "(von,3)\n" + "(von,4)\n" + "(von,5)\n" + "(von,6)\n" + "(von,7)\n" + "(von,8)\n"
+ + "(ueberlassen,1)\n" + "(dir,1)\n" + "(dir,2)\n" + "(dir,3)\n" + "(dir,4)\n" + "(vom,1)\n" + "(vom,2)\n" + "(vom,3)\n" + "(zu,1)\n" + "(zu,2)\n"
+ + "(zu,3)\n" + "(zu,4)\n" + "(zu,5)\n" + "(zu,6)\n" + "(zu,7)\n" + "(zu,8)\n" + "(zu,9)\n" + "(zu,10)\n" + "(zu,11)\n" + "(schwebt,1)\n"
+ + "(die,1)\n" + "(die,2)\n" + "(die,3)\n" + "(die,4)\n" + "(die,5)\n" + "(die,6)\n" + "(die,7)\n" + "(die,8)\n" + "(die,9)\n" + "(die,10)\n"
+ + "(die,11)\n" + "(die,12)\n" + "(die,13)\n" + "(die,14)\n" + "(die,15)\n" + "(die,16)\n" + "(die,17)\n" + "(die,18)\n" + "(die,19)\n"
+ + "(die,20)\n" + "(die,21)\n" + "(die,22)\n" + "(vor,1)\n" + "(vor,2)\n" + "(wangen,1)\n" + "(wettgesang,1)\n" + "(donnerschlags,1)\n"
+ + "(find,1)\n" + "(dich,1)\n" + "(dich,2)\n" + "(dich,3)\n" + "(umfass,1)\n" + "(verboten,1)\n" + "(laeg,1)\n" + "(nie,1)\n" + "(drei,1)\n"
+ + "(drei,2)\n" + "(dauern,1)\n" + "(toren,1)\n" + "(dauert,1)\n" + "(verheeren,1)\n" + "(fliegend,1)\n" + "(aus,1)\n" + "(staub,1)\n"
+ + "(fluessen,1)\n" + "(haus,1)\n" + "(auf,1)\n" + "(auf,2)\n" + "(auf,3)\n" + "(auf,4)\n" + "(auf,5)\n" + "(dient,1)\n" + "(dient,2)\n"
+ + "(tiefer,1)\n" + "(naeh,1)\n" + "(zieren,1)\n";
+
+ public static final String COUNTS_AS_TUPLES = "(machen,1)\n" + "(zeit,2)\n" + "(heerscharen,1)\n" + "(keiner,2)\n" + "(meine,3)\n"
+ + "(fuehr,1)\n" + "(triumph,1)\n" + "(kommst,1)\n" + "(frei,1)\n" + "(schaffen,1)\n" + "(gesinde,1)\n"
+ + "(langbeinigen,1)\n" + "(schalk,1)\n" + "(besser,1)\n" + "(solang,1)\n" + "(meer,4)\n" + "(fragst,1)\n"
+ + "(gabriel,1)\n" + "(selbst,2)\n" + "(bin,1)\n" + "(sich,7)\n" + "(du,11)\n" + "(sogar,1)\n" + "(geht,1)\n"
+ + "(immer,4)\n" + "(mensch,2)\n" + "(befestigt,1)\n" + "(lebt,2)\n" + "(mag,3)\n" + "(engeln,2)\n" + "(breiten,1)\n"
+ + "(blitzendes,1)\n" + "(tags,1)\n" + "(sie,2)\n" + "(plagen,2)\n" + "(allzu,1)\n" + "(meisten,1)\n" + "(o,1)\n"
+ + "(pfade,1)\n" + "(kennst,1)\n" + "(nichts,3)\n" + "(gedanken,1)\n" + "(befriedigt,1)\n" + "(mich,6)\n" + "(s,3)\n"
+ + "(es,8)\n" + "(verneinen,1)\n" + "(er,13)\n" + "(gleich,1)\n" + "(baeumchen,1)\n" + "(donnergang,1)\n"
+ + "(wunderlich,1)\n" + "(reise,1)\n" + "(urquell,1)\n" + "(doch,3)\n" + "(aufs,2)\n" + "(toten,1)\n" + "(niemals,1)\n"
+ + "(eine,2)\n" + "(hab,1)\n" + "(darfst,1)\n" + "(da,5)\n" + "(gen,1)\n" + "(einem,2)\n" + "(teil,1)\n" + "(das,7)\n"
+ + "(speise,1)\n" + "(wenig,1)\n" + "(sterne,1)\n" + "(geb,1)\n" + "(welten,1)\n" + "(alle,3)\n" + "(toent,1)\n"
+ + "(gras,1)\n" + "(felsen,1)\n" + "(kette,1)\n" + "(ich,14)\n" + "(fuer,2)\n" + "(als,3)\n" + "(mein,1)\n"
+ + "(schoene,1)\n" + "(verzeih,1)\n" + "(schwankender,1)\n" + "(wie,9)\n" + "(menschlich,1)\n" + "(gaertner,1)\n"
+ + "(taetigkeit,1)\n" + "(bange,1)\n" + "(liebe,1)\n" + "(sei,2)\n" + "(seh,1)\n" + "(tollheit,1)\n" + "(am,6)\n"
+ + "(michael,1)\n" + "(geist,1)\n" + "(ab,1)\n" + "(nahst,1)\n" + "(vollendet,1)\n" + "(liebt,1)\n" + "(brausen,1)\n"
+ + "(nase,1)\n" + "(erlaubt,1)\n" + "(weiss,2)\n" + "(schnellem,1)\n" + "(deinem,1)\n" + "(gleichem,1)\n"
+ + "(gaerung,1)\n" + "(dauernden,1)\n" + "(deines,1)\n" + "(vorgeschriebne,1)\n" + "(irdisch,1)\n" + "(worte,1)\n"
+ + "(verehren,1)\n" + "(hohen,2)\n" + "(weise,2)\n" + "(kuenft,1)\n" + "(werdende,1)\n" + "(wette,2)\n" + "(wuetend,1)\n"
+ + "(erscheinung,1)\n" + "(gar,2)\n" + "(verlieren,1)\n" + "(braucht,1)\n" + "(weiter,1)\n" + "(trank,1)\n"
+ + "(tierischer,1)\n" + "(wohl,1)\n" + "(verteilen,1)\n" + "(verhoehnt,1)\n" + "(schaeumt,1)\n" + "(himmelslichts,1)\n"
+ + "(unbedingte,1)\n" + "(herzlich,1)\n" + "(anblick,2)\n" + "(nennt,1)\n" + "(gruent,1)\n" + "(bluet,1)\n"
+ + "(leichnam,1)\n" + "(erschlaffen,1)\n" + "(jammertagen,1)\n" + "(zieh,1)\n" + "(ihm,3)\n" + "(besondre,1)\n"
+ + "(ihn,5)\n" + "(grossen,1)\n" + "(vollen,1)\n" + "(ihr,7)\n" + "(boten,1)\n" + "(voller,1)\n" + "(singt,1)\n"
+ + "(muhme,1)\n" + "(schon,1)\n" + "(last,1)\n" + "(kleine,1)\n" + "(paradieseshelle,1)\n" + "(nein,1)\n" + "(echten,1)\n"
+ + "(unter,1)\n" + "(bei,1)\n" + "(herr,11)\n" + "(gern,3)\n" + "(sphaerenlauf,1)\n" + "(stets,1)\n" + "(ganze,1)\n"
+ + "(braechte,1)\n" + "(fordert,1)\n" + "(schoensten,1)\n" + "(herrlich,2)\n" + "(gegeben,1)\n" + "(allein,2)\n"
+ + "(reichen,1)\n" + "(schauervoller,1)\n" + "(musst,1)\n" + "(recht,1)\n" + "(bleibt,1)\n" + "(pracht,1)\n"
+ + "(treibt,1)\n" + "(befangen,1)\n" + "(was,2)\n" + "(menschen,3)\n" + "(jede,1)\n" + "(hohe,1)\n" + "(tiefsten,1)\n"
+ + "(bilden,1)\n" + "(drum,1)\n" + "(gibt,2)\n" + "(guter,1)\n" + "(fuerwahr,1)\n" + "(im,3)\n" + "(grund,1)\n" + "(in,9)\n"
+ + "(hoechste,1)\n" + "(schliesst,1)\n" + "(fels,1)\n" + "(steh,1)\n" + "(euer,1)\n" + "(erster,1)\n" + "(ersten,3)\n"
+ + "(goettersoehne,1)\n" + "(brechen,1)\n" + "(tiefen,1)\n" + "(frucht,1)\n" + "(kreis,1)\n" + "(siehst,1)\n"
+ + "(wege,1)\n" + "(ist,8)\n" + "(zikaden,1)\n" + "(frischen,1)\n" + "(ruh,1)\n" + "(deine,2)\n" + "(maus,1)\n"
+ + "(brudersphaeren,1)\n" + "(nachher,1)\n" + "(euch,4)\n" + "(gnaden,1)\n" + "(anzuklagen,1)\n" + "(schlange,1)\n"
+ + "(staerke,2)\n" + "(erde,4)\n" + "(verlaub,1)\n" + "(sanfte,1)\n" + "(holden,1)\n" + "(sonst,1)\n" + "(treten,1)\n"
+ + "(sahst,1)\n" + "(alten,1)\n" + "(um,1)\n" + "(wieder,1)\n" + "(alter,1)\n" + "(altes,1)\n" + "(nun,1)\n" + "(lieb,1)\n"
+ + "(gesellen,1)\n" + "(erscheinen,1)\n" + "(wirkt,2)\n" + "(haettst,2)\n" + "(nur,7)\n" + "(tiefbewegte,1)\n"
+ + "(lachen,2)\n" + "(drange,1)\n" + "(schlag,1)\n" + "(schein,1)\n" + "(muss,1)\n" + "(verworren,1)\n" + "(weges,1)\n"
+ + "(allen,1)\n" + "(gewoehnlich,1)\n" + "(alles,1)\n" + "(halb,1)\n" + "(stuerme,1)\n" + "(springt,1)\n" + "(sollt,1)\n"
+ + "(klarheit,1)\n" + "(so,6)\n" + "(erfassen,1)\n" + "(liedchen,1)\n" + "(prolog,1)\n" + "(zur,1)\n" + "(fressen,1)\n"
+ + "(zum,1)\n" + "(faust,2)\n" + "(erzengel,2)\n" + "(jahre,1)\n" + "(sonn,1)\n" + "(raphael,1)\n" + "(land,2)\n"
+ + "(lang,1)\n" + "(gelange,1)\n" + "(lust,2)\n" + "(welt,1)\n" + "(sehe,1)\n" + "(ihre,1)\n" + "(jedes,1)\n"
+ + "(erfreut,1)\n" + "(seiner,1)\n" + "(denn,1)\n" + "(wandeln,1)\n" + "(wechselt,1)\n" + "(jeden,1)\n" + "(dort,1)\n"
+ + "(schlecht,1)\n" + "(wenigsten,1)\n" + "(wuerd,1)\n" + "(schranken,1)\n" + "(bewusst,2)\n" + "(seinem,2)\n"
+ + "(gehasst,1)\n" + "(sein,1)\n" + "(meinem,1)\n" + "(meinen,1)\n" + "(pathos,1)\n" + "(herrn,1)\n" + "(lange,2)\n"
+ + "(herab,1)\n" + "(diesen,1)\n" + "(ihren,1)\n" + "(beruehmte,1)\n" + "(goethe,1)\n" + "(tag,3)\n" + "(tier,1)\n"
+ + "(quark,1)\n" + "(dank,1)\n" + "(seine,1)\n" + "(teufel,2)\n" + "(zweck,1)\n" + "(wenn,7)\n" + "(soll,1)\n"
+ + "(wirkung,1)\n" + "(erlaubnis,1)\n" + "(lebendig,1)\n" + "(uns,1)\n" + "(leicht,1)\n" + "(gewiss,1)\n"
+ + "(schnell,1)\n" + "(und,29)\n" + "(gerne,1)\n" + "(rechten,1)\n" + "(umher,2)\n" + "(vernunft,1)\n" + "(grase,1)\n"
+ + "(nach,1)\n" + "(leben,1)\n" + "(gott,1)\n" + "(der,29)\n" + "(des,5)\n" + "(doktor,1)\n" + "(beschaemt,1)\n"
+ + "(dreht,1)\n" + "(habe,1)\n" + "(sagen,2)\n" + "(bekennen,1)\n" + "(dunklen,1)\n" + "(wettet,1)\n" + "(den,9)\n"
+ + "(mephistopheles,9)\n" + "(dem,4)\n" + "(auch,4)\n" + "(kann,2)\n" + "(armen,1)\n" + "(mir,9)\n" + "(strebt,1)\n"
+ + "(gut,2)\n" + "(mit,11)\n" + "(bald,2)\n" + "(himmlischen,1)\n" + "(himmel,3)\n" + "(noch,3)\n" + "(kannst,1)\n"
+ + "(deinesgleichen,1)\n" + "(flammt,1)\n" + "(ergruenden,2)\n" + "(nacht,1)\n" + "(scheint,1)\n" + "(ferne,2)\n"
+ + "(tragoedie,1)\n" + "(abgewoehnt,1)\n" + "(reizt,1)\n" + "(geistern,1)\n" + "(nicht,10)\n" + "(sacht,1)\n"
+ + "(unbegreiflich,2)\n" + "(schnelle,1)\n" + "(einmal,1)\n" + "(werd,1)\n" + "(werke,2)\n" + "(begraebt,1)\n"
+ + "(knecht,1)\n" + "(rings,1)\n" + "(wird,1)\n" + "(katze,1)\n" + "(huete,1)\n" + "(fortgerissen,1)\n" + "(gebt,1)\n"
+ + "(huebsch,1)\n" + "(hast,1)\n" + "(irrt,1)\n" + "(befinde,1)\n" + "(sind,2)\n" + "(fuehren,2)\n" + "(fliegt,1)\n"
+ + "(ewig,3)\n" + "(brust,2)\n" + "(sonne,1)\n" + "(sprechen,1)\n" + "(ein,3)\n" + "(strasse,1)\n" + "(von,8)\n"
+ + "(ueberlassen,1)\n" + "(dir,4)\n" + "(vom,3)\n" + "(zu,11)\n" + "(schwebt,1)\n" + "(die,22)\n" + "(vor,2)\n"
+ + "(wangen,1)\n" + "(wettgesang,1)\n" + "(donnerschlags,1)\n" + "(find,1)\n" + "(dich,3)\n" + "(umfass,1)\n"
+ + "(verboten,1)\n" + "(laeg,1)\n" + "(nie,1)\n" + "(drei,2)\n" + "(dauern,1)\n" + "(toren,1)\n" + "(dauert,1)\n"
+ + "(verheeren,1)\n" + "(fliegend,1)\n" + "(aus,1)\n" + "(staub,1)\n" + "(fluessen,1)\n" + "(haus,1)\n" + "(auf,5)\n"
+ + "(dient,2)\n" + "(tiefer,1)\n" + "(naeh,1)\n" + "(zieren,1)\n";
+
+ private WordCountData() {
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/AbstractTestBase.java
----------------------------------------------------------------------
diff --git a/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/AbstractTestBase.java b/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/AbstractTestBase.java
new file mode 100644
index 0000000..c2da691
--- /dev/null
+++ b/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/AbstractTestBase.java
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.test.util;
+
+import com.google.common.base.Charsets;
+import com.google.common.io.Files;
+import org.apache.flink.configuration.Configuration;
+import scala.concurrent.duration.FiniteDuration;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+
+import org.apache.flink.runtime.akka.AkkaUtils;
+
+/**
+ * A base class for tests that run test programs in a Flink mini cluster.
+ */
+public abstract class AbstractTestBase extends TestBaseUtils {
+
+ /** Configuration to start the testing cluster with */
+ protected final Configuration config;
+
+ private final List<File> tempFiles;
+
+ private final FiniteDuration timeout;
+
+ protected int taskManagerNumSlots = 1;
+
+ protected int numTaskManagers = 1;
+
+ /** The mini cluster that runs the test programs */
+ protected ForkableFlinkMiniCluster executor;
+
+
+ public AbstractTestBase(Configuration config) {
+ this.config = Objects.requireNonNull(config);
+ this.tempFiles = new ArrayList<File>();
+
+ timeout = AkkaUtils.getTimeout(config);
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // Local Test Cluster Life Cycle
+ // --------------------------------------------------------------------------------------------
+
+ public void startCluster() throws Exception {
+ this.executor = startCluster(
+ numTaskManagers,
+ taskManagerNumSlots,
+ false,
+ false,
+ true);
+ }
+
+ public void stopCluster() throws Exception {
+ stopCluster(executor, timeout);
+ deleteAllTempFiles();
+ }
+
+ //------------------
+ // Accessors
+ //------------------
+
+ public int getTaskManagerNumSlots() {
+ return taskManagerNumSlots;
+ }
+
+ public void setTaskManagerNumSlots(int taskManagerNumSlots) {
+ this.taskManagerNumSlots = taskManagerNumSlots;
+ }
+
+ public int getNumTaskManagers() {
+ return numTaskManagers;
+ }
+
+ public void setNumTaskManagers(int numTaskManagers) {
+ this.numTaskManagers = numTaskManagers;
+ }
+
+
+ // --------------------------------------------------------------------------------------------
+ // Temporary File Utilities
+ // --------------------------------------------------------------------------------------------
+
+ public String getTempDirPath(String dirName) throws IOException {
+ File f = createAndRegisterTempFile(dirName);
+ return f.toURI().toString();
+ }
+
+ public String getTempFilePath(String fileName) throws IOException {
+ File f = createAndRegisterTempFile(fileName);
+ return f.toURI().toString();
+ }
+
+ public String createTempFile(String fileName, String contents) throws IOException {
+ File f = createAndRegisterTempFile(fileName);
+ Files.write(contents, f, Charsets.UTF_8);
+ return f.toURI().toString();
+ }
+
+ public File createAndRegisterTempFile(String fileName) throws IOException {
+ File baseDir = new File(System.getProperty("java.io.tmpdir"));
+ File f = new File(baseDir, this.getClass().getName() + "-" + fileName);
+
+ if (f.exists()) {
+ deleteRecursively(f);
+ }
+
+ File parentToDelete = f;
+ while (true) {
+ File parent = parentToDelete.getParentFile();
+ if (parent == null) {
+ throw new IOException("Missed temp dir while traversing parents of a temp file.");
+ }
+ if (parent.equals(baseDir)) {
+ break;
+ }
+ parentToDelete = parent;
+ }
+
+ Files.createParentDirs(f);
+ this.tempFiles.add(parentToDelete);
+ return f;
+ }
+
+ private void deleteAllTempFiles() throws IOException {
+ for (File f : this.tempFiles) {
+ if (f.exists()) {
+ deleteRecursively(f);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/CollectionTestEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/CollectionTestEnvironment.java b/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/CollectionTestEnvironment.java
new file mode 100644
index 0000000..e56c7e8
--- /dev/null
+++ b/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/CollectionTestEnvironment.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.test.util;
+
+import org.apache.flink.api.common.JobExecutionResult;
+import org.apache.flink.api.java.CollectionEnvironment;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.ExecutionEnvironmentFactory;
+
+public class CollectionTestEnvironment extends CollectionEnvironment {
+
+ private CollectionTestEnvironment lastEnv = null;
+
+ @Override
+ public JobExecutionResult getLastJobExecutionResult() {
+ if (lastEnv == null) {
+ return this.lastJobExecutionResult;
+ }
+ else {
+ return lastEnv.getLastJobExecutionResult();
+ }
+ }
+
+ @Override
+ public JobExecutionResult execute() throws Exception {
+ return execute("test job");
+ }
+
+ @Override
+ public JobExecutionResult execute(String jobName) throws Exception {
+ JobExecutionResult result = super.execute(jobName);
+ this.lastJobExecutionResult = result;
+ return result;
+ }
+
+ protected void setAsContext() {
+ ExecutionEnvironmentFactory factory = new ExecutionEnvironmentFactory() {
+ @Override
+ public ExecutionEnvironment createExecutionEnvironment() {
+ lastEnv = new CollectionTestEnvironment();
+ return lastEnv;
+ }
+ };
+
+ initializeContextEnvironment(factory);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/JavaProgramTestBase.java
----------------------------------------------------------------------
diff --git a/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/JavaProgramTestBase.java b/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/JavaProgramTestBase.java
new file mode 100644
index 0000000..2bcf66d
--- /dev/null
+++ b/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/JavaProgramTestBase.java
@@ -0,0 +1,280 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.test.util;
+
+import org.apache.flink.api.common.JobExecutionResult;
+import org.apache.flink.api.java.tuple.Tuple;
+import org.apache.flink.configuration.Configuration;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.Comparator;
+
+
+public abstract class JavaProgramTestBase extends AbstractTestBase {
+
+ private static final int DEFAULT_PARALLELISM = 4;
+
+ private JobExecutionResult latestExecutionResult;
+
+ private int parallelism = DEFAULT_PARALLELISM;
+
+ /**
+ * The number of times a test should be repeated.
+ *
+ * <p> This is useful for runtime changes, which affect resource management. Running certain
+ * tests repeatedly might help to discover resource leaks, race conditions etc.
+ */
+ private int numberOfTestRepetitions = 1;
+
+ private boolean isCollectionExecution;
+
+ public JavaProgramTestBase() {
+ this(new Configuration());
+ }
+
+ public JavaProgramTestBase(Configuration config) {
+ super(config);
+ setTaskManagerNumSlots(parallelism);
+ }
+
+ public void setParallelism(int parallelism) {
+ this.parallelism = parallelism;
+ setTaskManagerNumSlots(parallelism);
+ }
+
+ public void setNumberOfTestRepetitions(int numberOfTestRepetitions) {
+ this.numberOfTestRepetitions = numberOfTestRepetitions;
+ }
+
+ public int getParallelism() {
+ return isCollectionExecution ? 1 : parallelism;
+ }
+
+ public JobExecutionResult getLatestExecutionResult() {
+ return this.latestExecutionResult;
+ }
+
+ public boolean isCollectionExecution() {
+ return isCollectionExecution;
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // Methods to create the test program and for pre- and post- test work
+ // --------------------------------------------------------------------------------------------
+
+ protected abstract void testProgram() throws Exception;
+
+ protected void preSubmit() throws Exception {}
+
+ protected void postSubmit() throws Exception {}
+
+ protected boolean skipCollectionExecution() {
+ return false;
+ };
+
+ // --------------------------------------------------------------------------------------------
+ // Test entry point
+ // --------------------------------------------------------------------------------------------
+
+ @Test
+ public void testJobWithObjectReuse() throws Exception {
+ isCollectionExecution = false;
+
+ startCluster();
+ try {
+ // pre-submit
+ try {
+ preSubmit();
+ }
+ catch (Exception e) {
+ System.err.println(e.getMessage());
+ e.printStackTrace();
+ Assert.fail("Pre-submit work caused an error: " + e.getMessage());
+ }
+
+ // prepare the test environment
+ TestEnvironment env = new TestEnvironment(this.executor, this.parallelism);
+ env.getConfig().enableObjectReuse();
+ env.setAsContext();
+
+ // Possibly run the test multiple times
+ for (int i = 0; i < numberOfTestRepetitions; i++) {
+ // call the test program
+ try {
+ testProgram();
+ this.latestExecutionResult = env.getLastJobExecutionResult();
+ }
+ catch (Exception e) {
+ System.err.println(e.getMessage());
+ e.printStackTrace();
+ Assert.fail("Error while calling the test program: " + e.getMessage());
+ }
+
+ Assert.assertNotNull("The test program never triggered an execution.",
+ this.latestExecutionResult);
+ }
+
+ // post-submit
+ try {
+ postSubmit();
+ }
+ catch (Exception e) {
+ System.err.println(e.getMessage());
+ e.printStackTrace();
+ Assert.fail("Post-submit work caused an error: " + e.getMessage());
+ }
+ } finally {
+ stopCluster();
+ }
+ }
+
+ @Test
+ public void testJobWithoutObjectReuse() throws Exception {
+ isCollectionExecution = false;
+
+ startCluster();
+ try {
+ // pre-submit
+ try {
+ preSubmit();
+ }
+ catch (Exception e) {
+ System.err.println(e.getMessage());
+ e.printStackTrace();
+ Assert.fail("Pre-submit work caused an error: " + e.getMessage());
+ }
+
+ // prepare the test environment
+ TestEnvironment env = new TestEnvironment(this.executor, this.parallelism);
+ env.getConfig().disableObjectReuse();
+ env.setAsContext();
+
+ // Possibly run the test multiple times
+ for (int i = 0; i < numberOfTestRepetitions; i++) {
+ // call the test program
+ try {
+ testProgram();
+ this.latestExecutionResult = env.getLastJobExecutionResult();
+ }
+ catch (Exception e) {
+ System.err.println(e.getMessage());
+ e.printStackTrace();
+ Assert.fail("Error while calling the test program: " + e.getMessage());
+ }
+
+ Assert.assertNotNull("The test program never triggered an execution.",
+ this.latestExecutionResult);
+ }
+
+ // post-submit
+ try {
+ postSubmit();
+ }
+ catch (Exception e) {
+ System.err.println(e.getMessage());
+ e.printStackTrace();
+ Assert.fail("Post-submit work caused an error: " + e.getMessage());
+ }
+ } finally {
+ stopCluster();
+ }
+ }
+
+ @Test
+ public void testJobCollectionExecution() throws Exception {
+
+ // check if collection execution should be skipped.
+ if(this.skipCollectionExecution()) {
+ return;
+ }
+
+ isCollectionExecution = true;
+
+ // pre-submit
+ try {
+ preSubmit();
+ }
+ catch (Exception e) {
+ System.err.println(e.getMessage());
+ e.printStackTrace();
+ Assert.fail("Pre-submit work caused an error: " + e.getMessage());
+ }
+
+ // prepare the test environment
+ CollectionTestEnvironment env = new CollectionTestEnvironment();
+ env.setAsContext();
+
+ // call the test program
+ try {
+ testProgram();
+ this.latestExecutionResult = env.getLastJobExecutionResult();
+ }
+ catch (Exception e) {
+ System.err.println(e.getMessage());
+ e.printStackTrace();
+ Assert.fail("Error while calling the test program: " + e.getMessage());
+ }
+
+ Assert.assertNotNull("The test program never triggered an execution.", this.latestExecutionResult);
+
+ // post-submit
+ try {
+ postSubmit();
+ }
+ catch (Exception e) {
+ System.err.println(e.getMessage());
+ e.printStackTrace();
+ Assert.fail("Post-submit work caused an error: " + e.getMessage());
+ }
+ }
+
+ public static class TupleComparator<T extends Tuple> implements Comparator<T> {
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public int compare(T o1, T o2) {
+ int a1 = o1.getArity();
+ int a2 = o2.getArity();
+
+ if (a1 < a2) {
+ return -1;
+ } else if (a2 < a1) {
+ return 1;
+ } else {
+ for (int i = 0; i < a1; i++) {
+ Object obj1 = o1.getField(i);
+ Object obj2 = o2.getField(i);
+
+ if (!(obj1 instanceof Comparable && obj2 instanceof Comparable)) {
+ Assert.fail("Cannot compare tuple fields");
+ }
+
+ int cmp = ((Comparable<Object>) obj1).compareTo(obj2);
+ if (cmp != 0) {
+ return cmp;
+ }
+ }
+
+ return 0;
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/MultipleProgramsTestBase.java
----------------------------------------------------------------------
diff --git a/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/MultipleProgramsTestBase.java b/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/MultipleProgramsTestBase.java
new file mode 100644
index 0000000..d7f09bd
--- /dev/null
+++ b/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/MultipleProgramsTestBase.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.test.util;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.runners.Parameterized;
+
+import java.util.Arrays;
+import java.util.Collection;
+
+/**
+ * Base class for unit tests that run multiple tests and want to reuse the same
+ * Flink cluster. This saves a significant amount of time, since the startup and
+ * shutdown of the Flink clusters (including actor systems, etc) usually dominates
+ * the execution of the actual tests.
+ *
+ * To write a unit test against this test base, simply extend it and add
+ * one or more regular test methods and retrieve the ExecutionEnvironment from
+ * the context:
+ *
+ * <pre>{@code
+ *
+ * {@literal @}Test
+ * public void someTest() {
+ * ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ * // test code
+ * env.execute();
+ * }
+ *
+ * {@literal @}Test
+ * public void anotherTest() {
+ * ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ * // test code
+ * env.execute();
+ * }
+ *
+ * }</pre>
+ */
+public class MultipleProgramsTestBase extends TestBaseUtils {
+
+ /**
+ * Enum that defines which execution environment to run the next test on:
+ * An embedded local flink cluster, or the collection execution backend.
+ */
+ public enum TestExecutionMode {
+ CLUSTER,
+ COLLECTION
+ }
+
+ // ------------------------------------------------------------------------
+ // The mini cluster that is shared across tests
+ // ------------------------------------------------------------------------
+
+ protected static final int DEFAULT_PARALLELISM = 4;
+
+ protected static boolean startWebServer = false;
+
+ protected static ForkableFlinkMiniCluster cluster = null;
+
+ // ------------------------------------------------------------------------
+
+ protected final TestExecutionMode mode;
+
+
+ public MultipleProgramsTestBase(TestExecutionMode mode) {
+ this.mode = mode;
+
+ switch(mode){
+ case CLUSTER:
+ TestEnvironment clusterEnv = new TestEnvironment(cluster, 4);
+ clusterEnv.setAsContext();
+ break;
+ case COLLECTION:
+ CollectionTestEnvironment collectionEnv = new CollectionTestEnvironment();
+ collectionEnv.setAsContext();
+ break;
+ }
+ }
+
+ // ------------------------------------------------------------------------
+ // Cluster setup & teardown
+ // ------------------------------------------------------------------------
+
+ @BeforeClass
+ public static void setup() throws Exception {
+ cluster = TestBaseUtils.startCluster(
+ 1,
+ DEFAULT_PARALLELISM,
+ startWebServer,
+ false,
+ true);
+ }
+
+ @AfterClass
+ public static void teardown() throws Exception {
+ stopCluster(cluster, TestBaseUtils.DEFAULT_TIMEOUT);
+ }
+
+ // ------------------------------------------------------------------------
+ // Parametrization lets the tests run in cluster and collection mode
+ // ------------------------------------------------------------------------
+
+ @Parameterized.Parameters(name = "Execution mode = {0}")
+ public static Collection<Object[]> executionModes() {
+ return Arrays.asList(
+ new Object[] { TestExecutionMode.CLUSTER },
+ new Object[] { TestExecutionMode.COLLECTION });
+ }
+}
[11/15] flink git commit: [FLINK-3995] [build] flink-test-utils also
contains the streaming test utilities.
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/b9f42e91/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/CoGroupJoinITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/CoGroupJoinITCase.java b/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/CoGroupJoinITCase.java
new file mode 100644
index 0000000..da3de3d
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/CoGroupJoinITCase.java
@@ -0,0 +1,369 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.test.streaming.runtime;
+
+import org.apache.flink.api.common.functions.CoGroupFunction;
+import org.apache.flink.api.common.functions.JoinFunction;
+import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.api.java.tuple.Tuple3;
+import org.apache.flink.streaming.api.TimeCharacteristic;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.functions.AssignerWithPunctuatedWatermarks;
+import org.apache.flink.streaming.api.functions.sink.SinkFunction;
+import org.apache.flink.streaming.api.functions.source.SourceFunction;
+import org.apache.flink.streaming.api.watermark.Watermark;
+import org.apache.flink.streaming.api.windowing.assigners.TumblingEventTimeWindows;
+import org.apache.flink.streaming.api.windowing.time.Time;
+import org.apache.flink.streaming.util.StreamingMultipleProgramsTestBase;
+import org.apache.flink.util.Collector;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+@SuppressWarnings("serial")
+public class CoGroupJoinITCase extends StreamingMultipleProgramsTestBase {
+
+ private static List<String> testResults;
+
+ @Test
+ public void testCoGroup() throws Exception {
+
+ testResults = new ArrayList<>();
+
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+ env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
+ env.setParallelism(1);
+
+ DataStream<Tuple2<String, Integer>> source1 = env.addSource(new SourceFunction<Tuple2<String, Integer>>() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public void run(SourceContext<Tuple2<String, Integer>> ctx) throws Exception {
+ ctx.collect(Tuple2.of("a", 0));
+ ctx.collect(Tuple2.of("a", 1));
+ ctx.collect(Tuple2.of("a", 2));
+
+ ctx.collect(Tuple2.of("b", 3));
+ ctx.collect(Tuple2.of("b", 4));
+ ctx.collect(Tuple2.of("b", 5));
+
+ ctx.collect(Tuple2.of("a", 6));
+ ctx.collect(Tuple2.of("a", 7));
+ ctx.collect(Tuple2.of("a", 8));
+
+ // source is finite, so it will have an implicit MAX watermark when it finishes
+ }
+
+ @Override
+ public void cancel() {
+ }
+ }).assignTimestampsAndWatermarks(new Tuple2TimestampExtractor());
+
+ DataStream<Tuple2<String, Integer>> source2 = env.addSource(new SourceFunction<Tuple2<String, Integer>>() {
+
+ @Override
+ public void run(SourceContext<Tuple2<String, Integer>> ctx) throws Exception {
+ ctx.collect(Tuple2.of("a", 0));
+ ctx.collect(Tuple2.of("a", 1));
+
+ ctx.collect(Tuple2.of("b", 3));
+
+ ctx.collect(Tuple2.of("c", 6));
+ ctx.collect(Tuple2.of("c", 7));
+ ctx.collect(Tuple2.of("c", 8));
+
+ // source is finite, so it will have an implicit MAX watermark when it finishes
+ }
+
+ @Override
+ public void cancel() {
+ }
+ }).assignTimestampsAndWatermarks(new Tuple2TimestampExtractor());
+
+
+ source1.coGroup(source2)
+ .where(new Tuple2KeyExtractor())
+ .equalTo(new Tuple2KeyExtractor())
+ .window(TumblingEventTimeWindows.of(Time.of(3, TimeUnit.MILLISECONDS)))
+ .apply(new CoGroupFunction<Tuple2<String,Integer>, Tuple2<String,Integer>, String>() {
+ @Override
+ public void coGroup(Iterable<Tuple2<String, Integer>> first,
+ Iterable<Tuple2<String, Integer>> second,
+ Collector<String> out) throws Exception {
+ StringBuilder result = new StringBuilder();
+ result.append("F:");
+ for (Tuple2<String, Integer> t: first) {
+ result.append(t.toString());
+ }
+ result.append(" S:");
+ for (Tuple2<String, Integer> t: second) {
+ result.append(t.toString());
+ }
+ out.collect(result.toString());
+ }
+ })
+ .addSink(new SinkFunction<String>() {
+ @Override
+ public void invoke(String value) throws Exception {
+ testResults.add(value);
+ }
+ });
+
+ env.execute("CoGroup Test");
+
+ List<String> expectedResult = Arrays.asList(
+ "F:(a,0)(a,1)(a,2) S:(a,0)(a,1)",
+ "F:(b,3)(b,4)(b,5) S:(b,3)",
+ "F:(a,6)(a,7)(a,8) S:",
+ "F: S:(c,6)(c,7)(c,8)");
+
+ Collections.sort(expectedResult);
+ Collections.sort(testResults);
+
+ Assert.assertEquals(expectedResult, testResults);
+ }
+
+ @Test
+ public void testJoin() throws Exception {
+
+ testResults = new ArrayList<>();
+
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+ env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
+ env.setParallelism(1);
+
+ DataStream<Tuple3<String, String, Integer>> source1 = env.addSource(new SourceFunction<Tuple3<String, String, Integer>>() {
+
+ @Override
+ public void run(SourceContext<Tuple3<String, String, Integer>> ctx) throws Exception {
+ ctx.collect(Tuple3.of("a", "x", 0));
+ ctx.collect(Tuple3.of("a", "y", 1));
+ ctx.collect(Tuple3.of("a", "z", 2));
+
+ ctx.collect(Tuple3.of("b", "u", 3));
+ ctx.collect(Tuple3.of("b", "w", 5));
+
+ ctx.collect(Tuple3.of("a", "i", 6));
+ ctx.collect(Tuple3.of("a", "j", 7));
+ ctx.collect(Tuple3.of("a", "k", 8));
+
+ // source is finite, so it will have an implicit MAX watermark when it finishes
+ }
+
+ @Override
+ public void cancel() {}
+
+ }).assignTimestampsAndWatermarks(new Tuple3TimestampExtractor());
+
+ DataStream<Tuple3<String, String, Integer>> source2 = env.addSource(new SourceFunction<Tuple3<String, String, Integer>>() {
+
+ @Override
+ public void run(SourceContext<Tuple3<String, String, Integer>> ctx) throws Exception {
+ ctx.collect(Tuple3.of("a", "u", 0));
+ ctx.collect(Tuple3.of("a", "w", 1));
+
+ ctx.collect(Tuple3.of("b", "i", 3));
+ ctx.collect(Tuple3.of("b", "k", 5));
+
+ ctx.collect(Tuple3.of("a", "x", 6));
+ ctx.collect(Tuple3.of("a", "z", 8));
+
+ // source is finite, so it will have an implicit MAX watermark when it finishes
+ }
+
+ @Override
+ public void cancel() {}
+
+ }).assignTimestampsAndWatermarks(new Tuple3TimestampExtractor());
+
+
+ source1.join(source2)
+ .where(new Tuple3KeyExtractor())
+ .equalTo(new Tuple3KeyExtractor())
+ .window(TumblingEventTimeWindows.of(Time.of(3, TimeUnit.MILLISECONDS)))
+ .apply(new JoinFunction<Tuple3<String, String, Integer>, Tuple3<String, String, Integer>, String>() {
+ @Override
+ public String join(Tuple3<String, String, Integer> first, Tuple3<String, String, Integer> second) throws Exception {
+ return first + ":" + second;
+ }
+ })
+ .addSink(new SinkFunction<String>() {
+ @Override
+ public void invoke(String value) throws Exception {
+ testResults.add(value);
+ }
+ });
+
+ env.execute("Join Test");
+
+ List<String> expectedResult = Arrays.asList(
+ "(a,x,0):(a,u,0)",
+ "(a,x,0):(a,w,1)",
+ "(a,y,1):(a,u,0)",
+ "(a,y,1):(a,w,1)",
+ "(a,z,2):(a,u,0)",
+ "(a,z,2):(a,w,1)",
+ "(b,u,3):(b,i,3)",
+ "(b,u,3):(b,k,5)",
+ "(b,w,5):(b,i,3)",
+ "(b,w,5):(b,k,5)",
+ "(a,i,6):(a,x,6)",
+ "(a,i,6):(a,z,8)",
+ "(a,j,7):(a,x,6)",
+ "(a,j,7):(a,z,8)",
+ "(a,k,8):(a,x,6)",
+ "(a,k,8):(a,z,8)");
+
+ Collections.sort(expectedResult);
+ Collections.sort(testResults);
+
+ Assert.assertEquals(expectedResult, testResults);
+ }
+
+ @Test
+ public void testSelfJoin() throws Exception {
+
+ testResults = new ArrayList<>();
+
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+ env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
+ env.setParallelism(1);
+
+ DataStream<Tuple3<String, String, Integer>> source1 = env.addSource(new SourceFunction<Tuple3<String, String, Integer>>() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public void run(SourceContext<Tuple3<String, String, Integer>> ctx) throws Exception {
+ ctx.collect(Tuple3.of("a", "x", 0));
+ ctx.collect(Tuple3.of("a", "y", 1));
+ ctx.collect(Tuple3.of("a", "z", 2));
+
+ ctx.collect(Tuple3.of("b", "u", 3));
+ ctx.collect(Tuple3.of("b", "w", 5));
+
+ ctx.collect(Tuple3.of("a", "i", 6));
+ ctx.collect(Tuple3.of("a", "j", 7));
+ ctx.collect(Tuple3.of("a", "k", 8));
+
+ // source is finite, so it will have an implicit MAX watermark when it finishes
+ }
+
+ @Override
+ public void cancel() {
+ }
+ }).assignTimestampsAndWatermarks(new Tuple3TimestampExtractor());
+
+ source1.join(source1)
+ .where(new Tuple3KeyExtractor())
+ .equalTo(new Tuple3KeyExtractor())
+ .window(TumblingEventTimeWindows.of(Time.of(3, TimeUnit.MILLISECONDS)))
+ .apply(new JoinFunction<Tuple3<String, String, Integer>, Tuple3<String, String, Integer>, String>() {
+ @Override
+ public String join(Tuple3<String, String, Integer> first, Tuple3<String, String, Integer> second) throws Exception {
+ return first + ":" + second;
+ }
+ })
+ .addSink(new SinkFunction<String>() {
+ @Override
+ public void invoke(String value) throws Exception {
+ testResults.add(value);
+ }
+ });
+
+ env.execute("Self-Join Test");
+
+ List<String> expectedResult = Arrays.asList(
+ "(a,x,0):(a,x,0)",
+ "(a,x,0):(a,y,1)",
+ "(a,x,0):(a,z,2)",
+ "(a,y,1):(a,x,0)",
+ "(a,y,1):(a,y,1)",
+ "(a,y,1):(a,z,2)",
+ "(a,z,2):(a,x,0)",
+ "(a,z,2):(a,y,1)",
+ "(a,z,2):(a,z,2)",
+ "(b,u,3):(b,u,3)",
+ "(b,u,3):(b,w,5)",
+ "(b,w,5):(b,u,3)",
+ "(b,w,5):(b,w,5)",
+ "(a,i,6):(a,i,6)",
+ "(a,i,6):(a,j,7)",
+ "(a,i,6):(a,k,8)",
+ "(a,j,7):(a,i,6)",
+ "(a,j,7):(a,j,7)",
+ "(a,j,7):(a,k,8)",
+ "(a,k,8):(a,i,6)",
+ "(a,k,8):(a,j,7)",
+ "(a,k,8):(a,k,8)");
+
+ Collections.sort(expectedResult);
+ Collections.sort(testResults);
+
+ Assert.assertEquals(expectedResult, testResults);
+ }
+
+ private static class Tuple2TimestampExtractor implements AssignerWithPunctuatedWatermarks<Tuple2<String, Integer>> {
+
+ @Override
+ public long extractTimestamp(Tuple2<String, Integer> element, long previousTimestamp) {
+ return element.f1;
+ }
+
+ @Override
+ public Watermark checkAndGetNextWatermark(Tuple2<String, Integer> element, long extractedTimestamp) {
+ return new Watermark(extractedTimestamp - 1);
+ }
+ }
+
+ private static class Tuple3TimestampExtractor implements AssignerWithPunctuatedWatermarks<Tuple3<String, String, Integer>> {
+
+ @Override
+ public long extractTimestamp(Tuple3<String, String, Integer> element, long previousTimestamp) {
+ return element.f2;
+ }
+
+ @Override
+ public Watermark checkAndGetNextWatermark(Tuple3<String, String, Integer> lastElement, long extractedTimestamp) {
+ return new Watermark(lastElement.f2 - 1);
+ }
+ }
+
+ private static class Tuple2KeyExtractor implements KeySelector<Tuple2<String,Integer>, String> {
+
+ @Override
+ public String getKey(Tuple2<String, Integer> value) throws Exception {
+ return value.f0;
+ }
+ }
+
+ private static class Tuple3KeyExtractor implements KeySelector<Tuple3<String, String, Integer>, String> {
+
+ @Override
+ public String getKey(Tuple3<String, String, Integer> value) throws Exception {
+ return value.f0;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/b9f42e91/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/CoStreamITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/CoStreamITCase.java b/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/CoStreamITCase.java
new file mode 100644
index 0000000..360ceb3
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/CoStreamITCase.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.test.streaming.runtime;
+
+import org.apache.flink.api.common.functions.FilterFunction;
+import org.apache.flink.api.common.functions.MapFunction;
+import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.functions.co.CoFlatMapFunction;
+import org.apache.flink.streaming.util.StreamingMultipleProgramsTestBase;
+import org.apache.flink.test.streaming.runtime.util.TestListResultSink;
+import org.apache.flink.util.Collector;
+
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+
+@SuppressWarnings("serial")
+public class CoStreamITCase extends StreamingMultipleProgramsTestBase {
+
+ @Test
+ public void test() throws Exception {
+
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+ env.setParallelism(1);
+
+ TestListResultSink<String> resultSink = new TestListResultSink<String>();
+
+ DataStream<Integer> src = env.fromElements(1, 3, 5);
+
+ DataStream<Integer> filter1 = src
+ .filter(new FilterFunction<Integer>() {
+ @Override
+ public boolean filter(Integer value) throws Exception {
+ return true;
+ }
+ })
+
+ .keyBy(new KeySelector<Integer, Integer>() {
+ @Override
+ public Integer getKey(Integer value) throws Exception {
+ return value;
+ }
+ });
+
+ DataStream<Tuple2<Integer, Integer>> filter2 = src
+ .map(new MapFunction<Integer, Tuple2<Integer, Integer>>() {
+
+ @Override
+ public Tuple2<Integer, Integer> map(Integer value) throws Exception {
+ return new Tuple2<>(value, value + 1);
+ }
+ })
+ .rebalance()
+ .filter(new FilterFunction<Tuple2<Integer, Integer>>() {
+
+ @Override
+ public boolean filter(Tuple2<Integer, Integer> value) throws Exception {
+ return true;
+ }
+ })
+ .disableChaining()
+ .keyBy(new KeySelector<Tuple2<Integer, Integer>, Integer>() {
+
+ @Override
+ public Integer getKey(Tuple2<Integer, Integer> value) throws Exception {
+ return value.f0;
+ }
+ });
+
+ DataStream<String> connected = filter1.connect(filter2)
+ .flatMap(new CoFlatMapFunction<Integer, Tuple2<Integer, Integer>, String>() {
+
+ @Override
+ public void flatMap1(Integer value, Collector<String> out) throws Exception {
+ out.collect(value.toString());
+ }
+
+ @Override
+ public void flatMap2(Tuple2<Integer, Integer> value, Collector<String> out) throws Exception {
+ out.collect(value.toString());
+ }
+ });
+
+ connected.addSink(resultSink);
+
+
+ env.execute();
+
+ List<String> expected = Arrays.asList("(1,2)", "(3,4)", "(5,6)", "1", "3", "5");
+ List<String> result = resultSink.getResult();
+ Collections.sort(result);
+ assertEquals(expected, result);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/b9f42e91/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/DataStreamPojoITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/DataStreamPojoITCase.java b/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/DataStreamPojoITCase.java
new file mode 100644
index 0000000..c345b37
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/DataStreamPojoITCase.java
@@ -0,0 +1,205 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.test.streaming.runtime;
+
+import org.apache.flink.api.common.functions.FlatMapFunction;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.util.StreamingMultipleProgramsTestBase;
+import org.apache.flink.util.Collector;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Integration test for streaming programs using POJOs and key selectors
+ *
+ * See FLINK-3697
+ */
+public class DataStreamPojoITCase extends StreamingMultipleProgramsTestBase {
+ static List<Data> elements = new ArrayList<>();
+ static {
+ elements.add(new Data(0,0,0));
+ elements.add(new Data(0,0,0));
+ elements.add(new Data(1,1,1));
+ elements.add(new Data(1,1,1));
+ elements.add(new Data(2,2,3));
+ elements.add(new Data(2,2,3));
+ }
+
+ /**
+ * Test composite key on the Data POJO (with nested fields)
+ */
+ @Test
+ public void testCompositeKeyOnNestedPojo() throws Exception {
+ StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment();
+ see.getConfig().disableObjectReuse();
+ see.setParallelism(3);
+
+ DataStream<Data> dataStream = see.fromCollection(elements);
+
+ DataStream<Data> summedStream = dataStream
+ .keyBy("aaa", "abc", "wxyz")
+ .sum("sum")
+ .keyBy("aaa", "abc", "wxyz")
+ .flatMap(new FlatMapFunction<Data, Data>() {
+ Data[] first = new Data[3];
+ @Override
+ public void flatMap(Data value, Collector<Data> out) throws Exception {
+ if(first[value.aaa] == null) {
+ first[value.aaa] = value;
+ if(value.sum != 1) {
+ throw new RuntimeException("Expected the sum to be one");
+ }
+ } else {
+ if(value.sum != 2) {
+ throw new RuntimeException("Expected the sum to be two");
+ }
+ if(first[value.aaa].aaa != value.aaa) {
+ throw new RuntimeException("aaa key wrong");
+ }
+ if(first[value.aaa].abc != value.abc) {
+ throw new RuntimeException("abc key wrong");
+ }
+ if(first[value.aaa].wxyz != value.wxyz) {
+ throw new RuntimeException("wxyz key wrong");
+ }
+ }
+ }
+ });
+
+ summedStream.print();
+
+ see.execute();
+ }
+
+ /**
+ * Test composite & nested key on the Data POJO
+ */
+ @Test
+ public void testNestedKeyOnNestedPojo() throws Exception {
+ StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment();
+ see.getConfig().disableObjectReuse();
+ see.setParallelism(4);
+
+ DataStream<Data> dataStream = see.fromCollection(elements);
+
+ DataStream<Data> summedStream = dataStream
+ .keyBy("aaa", "stats.count")
+ .sum("sum")
+ .keyBy("aaa", "stats.count")
+ .flatMap(new FlatMapFunction<Data, Data>() {
+ Data[] first = new Data[3];
+ @Override
+ public void flatMap(Data value, Collector<Data> out) throws Exception {
+ if(value.stats.count != 123) {
+ throw new RuntimeException("Wrong value for value.stats.count");
+ }
+ if(first[value.aaa] == null) {
+ first[value.aaa] = value;
+ if(value.sum != 1) {
+ throw new RuntimeException("Expected the sum to be one");
+ }
+ } else {
+ if(value.sum != 2) {
+ throw new RuntimeException("Expected the sum to be two");
+ }
+ if(first[value.aaa].aaa != value.aaa) {
+ throw new RuntimeException("aaa key wrong");
+ }
+ if(first[value.aaa].abc != value.abc) {
+ throw new RuntimeException("abc key wrong");
+ }
+ if(first[value.aaa].wxyz != value.wxyz) {
+ throw new RuntimeException("wxyz key wrong");
+ }
+ }
+ }
+ });
+
+ summedStream.print();
+
+ see.execute();
+ }
+
+
+ /**
+ * As per FLINK-3702 Flink doesn't support nested pojo fields for sum()
+ */
+ @Test(expected = IllegalArgumentException.class)
+ public void testFailOnNestedPojoFieldAccessor() throws Exception {
+ StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment();
+
+ DataStream<Data> dataStream = see.fromCollection(elements);
+ dataStream.keyBy("aaa", "stats.count").sum("stats.count");
+ }
+
+ public static class Data {
+ public int sum; // sum
+ public int aaa; // keyBy
+ public int abc; //keyBy
+ public long wxyz; // keyBy
+ public int t1;
+ public int t2;
+ public Policy policy;
+ public Stats stats;
+
+ public Data() {
+ }
+ public Data(int aaa, int abc, int wxyz) {
+ this.sum = 1;
+ this.aaa = aaa;
+ this.abc = abc;
+ this.wxyz = wxyz;
+ this.stats = new Stats();
+ this.stats.count = 123L;
+ }
+
+ @Override
+ public String toString() {
+ return "Data{" +
+ "sum=" + sum +
+ ", aaa=" + aaa +
+ ", abc=" + abc +
+ ", wxyz=" + wxyz +
+ '}';
+ }
+ }
+ public static class Policy {
+ public short a;
+ public short b;
+ public boolean c;
+ public boolean d;
+
+ public Policy() {}
+ }
+
+ public static class Stats {
+ public long count;
+ public float a;
+ public float b;
+ public float c;
+ public float d;
+ public float e;
+
+ public Stats() {}
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/flink/blob/b9f42e91/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/DirectedOutputITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/DirectedOutputITCase.java b/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/DirectedOutputITCase.java
new file mode 100644
index 0000000..8b84112
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/DirectedOutputITCase.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.test.streaming.runtime;
+
+import org.apache.flink.streaming.api.collector.selector.OutputSelector;
+import org.apache.flink.streaming.api.datastream.SplitStream;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.util.StreamingMultipleProgramsTestBase;
+import org.apache.flink.test.streaming.runtime.util.TestListResultSink;
+
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+
+public class DirectedOutputITCase extends StreamingMultipleProgramsTestBase {
+
+ private static final String TEN = "ten";
+ private static final String ODD = "odd";
+ private static final String EVEN = "even";
+ private static final String NON_SELECTED = "nonSelected";
+
+ static final class MyOutputSelector implements OutputSelector<Long> {
+ private static final long serialVersionUID = 1L;
+
+ List<String> outputs = new ArrayList<String>();
+
+ @Override
+ public Iterable<String> select(Long value) {
+ outputs.clear();
+ if (value % 2 == 0) {
+ outputs.add(EVEN);
+ } else {
+ outputs.add(ODD);
+ }
+
+ if (value == 10L) {
+ outputs.add(TEN);
+ }
+
+ if (value == 11L) {
+ outputs.add(NON_SELECTED);
+ }
+ return outputs;
+ }
+ }
+
+ @Test
+ public void outputSelectorTest() throws Exception {
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+ env.setParallelism(3);
+
+ TestListResultSink<Long> evenSink = new TestListResultSink<Long>();
+ TestListResultSink<Long> oddAndTenSink = new TestListResultSink<Long>();
+ TestListResultSink<Long> evenAndOddSink = new TestListResultSink<Long>();
+ TestListResultSink<Long> allSink = new TestListResultSink<Long>();
+
+ SplitStream<Long> source = env.generateSequence(1, 11).split(new MyOutputSelector());
+ source.select(EVEN).addSink(evenSink);
+ source.select(ODD, TEN).addSink(oddAndTenSink);
+ source.select(EVEN, ODD).addSink(evenAndOddSink);
+ source.addSink(allSink);
+
+ env.execute();
+ assertEquals(Arrays.asList(2L, 4L, 6L, 8L, 10L), evenSink.getSortedResult());
+ assertEquals(Arrays.asList(1L, 3L, 5L, 7L, 9L, 10L, 11L), oddAndTenSink.getSortedResult());
+ assertEquals(Arrays.asList(1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L, 10L, 11L),
+ evenAndOddSink.getSortedResult());
+ assertEquals(Arrays.asList(1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L, 10L, 11L),
+ allSink.getSortedResult());
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/b9f42e91/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/IterateITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/IterateITCase.java b/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/IterateITCase.java
new file mode 100644
index 0000000..1fbebd0
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/IterateITCase.java
@@ -0,0 +1,703 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.flink.test.streaming.runtime;
+
+import org.apache.flink.api.common.InvalidProgramException;
+import org.apache.flink.api.common.functions.MapFunction;
+import org.apache.flink.api.common.functions.RichFlatMapFunction;
+import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.runtime.jobgraph.JobGraph;
+import org.apache.flink.runtime.jobgraph.JobVertex;
+import org.apache.flink.streaming.api.CheckpointingMode;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.datastream.DataStreamSink;
+import org.apache.flink.streaming.api.datastream.IterativeStream;
+import org.apache.flink.streaming.api.datastream.IterativeStream.ConnectedIterativeStreams;
+import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
+import org.apache.flink.streaming.api.datastream.SplitStream;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.functions.co.CoMapFunction;
+import org.apache.flink.streaming.api.functions.co.RichCoFlatMapFunction;
+import org.apache.flink.streaming.api.functions.sink.SinkFunction;
+import org.apache.flink.streaming.api.graph.StreamEdge;
+import org.apache.flink.streaming.api.graph.StreamGraph;
+import org.apache.flink.streaming.api.graph.StreamNode;
+import org.apache.flink.streaming.runtime.partitioner.BroadcastPartitioner;
+import org.apache.flink.streaming.runtime.partitioner.ForwardPartitioner;
+import org.apache.flink.streaming.runtime.partitioner.RebalancePartitioner;
+import org.apache.flink.streaming.runtime.partitioner.ShufflePartitioner;
+import org.apache.flink.streaming.util.StreamingMultipleProgramsTestBase;
+import org.apache.flink.test.streaming.runtime.util.EvenOddOutputSelector;
+import org.apache.flink.test.streaming.runtime.util.NoOpIntMap;
+import org.apache.flink.test.streaming.runtime.util.ReceiveCheckNoOpSink;
+import org.apache.flink.util.Collector;
+import org.apache.flink.util.MathUtils;
+
+import org.junit.Test;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+@SuppressWarnings({ "unchecked", "unused", "serial" })
+public class IterateITCase extends StreamingMultipleProgramsTestBase {
+
+ private static final Logger LOG = LoggerFactory.getLogger(IterateITCase.class);
+
+ private static boolean iterated[];
+
+ @Test(expected = UnsupportedOperationException.class)
+ public void testIncorrectParallelism() throws Exception {
+
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+
+ DataStream<Integer> source = env.fromElements(1, 10);
+
+ IterativeStream<Integer> iter1 = source.iterate();
+ SingleOutputStreamOperator<Integer> map1 = iter1.map(NoOpIntMap);
+ iter1.closeWith(map1).print();
+ }
+
+ @Test
+ public void testDoubleClosing() throws Exception {
+
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+
+ // introduce dummy mapper to get to correct parallelism
+ DataStream<Integer> source = env.fromElements(1, 10).map(NoOpIntMap);
+
+ IterativeStream<Integer> iter1 = source.iterate();
+
+ iter1.closeWith(iter1.map(NoOpIntMap));
+ iter1.closeWith(iter1.map(NoOpIntMap));
+ }
+
+
+ @Test(expected = UnsupportedOperationException.class)
+ public void testDifferingParallelism() throws Exception {
+
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+
+ // introduce dummy mapper to get to correct parallelism
+ DataStream<Integer> source = env.fromElements(1, 10)
+ .map(NoOpIntMap);
+
+ IterativeStream<Integer> iter1 = source.iterate();
+
+
+ iter1.closeWith(iter1.map(NoOpIntMap).setParallelism(DEFAULT_PARALLELISM / 2));
+
+ }
+
+
+ @Test(expected = UnsupportedOperationException.class)
+ public void testCoDifferingParallelism() throws Exception {
+
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+
+ // introduce dummy mapper to get to correct parallelism
+ DataStream<Integer> source = env.fromElements(1, 10).map(NoOpIntMap);
+
+ ConnectedIterativeStreams<Integer, Integer> coIter = source.iterate().withFeedbackType(
+ Integer.class);
+
+
+ coIter.closeWith(coIter.map(NoOpIntCoMap).setParallelism(DEFAULT_PARALLELISM / 2));
+
+ }
+
+ @Test(expected = UnsupportedOperationException.class)
+ public void testClosingFromOutOfLoop() throws Exception {
+
+ // this test verifies that we cannot close an iteration with a DataStream that does not
+ // have the iteration in its predecessors
+
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+
+ // introduce dummy mapper to get to correct parallelism
+ DataStream<Integer> source = env.fromElements(1, 10).map(NoOpIntMap);
+
+ IterativeStream<Integer> iter1 = source.iterate();
+ IterativeStream<Integer> iter2 = source.iterate();
+
+
+ iter2.closeWith(iter1.map(NoOpIntMap));
+
+ }
+
+ @Test(expected = UnsupportedOperationException.class)
+ public void testCoIterClosingFromOutOfLoop() throws Exception {
+
+ // this test verifies that we cannot close an iteration with a DataStream that does not
+ // have the iteration in its predecessors
+
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+
+ // introduce dummy mapper to get to correct parallelism
+ DataStream<Integer> source = env.fromElements(1, 10).map(NoOpIntMap);
+
+ IterativeStream<Integer> iter1 = source.iterate();
+ ConnectedIterativeStreams<Integer, Integer> coIter = source.iterate().withFeedbackType(
+ Integer.class);
+
+
+ coIter.closeWith(iter1.map(NoOpIntMap));
+
+ }
+
+ @Test(expected = IllegalStateException.class)
+ public void testExecutionWithEmptyIteration() throws Exception {
+
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+
+ DataStream<Integer> source = env.fromElements(1, 10).map(NoOpIntMap);
+
+ IterativeStream<Integer> iter1 = source.iterate();
+
+ iter1.map(NoOpIntMap).print();
+
+ env.execute();
+ }
+
+ @Test
+ public void testImmutabilityWithCoiteration() {
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+
+ DataStream<Integer> source = env.fromElements(1, 10).map(NoOpIntMap); // for rebalance
+
+ IterativeStream<Integer> iter1 = source.iterate();
+ // Calling withFeedbackType should create a new iteration
+ ConnectedIterativeStreams<Integer, String> iter2 = iter1.withFeedbackType(String.class);
+
+ iter1.closeWith(iter1.map(NoOpIntMap)).print();
+ iter2.closeWith(iter2.map(NoOpCoMap)).print();
+
+ StreamGraph graph = env.getStreamGraph();
+
+ assertEquals(2, graph.getIterationSourceSinkPairs().size());
+
+ for (Tuple2<StreamNode, StreamNode> sourceSinkPair: graph.getIterationSourceSinkPairs()) {
+ assertEquals(sourceSinkPair.f0.getOutEdges().get(0).getTargetVertex(), sourceSinkPair.f1.getInEdges().get(0).getSourceVertex());
+ }
+ }
+
+ @Test
+ public void testmultipleHeadsTailsSimple() {
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+
+ DataStream<Integer> source1 = env.fromElements(1, 2, 3, 4, 5)
+ .shuffle()
+ .map(NoOpIntMap).name("ParallelizeMapShuffle");
+ DataStream<Integer> source2 = env.fromElements(1, 2, 3, 4, 5)
+ .map(NoOpIntMap).name("ParallelizeMapRebalance");
+
+ IterativeStream<Integer> iter1 = source1.union(source2).iterate();
+
+ DataStream<Integer> head1 = iter1.map(NoOpIntMap).name("IterRebalanceMap").setParallelism(DEFAULT_PARALLELISM / 2);
+ DataStream<Integer> head2 = iter1.map(NoOpIntMap).name("IterForwardMap");
+ DataStreamSink<Integer> head3 = iter1.map(NoOpIntMap).setParallelism(DEFAULT_PARALLELISM / 2).addSink(new ReceiveCheckNoOpSink<Integer>());
+ DataStreamSink<Integer> head4 = iter1.map(NoOpIntMap).addSink(new ReceiveCheckNoOpSink<Integer>());
+
+ SplitStream<Integer> source3 = env.fromElements(1, 2, 3, 4, 5)
+ .map(NoOpIntMap).name("EvenOddSourceMap")
+ .split(new EvenOddOutputSelector());
+
+ iter1.closeWith(source3.select("even").union(
+ head1.rebalance().map(NoOpIntMap).broadcast(), head2.shuffle()));
+
+ StreamGraph graph = env.getStreamGraph();
+
+ JobGraph jg = graph.getJobGraph();
+
+ assertEquals(1, graph.getIterationSourceSinkPairs().size());
+
+ Tuple2<StreamNode, StreamNode> sourceSinkPair = graph.getIterationSourceSinkPairs().iterator().next();
+ StreamNode itSource = sourceSinkPair.f0;
+ StreamNode itSink = sourceSinkPair.f1;
+
+ assertEquals(4, itSource.getOutEdges().size());
+ assertEquals(3, itSink.getInEdges().size());
+
+ assertEquals(itSource.getParallelism(), itSink.getParallelism());
+
+ for (StreamEdge edge : itSource.getOutEdges()) {
+ if (edge.getTargetVertex().getOperatorName().equals("IterRebalanceMap")) {
+ assertTrue(edge.getPartitioner() instanceof RebalancePartitioner);
+ } else if (edge.getTargetVertex().getOperatorName().equals("IterForwardMap")) {
+ assertTrue(edge.getPartitioner() instanceof ForwardPartitioner);
+ }
+ }
+ for (StreamEdge edge : itSink.getInEdges()) {
+ if (graph.getStreamNode(edge.getSourceId()).getOperatorName().equals("ParallelizeMapShuffle")) {
+ assertTrue(edge.getPartitioner() instanceof ShufflePartitioner);
+ }
+
+ if (graph.getStreamNode(edge.getSourceId()).getOperatorName().equals("ParallelizeMapForward")) {
+ assertTrue(edge.getPartitioner() instanceof ForwardPartitioner);
+ }
+
+ if (graph.getStreamNode(edge.getSourceId()).getOperatorName().equals("EvenOddSourceMap")) {
+ assertTrue(edge.getPartitioner() instanceof ForwardPartitioner);
+ assertTrue(edge.getSelectedNames().contains("even"));
+ }
+ }
+
+ // Test co-location
+
+ JobVertex itSource1 = null;
+ JobVertex itSink1 = null;
+
+ for (JobVertex vertex : jg.getVertices()) {
+ if (vertex.getName().contains("IterationSource")) {
+ itSource1 = vertex;
+ } else if (vertex.getName().contains("IterationSink")) {
+
+ itSink1 = vertex;
+
+ }
+ }
+
+ assertTrue(itSource1.getCoLocationGroup() != null);
+ assertEquals(itSource1.getCoLocationGroup(), itSink1.getCoLocationGroup());
+ }
+
+ @Test
+ public void testmultipleHeadsTailsWithTailPartitioning() {
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+
+ DataStream<Integer> source1 = env.fromElements(1, 2, 3, 4, 5)
+ .shuffle()
+ .map(NoOpIntMap);
+
+ DataStream<Integer> source2 = env.fromElements(1, 2, 3, 4, 5)
+ .map(NoOpIntMap);
+
+ IterativeStream<Integer> iter1 = source1.union(source2).iterate();
+
+ DataStream<Integer> head1 = iter1.map(NoOpIntMap).name("map1");
+ DataStream<Integer> head2 = iter1.map(NoOpIntMap)
+ .setParallelism(DEFAULT_PARALLELISM / 2)
+ .name("shuffle").rebalance();
+ DataStreamSink<Integer> head3 = iter1.map(NoOpIntMap).setParallelism(DEFAULT_PARALLELISM / 2)
+ .addSink(new ReceiveCheckNoOpSink<Integer>());
+ DataStreamSink<Integer> head4 = iter1.map(NoOpIntMap).addSink(new ReceiveCheckNoOpSink<Integer>());
+
+ SplitStream<Integer> source3 = env.fromElements(1, 2, 3, 4, 5)
+ .map(NoOpIntMap)
+ .name("split")
+ .split(new EvenOddOutputSelector());
+
+ iter1.closeWith(
+ source3.select("even").union(
+ head1.map(NoOpIntMap).name("bc").broadcast(),
+ head2.map(NoOpIntMap).shuffle()));
+
+ StreamGraph graph = env.getStreamGraph();
+
+ JobGraph jg = graph.getJobGraph();
+
+ assertEquals(1, graph.getIterationSourceSinkPairs().size());
+
+ Tuple2<StreamNode, StreamNode> sourceSinkPair = graph.getIterationSourceSinkPairs().iterator().next();
+ StreamNode itSource = sourceSinkPair.f0;
+ StreamNode itSink = sourceSinkPair.f1;
+
+ assertEquals(4, itSource.getOutEdges().size());
+ assertEquals(3, itSink.getInEdges().size());
+
+
+ assertEquals(itSource.getParallelism(), itSink.getParallelism());
+
+ for (StreamEdge edge : itSource.getOutEdges()) {
+ if (edge.getTargetVertex().getOperatorName().equals("map1")) {
+ assertTrue(edge.getPartitioner() instanceof ForwardPartitioner);
+ assertEquals(4, edge.getTargetVertex().getParallelism());
+ } else if (edge.getTargetVertex().getOperatorName().equals("shuffle")) {
+ assertTrue(edge.getPartitioner() instanceof RebalancePartitioner);
+ assertEquals(2, edge.getTargetVertex().getParallelism());
+ }
+ }
+ for (StreamEdge edge : itSink.getInEdges()) {
+ String tailName = edge.getSourceVertex().getOperatorName();
+ if (tailName.equals("split")) {
+ assertTrue(edge.getPartitioner() instanceof ForwardPartitioner);
+ assertTrue(edge.getSelectedNames().contains("even"));
+ } else if (tailName.equals("bc")) {
+ assertTrue(edge.getPartitioner() instanceof BroadcastPartitioner);
+ } else if (tailName.equals("shuffle")) {
+ assertTrue(edge.getPartitioner() instanceof ShufflePartitioner);
+ }
+ }
+
+ // Test co-location
+
+ JobVertex itSource1 = null;
+ JobVertex itSink1 = null;
+
+ for (JobVertex vertex : jg.getVertices()) {
+ if (vertex.getName().contains("IterationSource")) {
+ itSource1 = vertex;
+ } else if (vertex.getName().contains("IterationSink")) {
+ itSink1 = vertex;
+ }
+ }
+
+ assertTrue(itSource1.getCoLocationGroup() != null);
+ assertTrue(itSink1.getCoLocationGroup() != null);
+
+ assertEquals(itSource1.getCoLocationGroup(), itSink1.getCoLocationGroup());
+ }
+
+ @SuppressWarnings("rawtypes")
+ @Test
+ public void testSimpleIteration() throws Exception {
+ int numRetries = 5;
+ int timeoutScale = 1;
+
+ for (int numRetry = 0; numRetry < numRetries; numRetry++) {
+ try {
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+ iterated = new boolean[DEFAULT_PARALLELISM];
+
+ DataStream<Boolean> source = env.fromCollection(Collections.nCopies(DEFAULT_PARALLELISM * 2, false))
+ .map(NoOpBoolMap).name("ParallelizeMap");
+
+ IterativeStream<Boolean> iteration = source.iterate(3000 * timeoutScale);
+
+ DataStream<Boolean> increment = iteration.flatMap(new IterationHead()).map(NoOpBoolMap);
+
+ iteration.map(NoOpBoolMap).addSink(new ReceiveCheckNoOpSink());
+
+ iteration.closeWith(increment).addSink(new ReceiveCheckNoOpSink());
+
+ env.execute();
+
+ for (boolean iter : iterated) {
+ assertTrue(iter);
+ }
+
+ break; // success
+ } catch (Throwable t) {
+ LOG.info("Run " + (numRetry + 1) + "/" + numRetries + " failed", t);
+
+ if (numRetry >= numRetries - 1) {
+ throw t;
+ } else {
+ timeoutScale *= 2;
+ }
+ }
+ }
+ }
+
+ @Test
+ public void testCoIteration() throws Exception {
+ int numRetries = 5;
+ int timeoutScale = 1;
+
+ for (int numRetry = 0; numRetry < numRetries; numRetry++) {
+ try {
+ TestSink.collected = new ArrayList<>();
+
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+ env.setParallelism(2);
+
+ DataStream<String> otherSource = env.fromElements("1000", "2000")
+ .map(NoOpStrMap).name("ParallelizeMap");
+
+
+ ConnectedIterativeStreams<Integer, String> coIt = env.fromElements(0, 0)
+ .map(NoOpIntMap).name("ParallelizeMap")
+ .iterate(2000 * timeoutScale)
+ .withFeedbackType("String");
+
+ try {
+ coIt.keyBy(1, 2);
+ fail();
+ } catch (InvalidProgramException e) {
+ // this is expected
+ }
+
+ DataStream<String> head = coIt
+ .flatMap(new RichCoFlatMapFunction<Integer, String, String>() {
+
+ private static final long serialVersionUID = 1L;
+ boolean seenFromSource = false;
+
+ @Override
+ public void flatMap1(Integer value, Collector<String> out) throws Exception {
+ out.collect(((Integer) (value + 1)).toString());
+ }
+
+ @Override
+ public void flatMap2(String value, Collector<String> out) throws Exception {
+ Integer intVal = Integer.valueOf(value);
+ if (intVal < 2) {
+ out.collect(((Integer) (intVal + 1)).toString());
+ }
+ if (intVal == 1000 || intVal == 2000) {
+ seenFromSource = true;
+ }
+ }
+
+ @Override
+ public void close() {
+ assertTrue(seenFromSource);
+ }
+ });
+
+ coIt.map(new CoMapFunction<Integer, String, String>() {
+
+ @Override
+ public String map1(Integer value) throws Exception {
+ return value.toString();
+ }
+
+ @Override
+ public String map2(String value) throws Exception {
+ return value;
+ }
+ }).addSink(new ReceiveCheckNoOpSink<String>());
+
+ coIt.closeWith(head.broadcast().union(otherSource));
+
+ head.addSink(new TestSink()).setParallelism(1);
+
+ assertEquals(1, env.getStreamGraph().getIterationSourceSinkPairs().size());
+
+ env.execute();
+
+ Collections.sort(TestSink.collected);
+ assertEquals(Arrays.asList("1", "1", "2", "2", "2", "2"), TestSink.collected);
+
+ break; // success
+ } catch (Throwable t) {
+ LOG.info("Run " + (numRetry + 1) + "/" + numRetries + " failed", t);
+
+ if (numRetry >= numRetries - 1) {
+ throw t;
+ } else {
+ timeoutScale *= 2;
+ }
+ }
+ }
+ }
+
+ /**
+ * This test relies on the hash function used by the {@link DataStream#keyBy}, which is
+ * assumed to be {@link MathUtils#murmurHash}.
+ *
+ * For the test to pass all FlatMappers must see at least two records in the iteration,
+ * which can only be achieved if the hashed values of the input keys map to a complete
+ * congruence system. Given that the test is designed for 3 parallel FlatMapper instances
+ * keys chosen from the [1,3] range are a suitable choice.
+ */
+ @Test
+ public void testGroupByFeedback() throws Exception {
+ int numRetries = 5;
+ int timeoutScale = 1;
+
+ for (int numRetry = 0; numRetry < numRetries; numRetry++) {
+ try {
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+ env.setParallelism(DEFAULT_PARALLELISM - 1);
+
+ KeySelector<Integer, Integer> key = new KeySelector<Integer, Integer>() {
+
+ @Override
+ public Integer getKey(Integer value) throws Exception {
+ return value % 3;
+ }
+ };
+
+ DataStream<Integer> source = env.fromElements(1, 2, 3)
+ .map(NoOpIntMap).name("ParallelizeMap");
+
+ IterativeStream<Integer> it = source.keyBy(key).iterate(3000 * timeoutScale);
+
+ DataStream<Integer> head = it.flatMap(new RichFlatMapFunction<Integer, Integer>() {
+
+ int received = 0;
+ int key = -1;
+
+ @Override
+ public void flatMap(Integer value, Collector<Integer> out) throws Exception {
+ received++;
+ if (key == -1) {
+ key = MathUtils.murmurHash(value % 3) % 3;
+ } else {
+ assertEquals(key, MathUtils.murmurHash(value % 3) % 3);
+ }
+ if (value > 0) {
+ out.collect(value - 1);
+ }
+ }
+
+ @Override
+ public void close() {
+ assertTrue(received > 1);
+ }
+ });
+
+ it.closeWith(head.keyBy(key).union(head.map(NoOpIntMap).keyBy(key))).addSink(new ReceiveCheckNoOpSink<Integer>());
+
+ env.execute();
+
+ break; // success
+ } catch (Throwable t) {
+ LOG.info("Run " + (numRetry + 1) + "/" + numRetries + " failed", t);
+
+ if (numRetry >= numRetries - 1) {
+ throw t;
+ } else {
+ timeoutScale *= 2;
+ }
+ }
+ }
+ }
+
+ @SuppressWarnings("deprecation")
+ @Test
+ public void testWithCheckPointing() throws Exception {
+ int numRetries = 5;
+ int timeoutScale = 1;
+
+ for (int numRetry = 0; numRetry < numRetries; numRetry++) {
+ try {
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+
+ env.enableCheckpointing();
+
+ DataStream<Boolean> source = env.fromCollection(Collections.nCopies(DEFAULT_PARALLELISM * 2, false))
+ .map(NoOpBoolMap).name("ParallelizeMap");
+
+
+ IterativeStream<Boolean> iteration = source.iterate(3000 * timeoutScale);
+
+ iteration.closeWith(iteration.flatMap(new IterationHead())).addSink(new ReceiveCheckNoOpSink<Boolean>());
+
+ try {
+ env.execute();
+
+ // this statement should never be reached
+ fail();
+ } catch (UnsupportedOperationException e) {
+ // expected behaviour
+ }
+
+ // Test force checkpointing
+
+ try {
+ env.enableCheckpointing(1, CheckpointingMode.EXACTLY_ONCE, false);
+ env.execute();
+
+ // this statement should never be reached
+ fail();
+ } catch (UnsupportedOperationException e) {
+ // expected behaviour
+ }
+
+ env.enableCheckpointing(1, CheckpointingMode.EXACTLY_ONCE, true);
+ env.getStreamGraph().getJobGraph();
+
+ break; // success
+ } catch (Throwable t) {
+ LOG.info("Run " + (numRetry + 1) + "/" + numRetries + " failed", t);
+
+ if (numRetry >= numRetries - 1) {
+ throw t;
+ } else {
+ timeoutScale *= 2;
+ }
+ }
+ }
+ }
+
+ public static final class IterationHead extends RichFlatMapFunction<Boolean, Boolean> {
+ public void flatMap(Boolean value, Collector<Boolean> out) throws Exception {
+ int indx = getRuntimeContext().getIndexOfThisSubtask();
+ if (value) {
+ iterated[indx] = true;
+ } else {
+ out.collect(true);
+ }
+ }
+ }
+
+ public static CoMapFunction<Integer, String, String> NoOpCoMap = new CoMapFunction<Integer, String, String>() {
+
+ public String map1(Integer value) throws Exception {
+ return value.toString();
+ }
+
+ public String map2(String value) throws Exception {
+ return value;
+ }
+ };
+
+ public static MapFunction<Integer, Integer> NoOpIntMap = new NoOpIntMap();
+
+ public static MapFunction<String, String> NoOpStrMap = new MapFunction<String, String>() {
+
+ public String map(String value) throws Exception {
+ return value;
+ }
+
+ };
+
+ public static CoMapFunction<Integer, Integer, Integer> NoOpIntCoMap = new CoMapFunction<Integer, Integer, Integer>() {
+
+ public Integer map1(Integer value) throws Exception {
+ return value;
+ }
+
+ public Integer map2(Integer value) throws Exception {
+ return value;
+ }
+
+ };
+
+ public static MapFunction<Boolean, Boolean> NoOpBoolMap = new MapFunction<Boolean, Boolean>() {
+
+ public Boolean map(Boolean value) throws Exception {
+ return value;
+ }
+
+ };
+
+ public static class TestSink implements SinkFunction<String> {
+
+ private static final long serialVersionUID = 1L;
+ public static List<String> collected = new ArrayList<String>();
+
+ @Override
+ public void invoke(String value) throws Exception {
+ collected.add(value);
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/b9f42e91/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/OutputSplitterITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/OutputSplitterITCase.java b/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/OutputSplitterITCase.java
new file mode 100644
index 0000000..0902a3c
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/OutputSplitterITCase.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.test.streaming.runtime;
+
+import org.apache.flink.streaming.api.collector.selector.OutputSelector;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.util.StreamingMultipleProgramsTestBase;
+import org.apache.flink.test.streaming.runtime.util.TestListResultSink;
+
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+
+public class OutputSplitterITCase extends StreamingMultipleProgramsTestBase {
+
+ private static ArrayList<Integer> expectedSplitterResult = new ArrayList<Integer>();
+
+ @SuppressWarnings("unchecked")
+ @Test
+ public void testOnMergedDataStream() throws Exception {
+ TestListResultSink<Integer> splitterResultSink1 = new TestListResultSink<Integer>();
+ TestListResultSink<Integer> splitterResultSink2 = new TestListResultSink<Integer>();
+
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+ env.setParallelism(1);
+ env.setBufferTimeout(1);
+
+ DataStream<Integer> d1 = env.fromElements(0, 2, 4, 6, 8);
+ DataStream<Integer> d2 = env.fromElements(1, 3, 5, 7, 9);
+
+ d1 = d1.union(d2);
+
+ d1.split(new OutputSelector<Integer>() {
+ private static final long serialVersionUID = 8354166915727490130L;
+
+ @Override
+ public Iterable<String> select(Integer value) {
+ List<String> s = new ArrayList<String>();
+ if (value > 4) {
+ s.add(">");
+ } else {
+ s.add("<");
+ }
+ return s;
+ }
+ }).select(">").addSink(splitterResultSink1);
+
+ d1.split(new OutputSelector<Integer>() {
+ private static final long serialVersionUID = -6822487543355994807L;
+
+ @Override
+ public Iterable<String> select(Integer value) {
+ List<String> s = new ArrayList<String>();
+ if (value % 3 == 0) {
+ s.add("yes");
+ } else {
+ s.add("no");
+ }
+ return s;
+ }
+ }).select("yes").addSink(splitterResultSink2);
+ env.execute();
+
+ expectedSplitterResult.clear();
+ expectedSplitterResult.addAll(Arrays.asList(5, 6, 7, 8, 9));
+ assertEquals(expectedSplitterResult, splitterResultSink1.getSortedResult());
+
+ expectedSplitterResult.clear();
+ expectedSplitterResult.addAll(Arrays.asList(0, 3, 6, 9));
+ assertEquals(expectedSplitterResult, splitterResultSink2.getSortedResult());
+ }
+
+ @Test
+ public void testOnSingleDataStream() throws Exception {
+ TestListResultSink<Integer> splitterResultSink1 = new TestListResultSink<Integer>();
+ TestListResultSink<Integer> splitterResultSink2 = new TestListResultSink<Integer>();
+
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+ env.setParallelism(1);
+ env.setBufferTimeout(1);
+
+ DataStream<Integer> ds = env.fromElements(0, 1, 2, 3, 4, 5, 6, 7, 8, 9);
+
+ ds.split(new OutputSelector<Integer>() {
+ private static final long serialVersionUID = 2524335410904414121L;
+
+ @Override
+ public Iterable<String> select(Integer value) {
+ List<String> s = new ArrayList<String>();
+ if (value % 2 == 0) {
+ s.add("even");
+ } else {
+ s.add("odd");
+ }
+ return s;
+ }
+ }).select("even").addSink(splitterResultSink1);
+
+ ds.split(new OutputSelector<Integer>() {
+
+ private static final long serialVersionUID = -511693919586034092L;
+
+ @Override
+ public Iterable<String> select(Integer value) {
+ List<String> s = new ArrayList<String>();
+ if (value % 4 == 0) {
+ s.add("yes");
+ } else {
+ s.add("no");
+ }
+ return s;
+ }
+ }).select("yes").addSink(splitterResultSink2);
+ env.execute();
+
+ expectedSplitterResult.clear();
+ expectedSplitterResult.addAll(Arrays.asList(0, 2, 4, 6, 8));
+ assertEquals(expectedSplitterResult, splitterResultSink1.getSortedResult());
+
+ expectedSplitterResult.clear();
+ expectedSplitterResult.addAll(Arrays.asList(0, 4, 8));
+ assertEquals(expectedSplitterResult, splitterResultSink2.getSortedResult());
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/b9f42e91/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/PartitionerITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/PartitionerITCase.java b/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/PartitionerITCase.java
new file mode 100644
index 0000000..bff8df1
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/PartitionerITCase.java
@@ -0,0 +1,269 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.test.streaming.runtime;
+
+import org.apache.flink.api.common.functions.MapFunction;
+import org.apache.flink.api.common.functions.Partitioner;
+import org.apache.flink.api.common.functions.RichMapFunction;
+import org.apache.flink.api.common.functions.RuntimeContext;
+import org.apache.flink.api.java.tuple.Tuple1;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.util.StreamingMultipleProgramsTestBase;
+import org.apache.flink.test.streaming.runtime.util.NoOpIntMap;
+import org.apache.flink.test.streaming.runtime.util.TestListResultSink;
+
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+/**
+ * IT case that tests the different stream partitioning schemes.
+ */
+@SuppressWarnings("serial")
+public class PartitionerITCase extends StreamingMultipleProgramsTestBase {
+
+ @Test(expected = UnsupportedOperationException.class)
+ public void testForwardFailsLowToHighParallelism() throws Exception {
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+
+ DataStream<Integer> src = env.fromElements(1, 2, 3);
+
+ // this doesn't work because it goes from 1 to 3
+ src.forward().map(new NoOpIntMap());
+
+ env.execute();
+ }
+
+ @Test(expected = UnsupportedOperationException.class)
+ public void testForwardFailsHightToLowParallelism() throws Exception {
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+
+ // this does a rebalance that works
+ DataStream<Integer> src = env.fromElements(1, 2, 3).map(new NoOpIntMap());
+
+ // this doesn't work because it goes from 3 to 1
+ src.forward().map(new NoOpIntMap()).setParallelism(1);
+
+ env.execute();
+ }
+
+
+ @Test
+ public void partitionerTest() {
+
+ TestListResultSink<Tuple2<Integer, String>> hashPartitionResultSink =
+ new TestListResultSink<Tuple2<Integer, String>>();
+ TestListResultSink<Tuple2<Integer, String>> customPartitionResultSink =
+ new TestListResultSink<Tuple2<Integer, String>>();
+ TestListResultSink<Tuple2<Integer, String>> broadcastPartitionResultSink =
+ new TestListResultSink<Tuple2<Integer, String>>();
+ TestListResultSink<Tuple2<Integer, String>> forwardPartitionResultSink =
+ new TestListResultSink<Tuple2<Integer, String>>();
+ TestListResultSink<Tuple2<Integer, String>> rebalancePartitionResultSink =
+ new TestListResultSink<Tuple2<Integer, String>>();
+ TestListResultSink<Tuple2<Integer, String>> globalPartitionResultSink =
+ new TestListResultSink<Tuple2<Integer, String>>();
+
+
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+ env.setParallelism(3);
+
+ DataStream<Tuple1<String>> src = env.fromElements(
+ new Tuple1<String>("a"),
+ new Tuple1<String>("b"),
+ new Tuple1<String>("b"),
+ new Tuple1<String>("a"),
+ new Tuple1<String>("a"),
+ new Tuple1<String>("c"),
+ new Tuple1<String>("a")
+ );
+
+ // partition by hash
+ src
+ .keyBy(0)
+ .map(new SubtaskIndexAssigner())
+ .addSink(hashPartitionResultSink);
+
+ // partition custom
+ DataStream<Tuple2<Integer, String>> partitionCustom = src
+ .partitionCustom(new Partitioner<String>() {
+ @Override
+ public int partition(String key, int numPartitions) {
+ if (key.equals("c")) {
+ return 2;
+ } else {
+ return 0;
+ }
+ }
+ }, 0)
+ .map(new SubtaskIndexAssigner());
+
+ partitionCustom.addSink(customPartitionResultSink);
+
+ // partition broadcast
+ src.broadcast().map(new SubtaskIndexAssigner()).addSink(broadcastPartitionResultSink);
+
+ // partition rebalance
+ src.rebalance().map(new SubtaskIndexAssigner()).addSink(rebalancePartitionResultSink);
+
+ // partition forward
+ src.map(new MapFunction<Tuple1<String>, Tuple1<String>>() {
+ private static final long serialVersionUID = 1L;
+ @Override
+ public Tuple1<String> map(Tuple1<String> value) throws Exception {
+ return value;
+ }
+ })
+ .forward()
+ .map(new SubtaskIndexAssigner())
+ .addSink(forwardPartitionResultSink);
+
+ // partition global
+ src.global().map(new SubtaskIndexAssigner()).addSink(globalPartitionResultSink);
+
+ try {
+ env.execute();
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+
+ List<Tuple2<Integer, String>> hashPartitionResult = hashPartitionResultSink.getResult();
+ List<Tuple2<Integer, String>> customPartitionResult = customPartitionResultSink.getResult();
+ List<Tuple2<Integer, String>> broadcastPartitionResult = broadcastPartitionResultSink.getResult();
+ List<Tuple2<Integer, String>> forwardPartitionResult = forwardPartitionResultSink.getResult();
+ List<Tuple2<Integer, String>> rebalancePartitionResult = rebalancePartitionResultSink.getResult();
+ List<Tuple2<Integer, String>> globalPartitionResult = globalPartitionResultSink.getResult();
+
+ verifyHashPartitioning(hashPartitionResult);
+ verifyCustomPartitioning(customPartitionResult);
+ verifyBroadcastPartitioning(broadcastPartitionResult);
+ verifyRebalancePartitioning(forwardPartitionResult);
+ verifyRebalancePartitioning(rebalancePartitionResult);
+ verifyGlobalPartitioning(globalPartitionResult);
+ }
+
+ private static void verifyHashPartitioning(List<Tuple2<Integer, String>> hashPartitionResult) {
+ HashMap<String, Integer> verifier = new HashMap<String, Integer>();
+ for (Tuple2<Integer, String> elem : hashPartitionResult) {
+ Integer subtaskIndex = verifier.get(elem.f1);
+ if (subtaskIndex == null) {
+ verifier.put(elem.f1, elem.f0);
+ } else if (subtaskIndex != elem.f0) {
+ fail();
+ }
+ }
+ }
+
+ private static void verifyCustomPartitioning(List<Tuple2<Integer, String>> customPartitionResult) {
+ for (Tuple2<Integer, String> stringWithSubtask : customPartitionResult) {
+ if (stringWithSubtask.f1.equals("c")) {
+ assertEquals(new Integer(2), stringWithSubtask.f0);
+ } else {
+ assertEquals(new Integer(0), stringWithSubtask.f0);
+ }
+ }
+ }
+
+ private static void verifyBroadcastPartitioning(List<Tuple2<Integer, String>> broadcastPartitionResult) {
+ List<Tuple2<Integer, String>> expected = Arrays.asList(
+ new Tuple2<Integer, String>(0, "a"),
+ new Tuple2<Integer, String>(0, "b"),
+ new Tuple2<Integer, String>(0, "b"),
+ new Tuple2<Integer, String>(0, "a"),
+ new Tuple2<Integer, String>(0, "a"),
+ new Tuple2<Integer, String>(0, "c"),
+ new Tuple2<Integer, String>(0, "a"),
+ new Tuple2<Integer, String>(1, "a"),
+ new Tuple2<Integer, String>(1, "b"),
+ new Tuple2<Integer, String>(1, "b"),
+ new Tuple2<Integer, String>(1, "a"),
+ new Tuple2<Integer, String>(1, "a"),
+ new Tuple2<Integer, String>(1, "c"),
+ new Tuple2<Integer, String>(1, "a"),
+ new Tuple2<Integer, String>(2, "a"),
+ new Tuple2<Integer, String>(2, "b"),
+ new Tuple2<Integer, String>(2, "b"),
+ new Tuple2<Integer, String>(2, "a"),
+ new Tuple2<Integer, String>(2, "a"),
+ new Tuple2<Integer, String>(2, "c"),
+ new Tuple2<Integer, String>(2, "a"));
+
+ assertEquals(
+ new HashSet<Tuple2<Integer, String>>(expected),
+ new HashSet<Tuple2<Integer, String>>(broadcastPartitionResult));
+ }
+
+ private static void verifyRebalancePartitioning(List<Tuple2<Integer, String>> rebalancePartitionResult) {
+ List<Tuple2<Integer, String>> expected = Arrays.asList(
+ new Tuple2<Integer, String>(0, "a"),
+ new Tuple2<Integer, String>(1, "b"),
+ new Tuple2<Integer, String>(2, "b"),
+ new Tuple2<Integer, String>(0, "a"),
+ new Tuple2<Integer, String>(1, "a"),
+ new Tuple2<Integer, String>(2, "c"),
+ new Tuple2<Integer, String>(0, "a"));
+
+ assertEquals(
+ new HashSet<Tuple2<Integer, String>>(expected),
+ new HashSet<Tuple2<Integer, String>>(rebalancePartitionResult));
+ }
+
+ private static void verifyGlobalPartitioning(List<Tuple2<Integer, String>> globalPartitionResult) {
+ List<Tuple2<Integer, String>> expected = Arrays.asList(
+ new Tuple2<Integer, String>(0, "a"),
+ new Tuple2<Integer, String>(0, "b"),
+ new Tuple2<Integer, String>(0, "b"),
+ new Tuple2<Integer, String>(0, "a"),
+ new Tuple2<Integer, String>(0, "a"),
+ new Tuple2<Integer, String>(0, "c"),
+ new Tuple2<Integer, String>(0, "a"));
+
+ assertEquals(
+ new HashSet<Tuple2<Integer, String>>(expected),
+ new HashSet<Tuple2<Integer, String>>(globalPartitionResult));
+ }
+
+ private static class SubtaskIndexAssigner extends RichMapFunction<Tuple1<String>, Tuple2<Integer, String>> {
+ private static final long serialVersionUID = 1L;
+
+ private int indexOfSubtask;
+
+ @Override
+ public void open(Configuration parameters) throws Exception {
+ super.open(parameters);
+ RuntimeContext runtimeContext = getRuntimeContext();
+ indexOfSubtask = runtimeContext.getIndexOfThisSubtask();
+ }
+
+ @Override
+ public Tuple2<Integer, String> map(Tuple1<String> value) throws Exception {
+ return new Tuple2<Integer, String>(indexOfSubtask, value.f0);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/b9f42e91/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/SelfConnectionITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/SelfConnectionITCase.java b/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/SelfConnectionITCase.java
new file mode 100644
index 0000000..d33a2b1
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/SelfConnectionITCase.java
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.test.streaming.runtime;
+
+import org.apache.flink.api.common.functions.FlatMapFunction;
+import org.apache.flink.api.common.functions.MapFunction;
+import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.functions.co.CoMapFunction;
+import org.apache.flink.streaming.util.StreamingMultipleProgramsTestBase;
+import org.apache.flink.test.streaming.runtime.util.TestListResultSink;
+import org.apache.flink.util.Collector;
+
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+
+public class SelfConnectionITCase extends StreamingMultipleProgramsTestBase {
+
+ /**
+ * We connect two different data streams in a chain to a CoMap.
+ */
+ @Test
+ public void differentDataStreamSameChain() throws Exception {
+
+ TestListResultSink<String> resultSink = new TestListResultSink<String>();
+
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+ env.setParallelism(1);
+
+ DataStream<Integer> src = env.fromElements(1, 3, 5);
+
+ DataStream<String> stringMap = src.map(new MapFunction<Integer, String>() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public String map(Integer value) throws Exception {
+ return "x " + value;
+ }
+ });
+
+ stringMap.connect(src).map(new CoMapFunction<String, Integer, String>() {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public String map1(String value) {
+ return value;
+ }
+
+ @Override
+ public String map2(Integer value) {
+ return String.valueOf(value + 1);
+ }
+ }).addSink(resultSink);
+
+ env.execute();
+
+ List<String> expected = Arrays.asList("x 1", "x 3", "x 5", "2", "4", "6");
+
+ List<String> result = resultSink.getResult();
+
+ Collections.sort(expected);
+ Collections.sort(result);
+
+ assertEquals(expected, result);
+ }
+
+ /**
+ * We connect two different data streams in different chains to a CoMap.
+ * (This is not actually self-connect.)
+ */
+ @Test
+ public void differentDataStreamDifferentChain() {
+
+ TestListResultSink<String> resultSink = new TestListResultSink<String>();
+
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+ env.setParallelism(3);
+
+ DataStream<Integer> src = env.fromElements(1, 3, 5).disableChaining();
+
+ DataStream<String> stringMap = src.flatMap(new FlatMapFunction<Integer, String>() {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public void flatMap(Integer value, Collector<String> out) throws Exception {
+ out.collect("x " + value);
+ }
+ }).keyBy(new KeySelector<String, Integer>() {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Integer getKey(String value) throws Exception {
+ return value.length();
+ }
+ });
+
+ DataStream<Long> longMap = src.map(new MapFunction<Integer, Long>() {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Long map(Integer value) throws Exception {
+ return (long) (value + 1);
+ }
+ }).keyBy(new KeySelector<Long, Integer>() {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Integer getKey(Long value) throws Exception {
+ return value.intValue();
+ }
+ });
+
+
+ stringMap.connect(longMap).map(new CoMapFunction<String, Long, String>() {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public String map1(String value) {
+ return value;
+ }
+
+ @Override
+ public String map2(Long value) {
+ return value.toString();
+ }
+ }).addSink(resultSink);
+
+ try {
+ env.execute();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+
+ List<String> expected = Arrays.asList("x 1", "x 3", "x 5", "2", "4", "6");
+ List<String> result = resultSink.getResult();
+
+ Collections.sort(expected);
+ Collections.sort(result);
+
+ assertEquals(expected, result);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/b9f42e91/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/StateBackendITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/StateBackendITCase.java b/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/StateBackendITCase.java
new file mode 100644
index 0000000..6288946
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/StateBackendITCase.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.test.streaming.runtime;
+
+import org.apache.flink.api.common.functions.RichMapFunction;
+import org.apache.flink.api.common.restartstrategy.RestartStrategies;
+import org.apache.flink.api.common.state.FoldingState;
+import org.apache.flink.api.common.state.FoldingStateDescriptor;
+import org.apache.flink.api.common.state.ListState;
+import org.apache.flink.api.common.state.ListStateDescriptor;
+import org.apache.flink.api.common.state.ReducingState;
+import org.apache.flink.api.common.state.ReducingStateDescriptor;
+import org.apache.flink.api.common.state.ValueState;
+import org.apache.flink.api.common.state.ValueStateDescriptor;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.runtime.client.JobExecutionException;
+import org.apache.flink.runtime.execution.Environment;
+import org.apache.flink.runtime.state.AbstractStateBackend;
+import org.apache.flink.runtime.state.StateHandle;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.util.StreamingMultipleProgramsTestBase;
+import org.junit.Test;
+
+import java.io.Serializable;
+
+import static org.junit.Assert.fail;
+
+public class StateBackendITCase extends StreamingMultipleProgramsTestBase {
+
+ /**
+ * Verify that the user-specified state backend is used even if checkpointing is disabled.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testStateBackendWithoutCheckpointing() throws Exception {
+
+ StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment();
+ see.setParallelism(1);
+
+ see.getConfig().setRestartStrategy(RestartStrategies.noRestart());
+ see.setStateBackend(new FailingStateBackend());
+
+
+ see.fromElements(new Tuple2<>("Hello", 1))
+ .keyBy(0)
+ .map(new RichMapFunction<Tuple2<String,Integer>, String>() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public void open(Configuration parameters) throws Exception {
+ super.open(parameters);
+ getRuntimeContext().getKeyValueState("test", String.class, "");
+ }
+
+ @Override
+ public String map(Tuple2<String, Integer> value) throws Exception {
+ return value.f0;
+ }
+ })
+ .print();
+
+ try {
+ see.execute();
+ fail();
+ }
+ catch (JobExecutionException e) {
+ Throwable t = e.getCause();
+ if (!(t != null && t.getCause() instanceof SuccessException)) {
+ throw e;
+ }
+ }
+ }
+
+
+ public static class FailingStateBackend extends AbstractStateBackend {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public void initializeForJob(Environment env, String operatorIdentifier, TypeSerializer<?> keySerializer) throws Exception {
+ throw new SuccessException();
+ }
+
+ @Override
+ public void disposeAllStateForCurrentJob() throws Exception {}
+
+ @Override
+ public void close() throws Exception {}
+
+ @Override
+ protected <N, T> ValueState<T> createValueState(TypeSerializer<N> namespaceSerializer, ValueStateDescriptor<T> stateDesc) throws Exception {
+ return null;
+ }
+
+ @Override
+ protected <N, T> ListState<T> createListState(TypeSerializer<N> namespaceSerializer, ListStateDescriptor<T> stateDesc) throws Exception {
+ return null;
+ }
+
+ @Override
+ protected <N, T> ReducingState<T> createReducingState(TypeSerializer<N> namespaceSerializer, ReducingStateDescriptor<T> stateDesc) throws Exception {
+ return null;
+ }
+
+ @Override
+ protected <N, T, ACC> FoldingState<T, ACC> createFoldingState(TypeSerializer<N> namespaceSerializer,
+ FoldingStateDescriptor<T, ACC> stateDesc) throws Exception {
+ return null;
+ }
+
+ @Override
+ public CheckpointStateOutputStream createCheckpointStateOutputStream(long checkpointID,
+ long timestamp) throws Exception {
+ return null;
+ }
+
+ @Override
+ public <S extends Serializable> StateHandle<S> checkpointStateSerializable(S state,
+ long checkpointID,
+ long timestamp) throws Exception {
+ return null;
+ }
+ }
+
+ static final class SuccessException extends Exception {
+ private static final long serialVersionUID = -9218191172606739598L;
+ }
+
+}
[02/15] flink git commit: [FLINK-3995] [build] Properly structure
test scopes and dependencies
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-test-utils/src/test/java/org/apache/flink/test/testdata/WebLogAnalysisData.java
----------------------------------------------------------------------
diff --git a/flink-test-utils/src/test/java/org/apache/flink/test/testdata/WebLogAnalysisData.java b/flink-test-utils/src/test/java/org/apache/flink/test/testdata/WebLogAnalysisData.java
deleted file mode 100644
index ffbf7d1..0000000
--- a/flink-test-utils/src/test/java/org/apache/flink/test/testdata/WebLogAnalysisData.java
+++ /dev/null
@@ -1,149 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.test.testdata;
-
-
-public class WebLogAnalysisData {
- public static final String DOCS =
- "url_10|volutpat magna quis consectetuer volutpat ad erat editors exerci oscillations euismod volutpat Lorem convectionullamcorper Lorem volutpat enim tation elit |\n" +
- "url_11|adipiscing enim diam ex tincidunt tincidunt nonummy volutpat minim euismod volutpat suscipit ex sed laoreet aliquip quis diam tincidunt wisi diam elit sed ut minim ad nonummy amet volutpat nostrud erat |\n" +
- "url_12|ut nostrud adipiscing adipiscing ipsum nonummy amet volutpat volutpat sit enim Ut amet |\n" +
- "url_13|euismod sit adipiscing ex suscipit ea veniam tincidunt laoreet nibh editors ullamcorper consectetuer convection commodo sed nostrud ea ex ullamcorper dolor dolore ad diam dolore amet ut tincidunt nonummy euismod enim ullamcorper tincidunt dolor sit volutpat dolor tincidunt aliquam nisl tation ullamcorper sed consectetuer sit sit laoreet ex |\n" +
- "url_14|ad magna ipsum nonummy aliquip dolore aliquam veniam lobortis nostrud aliquip nibh amet aliquam editors magna aliquam volutpat nonummy sed tation erat adipiscing nostrud magna ut sit dolore volutpat laoreet nisl wisi Ut veniam nibh laoreet ad nostrud ut aliquip |\n" +
- "url_15|enim ipsum veniam ex editor Lorem elit laoreet exerci ea wisi oscillations convection euismod euismod diam ut euismod ad Lorem ut Ut tation wisi diam suscipit nibh nostrud minim dolor |\n" +
- "url_16|tation ad sed veniam lobortis editor nonummy Ut ea ipsum aliquip dolore ut laoreet tation ad ut Lorem ipsum minim nostrud quis Lorem enim |\n" +
- "url_17|veniam editor veniam Lorem aliquam ipsum amet ut convection veniam enim commodo ad ex magna nibh Ut ex sed ut nostrud amet volutpat nibh Ut wisi ea laoreet Lorem amet minim ullamcorper tincidunt veniam laoreet laoreet |\n" +
- "url_18|veniam diam tincidunt ullamcorper adipiscing oscillations adipiscing aliquip quis nibh suscipit ex sit sit wisi diam aliquam aliquip ea diam euismod ad erat ut ipsum lobortis ea exerci |\n" +
- "url_19|adipiscing nisl oscillations ea editors tincidunt tation convection tincidunt tincidunt ut wisi tincidunt ut ut Ut nibh aliquam laoreet exerci enim ea nibh erat nonummy |\n" +
- "url_20|ut diam aliquip ipsum laoreet elit volutpat suscipit nostrud convection tation nisl suscipit nonummy tation tation ut enim dolor nisl magna aliquam enim consectetuer sed tincidunt quis amet elit |\n" +
- "url_21|tation ad exerci volutpat tincidunt sit ullamcorper oscillations ut minim sed diam Lorem adipiscing |\n" +
- "url_22|sit exerci ad ut minim convection ad ea dolore Lorem ipsum amet sit wisi sed ullamcorper ipsum enim aliquam tincidunt elit nonummy laoreet laoreet dolore tation ullamcorper commodo Ut elit sit minim suscipit nisl laoreet |\n" +
- "url_23|Lorem nibh ea ea ex ut tation euismod tincidunt lobortis enim ullamcorper euismod amet magna sit erat enim editor diam dolor volutpat diam nisl ut erat commodo amet veniam consectetuer ex commodo sed magna ea erat ullamcorper dolor sed diam enim euismod |\n" +
- "url_24|nisl dolor magna editors ad consectetuer veniam commodo Ut dolor wisi dolore amet dolore dolore volutpat convection Ut oscillations |\n" +
- "url_25|ut euismod quis ad nostrud volutpat dolor wisi oscillations nostrud sed nonummy wisi exerci convection sed erat nostrud quis editors Ut nostrud consectetuer aliquip exerci ut Ut tincidunt aliquam suscipit erat |\n" +
- "url_26|nisl elit ea minim aliquam dolor consectetuer consectetuer quis Ut convection laoreet sit enim nostrud sed dolor magna dolor elit adipiscing |\n" +
- "url_27|nisl euismod ipsum ex adipiscing erat euismod diam quis oscillations aliquip nisl ut sit dolor wisi enim tincidunt amet ullamcorper adipiscing nibh Ut volutpat sed nonummy ex ea wisi exerci aliquam elit Ut aliquip nostrud ad nibh ut sit suscipit ut commodo wisi Lorem nibh |\n" +
- "url_28|laoreet sed editor dolor diam convection sed diam exerci laoreet oscillations consectetuer ullamcorper suscipit ut editors quis commodo editor veniam nibh ea diam ex magna ea elit sit sed nibh lobortis consectetuer erat erat sit minim ea ad ea nisl magna volutpat ut |\n" +
- "url_29|consectetuer convection amet diam erat euismod erat editor oscillations ipsum tation tation editors minim wisi tation quis adipiscing euismod nonummy erat ut diam suscipit tincidunt ut consectetuer Lorem editor sit quis euismod veniam tincidunt aliquam ut veniam adipiscing magna Ut ut dolore euismod minim veniam |";
-
- public static final String RANKS =
- "77|url_10|51|\n" + "7|url_11|24|\n" + "19|url_12|28|\n" + "13|url_13|50|\n" +
- "81|url_14|41|\n" + "87|url_15|28|\n" + "78|url_16|29|\n" + "70|url_17|7|\n" +
- "91|url_18|38|\n" + "99|url_19|11|\n" + "37|url_20|46|\n" + "64|url_21|46|\n" +
- "25|url_22|14|\n" + "6|url_23|12|\n" + "87|url_24|39|\n" + "0|url_25|27|\n" +
- "97|url_26|27|\n" + "9|url_27|54|\n" + "59|url_28|41|\n" + "40|url_29|11|";
-
- public static final String VISITS =
- "112.99.215.248|url_20|2011-5-1|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "189.69.147.166|url_19|2009-8-2|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "83.7.97.153|url_21|2010-3-20|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "215.57.124.59|url_10|2008-1-26|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "174.176.121.10|url_25|2007-12-17|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "66.225.121.11|url_16|2007-10-8|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "57.11.211.126|url_14|2010-12-6|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "129.127.115.129|url_28|2009-2-5|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "137.249.175.115|url_27|2011-5-7|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "101.92.9.112|url_12|2012-1-10|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "113.108.207.12|url_24|2011-3-20|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "179.141.87.220|url_18|2007-10-3|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "46.10.118.101|url_20|2009-7-4|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "91.67.84.155|url_20|2012-7-7|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "102.142.34.164|url_22|2010-7-18|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "203.67.94.61|url_14|2007-7-27|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "229.121.13.143|url_19|2010-4-17|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "209.154.189.12|url_16|2011-4-24|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "57.113.29.197|url_13|2008-9-23|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "132.90.139.12|url_16|2007-10-17|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "101.21.109.146|url_29|2009-2-24|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "171.69.34.255|url_29|2009-2-1|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "11.114.192.13|url_25|2008-9-8|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "37.188.212.141|url_15|2011-1-21|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "102.40.251.47|url_26|2010-7-17|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "136.85.230.245|url_17|2010-8-12|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "68.96.194.82|url_29|2007-10-12|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "62.126.40.210|url_29|2011-2-13|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "12.81.184.191|url_11|2008-1-12|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "12.232.205.52|url_18|2008-1-26|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "227.83.79.30|url_25|2008-6-4|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "88.222.17.152|url_21|2007-10-1|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "55.87.23.63|url_22|2008-8-16|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "54.41.77.25|url_18|2012-12-18|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "203.83.250.233|url_10|2009-4-5|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "42.13.134.196|url_15|2012-9-23|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "224.51.182.133|url_27|2008-11-13|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "180.209.212.98|url_27|2008-6-27|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "29.173.47.29|url_24|2011-9-18|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "251.153.164.73|url_18|2008-3-6|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "195.164.244.58|url_23|2010-12-21|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "96.167.244.51|url_18|2007-2-10|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "131.169.48.126|url_19|2009-6-10|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "171.151.109.83|url_12|2011-6-24|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "150.87.176.48|url_22|2008-10-3|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "56.172.201.54|TPCHQuery3ITCaseurl_25|2012-11-3|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "175.241.83.44|url_13|2008-2-22|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "105.222.1.251|url_18|2012-7-6|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "195.117.130.95|url_10|2012-4-24|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "10.139.236.176|url_14|2011-4-9|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "207.100.168.191|url_22|2007-5-1|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "196.129.126.177|url_25|2008-4-2|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "170.239.116.200|url_27|2010-3-15|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "99.79.118.85|url_13|2008-3-12|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "112.65.132.135|url_15|2010-8-3|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "109.15.235.196|url_14|2012-8-25|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "240.201.229.137|url_21|2008-9-22|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "74.53.245.171|url_18|2007-6-19|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "158.230.101.250|url_17|2011-4-17|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "198.197.213.210|url_11|2012-5-1|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "1.54.145.90|url_11|2009-6-16|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "2.236.14.86|url_13|2007-9-7|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "203.180.186.150|url_16|2010-2-17|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "135.83.247.101|url_22|2007-12-7|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "54.214.72.16|url_12|2007-5-2|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "238.115.186.34|url_10|2007-5-24|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "6.251.55.92|url_26|2007-5-18|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "196.96.11.111|url_16|2007-11-7|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "209.40.86.220|url_24|2009-5-17|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "130.208.66.140|url_19|2011-9-22|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "176.157.170.178|url_27|2008-12-12|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "103.4.119.128|url_15|2010-6-19|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "31.181.160.159|url_12|2009-8-10|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "166.238.60.79|url_23|2009-2-1|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "64.165.114.139|url_19|2007-9-17|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "11.45.238.55|url_26|2011-8-17|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "189.121.92.228|url_18|2008-1-16|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "137.114.181.208|url_14|2007-6-15|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "165.194.234.136|url_18|2012-3-7|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "210.145.85.234|url_10|2007-9-3|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "227.37.135.138|url_21|2010-5-4|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "39.87.120.86|url_25|2007-2-6|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "107.144.159.246|url_11|2010-7-6|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "160.38.166.236|url_15|2009-6-18|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "79.44.39.160|url_10|2008-12-19|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "143.52.62.175|url_19|2011-12-8|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "68.54.137.193|url_19|2007-6-6|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "199.137.128.158|url_27|2008-4-26|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "10.0.80.61|url_10|2011-3-15|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "4.30.48.164|url_29|2007-2-26|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "136.29.50.223|url_12|2008-3-5|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "83.101.200.110|url_23|2009-1-26|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "11.149.83.246|url_29|2010-9-18|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "154.119.20.134|url_14|2011-1-4|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "191.124.167.123|url_15|2009-6-24|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "27.158.167.38|url_11|2011-5-16|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "16.197.156.206|url_16|2009-2-1|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "222.150.206.233|url_20|2012-2-11|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "198.59.29.225|url_21|2007-2-25|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "182.91.244.40|url_18|2010-5-7|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "23.110.45.72|url_28|2008-6-23|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "94.241.8.36|url_11|2007-7-4|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "226.223.202.210|url_17|2007-10-15|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "73.16.194.132|url_14|2012-7-2|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "195.45.98.110|url_13|2012-2-23|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "189.123.80.178|url_29|2011-1-1|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "190.134.174.18|url_23|2011-12-17|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "133.54.252.245|url_13|2011-3-13|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "244.29.167.182|url_18|2012-10-11|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "125.32.251.232|url_26|2010-4-22|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "74.139.30.20|url_16|2012-2-8|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "100.73.180.209|url_29|2010-4-4|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "133.155.158.147|url_16|2011-11-22|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "185.53.212.102|url_15|2010-12-19|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "154.189.125.92|url_27|2007-10-5|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "12.209.166.107|url_26|2007-5-1|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "106.99.152.22|url_25|2011-12-4|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "37.120.32.53|url_22|2008-12-8|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "29.17.125.185|url_12|2011-8-18|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "233.125.12.14|url_19|2011-2-7|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "241.78.212.114|url_25|2007-12-22|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "153.183.247.90|url_18|2011-9-23|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "245.43.17.177|url_15|2012-4-21|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "55.13.54.220|url_24|2008-11-17|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "104.149.82.217|url_10|2011-9-26|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "36.15.167.145|url_24|2008-1-9|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "145.149.81.203|url_20|2008-5-27|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "68.89.4.115|url_18|2007-7-17|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "236.88.134.53|url_15|2010-5-2|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "180.140.58.209|url_29|2012-8-5|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "138.46.241.237|url_17|2010-11-13|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "254.217.79.58|url_29|2007-1-11|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "142.184.197.136|url_10|2007-11-3|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "216.118.94.220|url_12|2007-2-18|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "53.99.54.0|url_14|2008-3-9|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "95.151.221.37|url_28|2009-5-2|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "46.110.43.78|url_27|2007-11-9|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "172.96.203.180|url_11|2012-1-17|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "22.188.231.143|url_21|2007-10-23|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "216.46.80.239|url_22|2009-9-18|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "103.213.7.86|url_26|2007-12-25|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "186.204.3.180|url_27|2012-1-22|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "129.197.116.192|url_10|2012-5-1|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "8.164.181.242|url_24|2008-11-17|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "145.52.12.132|url_28|2012-8-6|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "20.22.62.146|url_11|2007-4-27|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "233.255.32.151|url_19|2008-6-25|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "31.233.226.148|url_24|2011-8-4|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "57.130.120.117|url_25|2008-12-9|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "29.98.249.152|url_21|2011-8-7|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "25.187.66.122|url_20|2008-12-17|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "236.205.44.215|url_26|2008-9-13|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "18.40.99.157|url_23|2012-1-7|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "204.221.88.206|url_24|2012-11-8|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "199.5.94.240|url_24|2011-9-24|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "57.175.34.24|url_14|2009-12-10|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "130.25.116.220|url_10|2007-6-5|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "134.174.12.250|url_22|2011-6-22|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "162.185.114.212|url_13|2011-8-24|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "8.113.137.95|url_14|2011-6-22|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "176.253.5.99|url_12|2010-4-12|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "3.110.24.0|url_12|2009-6-18|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "159.128.152.207|url_17|2012-2-2|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "104.145.150.244|url_10|2012-2-13|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "203.19.224.207|url_12|2012-9-24|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "225.163.125.108|url_20|2010-4-8|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "197.137.29.71|url_21|2007-10-25|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "29.207.144.144|url_12|2012-7-13|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "139.222.222.113|url_14|2010-4-15|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "169.225.20.141|url_29|2012-2-18|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "137.86.141.104|url_26|2012-3-10|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "193.100.2.122|url_24|2011-12-15|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "20.105.209.161|url_13|2010-2-7|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "116.81.52.207|url_11|2007-4-13|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "141.88.224.116|url_19|2009-4-12|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "72.198.58.37|url_26|2012-2-2|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "13.37.92.90|url_22|2011-5-16|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "191.163.173.142|url_24|2012-5-2|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "183.57.175.105|url_27|2007-8-13|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "230.212.34.87|url_10|2010-12-21|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" +
- "133.150.217.96|url_24|2012-7-27|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|\n" + "62.254.96.239|url_24|2011-5-15|0.12|Mozilla Firefox 3.1|de|de|Nothing special|124|";
-
- public static final String EXCEPTED_RESULT = "87|url_24|39\n" + "59|url_28|41\n";
-
- private WebLogAnalysisData() {}
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-test-utils/src/test/java/org/apache/flink/test/testdata/WordCountData.java
----------------------------------------------------------------------
diff --git a/flink-test-utils/src/test/java/org/apache/flink/test/testdata/WordCountData.java b/flink-test-utils/src/test/java/org/apache/flink/test/testdata/WordCountData.java
deleted file mode 100644
index 23c649b..0000000
--- a/flink-test-utils/src/test/java/org/apache/flink/test/testdata/WordCountData.java
+++ /dev/null
@@ -1,329 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.test.testdata;
-
-
-/**
- *
- */
-public class WordCountData {
-
- public static final String TEXT = "Goethe - Faust: Der Tragoedie erster Teil\n" + "Prolog im Himmel.\n"
- + "Der Herr. Die himmlischen Heerscharen. Nachher Mephistopheles. Die drei\n" + "Erzengel treten vor.\n"
- + "RAPHAEL: Die Sonne toent, nach alter Weise, In Brudersphaeren Wettgesang,\n"
- + "Und ihre vorgeschriebne Reise Vollendet sie mit Donnergang. Ihr Anblick\n"
- + "gibt den Engeln Staerke, Wenn keiner Sie ergruenden mag; die unbegreiflich\n"
- + "hohen Werke Sind herrlich wie am ersten Tag.\n"
- + "GABRIEL: Und schnell und unbegreiflich schnelle Dreht sich umher der Erde\n"
- + "Pracht; Es wechselt Paradieseshelle Mit tiefer, schauervoller Nacht. Es\n"
- + "schaeumt das Meer in breiten Fluessen Am tiefen Grund der Felsen auf, Und\n"
- + "Fels und Meer wird fortgerissen Im ewig schnellem Sphaerenlauf.\n"
- + "MICHAEL: Und Stuerme brausen um die Wette Vom Meer aufs Land, vom Land\n"
- + "aufs Meer, und bilden wuetend eine Kette Der tiefsten Wirkung rings umher.\n"
- + "Da flammt ein blitzendes Verheeren Dem Pfade vor des Donnerschlags. Doch\n"
- + "deine Boten, Herr, verehren Das sanfte Wandeln deines Tags.\n"
- + "ZU DREI: Der Anblick gibt den Engeln Staerke, Da keiner dich ergruenden\n"
- + "mag, Und alle deine hohen Werke Sind herrlich wie am ersten Tag.\n"
- + "MEPHISTOPHELES: Da du, o Herr, dich einmal wieder nahst Und fragst, wie\n"
- + "alles sich bei uns befinde, Und du mich sonst gewoehnlich gerne sahst, So\n"
- + "siehst du mich auch unter dem Gesinde. Verzeih, ich kann nicht hohe Worte\n"
- + "machen, Und wenn mich auch der ganze Kreis verhoehnt; Mein Pathos braechte\n"
- + "dich gewiss zum Lachen, Haettst du dir nicht das Lachen abgewoehnt. Von\n"
- + "Sonn' und Welten weiss ich nichts zu sagen, Ich sehe nur, wie sich die\n"
- + "Menschen plagen. Der kleine Gott der Welt bleibt stets von gleichem\n"
- + "Schlag, Und ist so wunderlich als wie am ersten Tag. Ein wenig besser\n"
- + "wuerd er leben, Haettst du ihm nicht den Schein des Himmelslichts gegeben;\n"
- + "Er nennt's Vernunft und braucht's allein, Nur tierischer als jedes Tier\n"
- + "zu sein. Er scheint mir, mit Verlaub von euer Gnaden, Wie eine der\n"
- + "langbeinigen Zikaden, Die immer fliegt und fliegend springt Und gleich im\n"
- + "Gras ihr altes Liedchen singt; Und laeg er nur noch immer in dem Grase! In\n"
- + "jeden Quark begraebt er seine Nase.\n"
- + "DER HERR: Hast du mir weiter nichts zu sagen? Kommst du nur immer\n"
- + "anzuklagen? Ist auf der Erde ewig dir nichts recht?\n"
- + "MEPHISTOPHELES: Nein Herr! ich find es dort, wie immer, herzlich\n"
- + "schlecht. Die Menschen dauern mich in ihren Jammertagen, Ich mag sogar\n"
- + "die armen selbst nicht plagen.\n" + "DER HERR: Kennst du den Faust?\n" + "MEPHISTOPHELES: Den Doktor?\n"
- + "DER HERR: Meinen Knecht!\n"
- + "MEPHISTOPHELES: Fuerwahr! er dient Euch auf besondre Weise. Nicht irdisch\n"
- + "ist des Toren Trank noch Speise. Ihn treibt die Gaerung in die Ferne, Er\n"
- + "ist sich seiner Tollheit halb bewusst; Vom Himmel fordert er die schoensten\n"
- + "Sterne Und von der Erde jede hoechste Lust, Und alle Naeh und alle Ferne\n"
- + "Befriedigt nicht die tiefbewegte Brust.\n"
- + "DER HERR: Wenn er mir auch nur verworren dient, So werd ich ihn bald in\n"
- + "die Klarheit fuehren. Weiss doch der Gaertner, wenn das Baeumchen gruent, Das\n"
- + "Bluet und Frucht die kuenft'gen Jahre zieren.\n"
- + "MEPHISTOPHELES: Was wettet Ihr? den sollt Ihr noch verlieren! Wenn Ihr\n"
- + "mir die Erlaubnis gebt, Ihn meine Strasse sacht zu fuehren.\n"
- + "DER HERR: Solang er auf der Erde lebt, So lange sei dir's nicht verboten,\n"
- + "Es irrt der Mensch so lang er strebt.\n"
- + "MEPHISTOPHELES: Da dank ich Euch; denn mit den Toten Hab ich mich niemals\n"
- + "gern befangen. Am meisten lieb ich mir die vollen, frischen Wangen. Fuer\n"
- + "einem Leichnam bin ich nicht zu Haus; Mir geht es wie der Katze mit der Maus.\n"
- + "DER HERR: Nun gut, es sei dir ueberlassen! Zieh diesen Geist von seinem\n"
- + "Urquell ab, Und fuehr ihn, kannst du ihn erfassen, Auf deinem Wege mit\n"
- + "herab, Und steh beschaemt, wenn du bekennen musst: Ein guter Mensch, in\n"
- + "seinem dunklen Drange, Ist sich des rechten Weges wohl bewusst.\n"
- + "MEPHISTOPHELES: Schon gut! nur dauert es nicht lange. Mir ist fuer meine\n"
- + "Wette gar nicht bange. Wenn ich zu meinem Zweck gelange, Erlaubt Ihr mir\n"
- + "Triumph aus voller Brust. Staub soll er fressen, und mit Lust, Wie meine\n"
- + "Muhme, die beruehmte Schlange.\n"
- + "DER HERR: Du darfst auch da nur frei erscheinen; Ich habe deinesgleichen\n"
- + "nie gehasst. Von allen Geistern, die verneinen, ist mir der Schalk am\n"
- + "wenigsten zur Last. Des Menschen Taetigkeit kann allzu leicht erschlaffen,\n"
- + "er liebt sich bald die unbedingte Ruh; Drum geb ich gern ihm den Gesellen\n"
- + "zu, Der reizt und wirkt und muss als Teufel schaffen. Doch ihr, die echten\n"
- + "Goettersoehne, Erfreut euch der lebendig reichen Schoene! Das Werdende, das\n"
- + "ewig wirkt und lebt, Umfass euch mit der Liebe holden Schranken, Und was\n"
- + "in schwankender Erscheinung schwebt, Befestigt mit dauernden Gedanken!\n"
- + "(Der Himmel schliesst, die Erzengel verteilen sich.)\n"
- + "MEPHISTOPHELES (allein): Von Zeit zu Zeit seh ich den Alten gern, Und\n"
- + "huete mich, mit ihm zu brechen. Es ist gar huebsch von einem grossen Herrn,\n"
- + "So menschlich mit dem Teufel selbst zu sprechen.";
-
- public static final String COUNTS = "machen 1\n" + "zeit 2\n" + "heerscharen 1\n" + "keiner 2\n" + "meine 3\n"
- + "fuehr 1\n" + "triumph 1\n" + "kommst 1\n" + "frei 1\n" + "schaffen 1\n" + "gesinde 1\n"
- + "langbeinigen 1\n" + "schalk 1\n" + "besser 1\n" + "solang 1\n" + "meer 4\n" + "fragst 1\n"
- + "gabriel 1\n" + "selbst 2\n" + "bin 1\n" + "sich 7\n" + "du 11\n" + "sogar 1\n" + "geht 1\n"
- + "immer 4\n" + "mensch 2\n" + "befestigt 1\n" + "lebt 2\n" + "mag 3\n" + "engeln 2\n" + "breiten 1\n"
- + "blitzendes 1\n" + "tags 1\n" + "sie 2\n" + "plagen 2\n" + "allzu 1\n" + "meisten 1\n" + "o 1\n"
- + "pfade 1\n" + "kennst 1\n" + "nichts 3\n" + "gedanken 1\n" + "befriedigt 1\n" + "mich 6\n" + "s 3\n"
- + "es 8\n" + "verneinen 1\n" + "er 13\n" + "gleich 1\n" + "baeumchen 1\n" + "donnergang 1\n"
- + "wunderlich 1\n" + "reise 1\n" + "urquell 1\n" + "doch 3\n" + "aufs 2\n" + "toten 1\n" + "niemals 1\n"
- + "eine 2\n" + "hab 1\n" + "darfst 1\n" + "da 5\n" + "gen 1\n" + "einem 2\n" + "teil 1\n" + "das 7\n"
- + "speise 1\n" + "wenig 1\n" + "sterne 1\n" + "geb 1\n" + "welten 1\n" + "alle 3\n" + "toent 1\n"
- + "gras 1\n" + "felsen 1\n" + "kette 1\n" + "ich 14\n" + "fuer 2\n" + "als 3\n" + "mein 1\n"
- + "schoene 1\n" + "verzeih 1\n" + "schwankender 1\n" + "wie 9\n" + "menschlich 1\n" + "gaertner 1\n"
- + "taetigkeit 1\n" + "bange 1\n" + "liebe 1\n" + "sei 2\n" + "seh 1\n" + "tollheit 1\n" + "am 6\n"
- + "michael 1\n" + "geist 1\n" + "ab 1\n" + "nahst 1\n" + "vollendet 1\n" + "liebt 1\n" + "brausen 1\n"
- + "nase 1\n" + "erlaubt 1\n" + "weiss 2\n" + "schnellem 1\n" + "deinem 1\n" + "gleichem 1\n"
- + "gaerung 1\n" + "dauernden 1\n" + "deines 1\n" + "vorgeschriebne 1\n" + "irdisch 1\n" + "worte 1\n"
- + "verehren 1\n" + "hohen 2\n" + "weise 2\n" + "kuenft 1\n" + "werdende 1\n" + "wette 2\n" + "wuetend 1\n"
- + "erscheinung 1\n" + "gar 2\n" + "verlieren 1\n" + "braucht 1\n" + "weiter 1\n" + "trank 1\n"
- + "tierischer 1\n" + "wohl 1\n" + "verteilen 1\n" + "verhoehnt 1\n" + "schaeumt 1\n" + "himmelslichts 1\n"
- + "unbedingte 1\n" + "herzlich 1\n" + "anblick 2\n" + "nennt 1\n" + "gruent 1\n" + "bluet 1\n"
- + "leichnam 1\n" + "erschlaffen 1\n" + "jammertagen 1\n" + "zieh 1\n" + "ihm 3\n" + "besondre 1\n"
- + "ihn 5\n" + "grossen 1\n" + "vollen 1\n" + "ihr 7\n" + "boten 1\n" + "voller 1\n" + "singt 1\n"
- + "muhme 1\n" + "schon 1\n" + "last 1\n" + "kleine 1\n" + "paradieseshelle 1\n" + "nein 1\n" + "echten 1\n"
- + "unter 1\n" + "bei 1\n" + "herr 11\n" + "gern 3\n" + "sphaerenlauf 1\n" + "stets 1\n" + "ganze 1\n"
- + "braechte 1\n" + "fordert 1\n" + "schoensten 1\n" + "herrlich 2\n" + "gegeben 1\n" + "allein 2\n"
- + "reichen 1\n" + "schauervoller 1\n" + "musst 1\n" + "recht 1\n" + "bleibt 1\n" + "pracht 1\n"
- + "treibt 1\n" + "befangen 1\n" + "was 2\n" + "menschen 3\n" + "jede 1\n" + "hohe 1\n" + "tiefsten 1\n"
- + "bilden 1\n" + "drum 1\n" + "gibt 2\n" + "guter 1\n" + "fuerwahr 1\n" + "im 3\n" + "grund 1\n" + "in 9\n"
- + "hoechste 1\n" + "schliesst 1\n" + "fels 1\n" + "steh 1\n" + "euer 1\n" + "erster 1\n" + "ersten 3\n"
- + "goettersoehne 1\n" + "brechen 1\n" + "tiefen 1\n" + "frucht 1\n" + "kreis 1\n" + "siehst 1\n"
- + "wege 1\n" + "ist 8\n" + "zikaden 1\n" + "frischen 1\n" + "ruh 1\n" + "deine 2\n" + "maus 1\n"
- + "brudersphaeren 1\n" + "nachher 1\n" + "euch 4\n" + "gnaden 1\n" + "anzuklagen 1\n" + "schlange 1\n"
- + "staerke 2\n" + "erde 4\n" + "verlaub 1\n" + "sanfte 1\n" + "holden 1\n" + "sonst 1\n" + "treten 1\n"
- + "sahst 1\n" + "alten 1\n" + "um 1\n" + "wieder 1\n" + "alter 1\n" + "altes 1\n" + "nun 1\n" + "lieb 1\n"
- + "gesellen 1\n" + "erscheinen 1\n" + "wirkt 2\n" + "haettst 2\n" + "nur 7\n" + "tiefbewegte 1\n"
- + "lachen 2\n" + "drange 1\n" + "schlag 1\n" + "schein 1\n" + "muss 1\n" + "verworren 1\n" + "weges 1\n"
- + "allen 1\n" + "gewoehnlich 1\n" + "alles 1\n" + "halb 1\n" + "stuerme 1\n" + "springt 1\n" + "sollt 1\n"
- + "klarheit 1\n" + "so 6\n" + "erfassen 1\n" + "liedchen 1\n" + "prolog 1\n" + "zur 1\n" + "fressen 1\n"
- + "zum 1\n" + "faust 2\n" + "erzengel 2\n" + "jahre 1\n" + "sonn 1\n" + "raphael 1\n" + "land 2\n"
- + "lang 1\n" + "gelange 1\n" + "lust 2\n" + "welt 1\n" + "sehe 1\n" + "ihre 1\n" + "jedes 1\n"
- + "erfreut 1\n" + "seiner 1\n" + "denn 1\n" + "wandeln 1\n" + "wechselt 1\n" + "jeden 1\n" + "dort 1\n"
- + "schlecht 1\n" + "wenigsten 1\n" + "wuerd 1\n" + "schranken 1\n" + "bewusst 2\n" + "seinem 2\n"
- + "gehasst 1\n" + "sein 1\n" + "meinem 1\n" + "meinen 1\n" + "pathos 1\n" + "herrn 1\n" + "lange 2\n"
- + "herab 1\n" + "diesen 1\n" + "ihren 1\n" + "beruehmte 1\n" + "goethe 1\n" + "tag 3\n" + "tier 1\n"
- + "quark 1\n" + "dank 1\n" + "seine 1\n" + "teufel 2\n" + "zweck 1\n" + "wenn 7\n" + "soll 1\n"
- + "wirkung 1\n" + "erlaubnis 1\n" + "lebendig 1\n" + "uns 1\n" + "leicht 1\n" + "gewiss 1\n"
- + "schnell 1\n" + "und 29\n" + "gerne 1\n" + "rechten 1\n" + "umher 2\n" + "vernunft 1\n" + "grase 1\n"
- + "nach 1\n" + "leben 1\n" + "gott 1\n" + "der 29\n" + "des 5\n" + "doktor 1\n" + "beschaemt 1\n"
- + "dreht 1\n" + "habe 1\n" + "sagen 2\n" + "bekennen 1\n" + "dunklen 1\n" + "wettet 1\n" + "den 9\n"
- + "mephistopheles 9\n" + "dem 4\n" + "auch 4\n" + "kann 2\n" + "armen 1\n" + "mir 9\n" + "strebt 1\n"
- + "gut 2\n" + "mit 11\n" + "bald 2\n" + "himmlischen 1\n" + "himmel 3\n" + "noch 3\n" + "kannst 1\n"
- + "deinesgleichen 1\n" + "flammt 1\n" + "ergruenden 2\n" + "nacht 1\n" + "scheint 1\n" + "ferne 2\n"
- + "tragoedie 1\n" + "abgewoehnt 1\n" + "reizt 1\n" + "geistern 1\n" + "nicht 10\n" + "sacht 1\n"
- + "unbegreiflich 2\n" + "schnelle 1\n" + "einmal 1\n" + "werd 1\n" + "werke 2\n" + "begraebt 1\n"
- + "knecht 1\n" + "rings 1\n" + "wird 1\n" + "katze 1\n" + "huete 1\n" + "fortgerissen 1\n" + "gebt 1\n"
- + "huebsch 1\n" + "hast 1\n" + "irrt 1\n" + "befinde 1\n" + "sind 2\n" + "fuehren 2\n" + "fliegt 1\n"
- + "ewig 3\n" + "brust 2\n" + "sonne 1\n" + "sprechen 1\n" + "ein 3\n" + "strasse 1\n" + "von 8\n"
- + "ueberlassen 1\n" + "dir 4\n" + "vom 3\n" + "zu 11\n" + "schwebt 1\n" + "die 22\n" + "vor 2\n"
- + "wangen 1\n" + "wettgesang 1\n" + "donnerschlags 1\n" + "find 1\n" + "dich 3\n" + "umfass 1\n"
- + "verboten 1\n" + "laeg 1\n" + "nie 1\n" + "drei 2\n" + "dauern 1\n" + "toren 1\n" + "dauert 1\n"
- + "verheeren 1\n" + "fliegend 1\n" + "aus 1\n" + "staub 1\n" + "fluessen 1\n" + "haus 1\n" + "auf 5\n"
- + "dient 2\n" + "tiefer 1\n" + "naeh 1\n" + "zieren 1\n";
-
- public static final String STREAMING_COUNTS_AS_TUPLES = "(machen,1)\n" + "(zeit,1)\n" + "(zeit,2)\n" + "(heerscharen,1)\n" + "(keiner,1)\n"
- + "(keiner,2)\n" + "(meine,1)\n" + "(meine,2)\n" + "(meine,3)\n" + "(fuehr,1)\n" + "(triumph,1)\n" + "(kommst,1)\n"
- + "(frei,1)\n" + "(schaffen,1)\n" + "(gesinde,1)\n" + "(langbeinigen,1)\n" + "(schalk,1)\n" + "(besser,1)\n" + "(solang,1)\n"
- + "(meer,1)\n" + "(meer,2)\n" + "(meer,3)\n" + "(meer,4)\n" + "(fragst,1)\n" + "(gabriel,1)\n" + "(selbst,1)\n" + "(selbst,2)\n"
- + "(bin,1)\n" + "(sich,1)\n" + "(sich,2)\n" + "(sich,3)\n" + "(sich,4)\n" + "(sich,5)\n" + "(sich,6)\n" + "(sich,7)\n" + "(du,1)\n"
- + "(du,2)\n" + "(du,3)\n" + "(du,4)\n" + "(du,5)\n" + "(du,6)\n" + "(du,7)\n" + "(du,8)\n" + "(du,9)\n" + "(du,10)\n" + "(du,11)\n"
- + "(sogar,1)\n" + "(geht,1)\n" + "(immer,1)\n" + "(immer,2)\n" + "(immer,3)\n" + "(immer,4)\n" + "(mensch,1)\n" + "(mensch,2)\n"
- + "(befestigt,1)\n" + "(lebt,1)\n" + "(lebt,2)\n" + "(mag,1)\n" + "(mag,2)\n" + "(mag,3)\n" + "(engeln,1)\n" + "(engeln,2)\n"
- + "(breiten,1)\n" + "(blitzendes,1)\n" + "(tags,1)\n" + "(sie,1)\n" + "(sie,2)\n" + "(plagen,1)\n" + "(plagen,2)\n" + "(allzu,1)\n"
- + "(meisten,1)\n" + "(o,1)\n" + "(pfade,1)\n" + "(kennst,1)\n" + "(nichts,1)\n" + "(nichts,2)\n" + "(nichts,3)\n" + "(gedanken,1)\n"
- + "(befriedigt,1)\n" + "(mich,1)\n" + "(mich,2)\n" + "(mich,3)\n" + "(mich,4)\n" + "(mich,5)\n" + "(mich,6)\n" + "(s,1)\n" + "(s,2)\n"
- + "(s,3)\n" + "(es,1)\n" + "(es,2)\n" + "(es,3)\n" + "(es,4)\n" + "(es,5)\n" + "(es,6)\n" + "(es,7)\n" + "(es,8)\n" + "(verneinen,1)\n"
- + "(er,1)\n" + "(er,2)\n" + "(er,3)\n" + "(er,4)\n" + "(er,5)\n" + "(er,6)\n" + "(er,7)\n" + "(er,8)\n" + "(er,9)\n" + "(er,10)\n"
- + "(er,11)\n" + "(er,12)\n" + "(er,13)\n" + "(gleich,1)\n" + "(baeumchen,1)\n" + "(donnergang,1)\n" + "(wunderlich,1)\n"
- + "(reise,1)\n" + "(urquell,1)\n" + "(doch,1)\n" + "(doch,2)\n" + "(doch,3)\n" + "(aufs,1)\n" + "(aufs,2)\n" + "(toten,1)\n"
- + "(niemals,1)\n" + "(eine,1)\n" + "(eine,2)\n" + "(hab,1)\n" + "(darfst,1)\n" + "(da,1)\n" + "(da,2)\n" + "(da,3)\n" + "(da,4)\n"
- + "(da,5)\n" + "(gen,1)\n" + "(einem,1)\n" + "(einem,2)\n" + "(teil,1)\n" + "(das,1)\n" + "(das,2)\n" + "(das,3)\n" + "(das,4)\n"
- + "(das,5)\n" + "(das,6)\n" + "(das,7)\n" + "(speise,1)\n" + "(wenig,1)\n" + "(sterne,1)\n" + "(geb,1)\n" + "(welten,1)\n"
- + "(alle,1)\n" + "(alle,2)\n" + "(alle,3)\n" + "(toent,1)\n" + "(gras,1)\n" + "(felsen,1)\n" + "(kette,1)\n" + "(ich,1)\n"
- + "(ich,2)\n" + "(ich,3)\n" + "(ich,4)\n" + "(ich,5)\n" + "(ich,6)\n" + "(ich,7)\n" + "(ich,8)\n" + "(ich,9)\n" + "(ich,10)\n"
- + "(ich,11)\n" + "(ich,12)\n" + "(ich,13)\n" + "(ich,14)\n" + "(fuer,1)\n" + "(fuer,2)\n" + "(als,1)\n" + "(als,2)\n" + "(als,3)\n"
- + "(mein,1)\n" + "(schoene,1)\n" + "(verzeih,1)\n" + "(schwankender,1)\n" + "(wie,1)\n" + "(wie,2)\n" + "(wie,3)\n" + "(wie,4)\n"
- + "(wie,5)\n" + "(wie,6)\n" + "(wie,7)\n" + "(wie,8)\n" + "(wie,9)\n" + "(menschlich,1)\n" + "(gaertner,1)\n" + "(taetigkeit,1)\n"
- + "(bange,1)\n" + "(liebe,1)\n" + "(sei,1)\n" + "(sei,2)\n" + "(seh,1)\n" + "(tollheit,1)\n" + "(am,1)\n" + "(am,2)\n" + "(am,3)\n"
- + "(am,4)\n" + "(am,5)\n" + "(am,6)\n" + "(michael,1)\n" + "(geist,1)\n" + "(ab,1)\n" + "(nahst,1)\n" + "(vollendet,1)\n"
- + "(liebt,1)\n" + "(brausen,1)\n" + "(nase,1)\n" + "(erlaubt,1)\n" + "(weiss,1)\n" + "(weiss,2)\n" + "(schnellem,1)\n"
- + "(deinem,1)\n" + "(gleichem,1)\n" + "(gaerung,1)\n" + "(dauernden,1)\n" + "(deines,1)\n" + "(vorgeschriebne,1)\n"
- + "(irdisch,1)\n" + "(worte,1)\n" + "(verehren,1)\n" + "(hohen,1)\n" + "(hohen,2)\n" + "(weise,1)\n" + "(weise,2)\n"
- + "(kuenft,1)\n" + "(werdende,1)\n" + "(wette,1)\n" + "(wette,2)\n" + "(wuetend,1)\n" + "(erscheinung,1)\n" + "(gar,1)\n"
- + "(gar,2)\n" + "(verlieren,1)\n" + "(braucht,1)\n" + "(weiter,1)\n" + "(trank,1)\n" + "(tierischer,1)\n" + "(wohl,1)\n"
- + "(verteilen,1)\n" + "(verhoehnt,1)\n" + "(schaeumt,1)\n" + "(himmelslichts,1)\n" + "(unbedingte,1)\n" + "(herzlich,1)\n"
- + "(anblick,1)\n" + "(anblick,2)\n" + "(nennt,1)\n" + "(gruent,1)\n" + "(bluet,1)\n" + "(leichnam,1)\n" + "(erschlaffen,1)\n"
- + "(jammertagen,1)\n" + "(zieh,1)\n" + "(ihm,1)\n" + "(ihm,2)\n" + "(ihm,3)\n" + "(besondre,1)\n" + "(ihn,1)\n" + "(ihn,2)\n"
- + "(ihn,3)\n" + "(ihn,4)\n" + "(ihn,5)\n" + "(grossen,1)\n" + "(vollen,1)\n" + "(ihr,1)\n" + "(ihr,2)\n" + "(ihr,3)\n"
- + "(ihr,4)\n" + "(ihr,5)\n" + "(ihr,6)\n" + "(ihr,7)\n" + "(boten,1)\n" + "(voller,1)\n" + "(singt,1)\n" + "(muhme,1)\n"
- + "(schon,1)\n" + "(last,1)\n" + "(kleine,1)\n" + "(paradieseshelle,1)\n" + "(nein,1)\n" + "(echten,1)\n" + "(unter,1)\n"
- + "(bei,1)\n" + "(herr,1)\n" + "(herr,2)\n" + "(herr,3)\n" + "(herr,4)\n" + "(herr,5)\n" + "(herr,6)\n" + "(herr,7)\n"
- + "(herr,8)\n" + "(herr,9)\n" + "(herr,10)\n" + "(herr,11)\n" + "(gern,1)\n" + "(gern,2)\n" + "(gern,3)\n" + "(sphaerenlauf,1)\n"
- + "(stets,1)\n" + "(ganze,1)\n" + "(braechte,1)\n" + "(fordert,1)\n" + "(schoensten,1)\n" + "(herrlich,1)\n" + "(herrlich,2)\n"
- + "(gegeben,1)\n" + "(allein,1)\n" + "(allein,2)\n" + "(reichen,1)\n" + "(schauervoller,1)\n" + "(musst,1)\n" + "(recht,1)\n"
- + "(bleibt,1)\n" + "(pracht,1)\n" + "(treibt,1)\n" + "(befangen,1)\n" + "(was,1)\n" + "(was,2)\n" + "(menschen,1)\n"
- + "(menschen,2)\n" + "(menschen,3)\n" + "(jede,1)\n" + "(hohe,1)\n" + "(tiefsten,1)\n" + "(bilden,1)\n" + "(drum,1)\n"
- + "(gibt,1)\n" + "(gibt,2)\n" + "(guter,1)\n" + "(fuerwahr,1)\n" + "(im,1)\n" + "(im,2)\n" + "(im,3)\n" + "(grund,1)\n"
- + "(in,1)\n" + "(in,2)\n" + "(in,3)\n" + "(in,4)\n" + "(in,5)\n" + "(in,6)\n" + "(in,7)\n" + "(in,8)\n" + "(in,9)\n" + "(hoechste,1)\n"
- + "(schliesst,1)\n" + "(fels,1)\n" + "(steh,1)\n" + "(euer,1)\n" + "(erster,1)\n" + "(ersten,1)\n" + "(ersten,2)\n" + "(ersten,3)\n"
- + "(goettersoehne,1)\n" + "(brechen,1)\n" + "(tiefen,1)\n" + "(frucht,1)\n" + "(kreis,1)\n" + "(siehst,1)\n" + "(wege,1)\n"
- + "(ist,1)\n" + "(ist,2)\n" + "(ist,3)\n" + "(ist,4)\n" + "(ist,5)\n" + "(ist,6)\n" + "(ist,7)\n" + "(ist,8)\n" + "(zikaden,1)\n"
- + "(frischen,1)\n" + "(ruh,1)\n" + "(deine,1)\n" + "(deine,2)\n" + "(maus,1)\n" + "(brudersphaeren,1)\n" + "(nachher,1)\n"
- + "(euch,1)\n" + "(euch,2)\n" + "(euch,3)\n" + "(euch,4)\n" + "(gnaden,1)\n" + "(anzuklagen,1)\n" + "(schlange,1)\n" + "(staerke,1)\n"
- + "(staerke,2)\n" + "(erde,1)\n" + "(erde,2)\n" + "(erde,3)\n" + "(erde,4)\n" + "(verlaub,1)\n" + "(sanfte,1)\n" + "(holden,1)\n"
- + "(sonst,1)\n" + "(treten,1)\n" + "(sahst,1)\n" + "(alten,1)\n" + "(um,1)\n" + "(wieder,1)\n" + "(alter,1)\n" + "(altes,1)\n"
- + "(nun,1)\n" + "(lieb,1)\n" + "(gesellen,1)\n" + "(erscheinen,1)\n" + "(wirkt,1)\n" + "(wirkt,2)\n" + "(haettst,1)\n" + "(haettst,2)\n"
- + "(nur,1)\n" + "(nur,2)\n" + "(nur,3)\n" + "(nur,4)\n" + "(nur,5)\n" + "(nur,6)\n" + "(nur,7)\n" + "(tiefbewegte,1)\n" + "(lachen,1)\n"
- + "(lachen,2)\n" + "(drange,1)\n" + "(schlag,1)\n" + "(schein,1)\n" + "(muss,1)\n" + "(verworren,1)\n" + "(weges,1)\n" + "(allen,1)\n"
- + "(gewoehnlich,1)\n" + "(alles,1)\n" + "(halb,1)\n" + "(stuerme,1)\n" + "(springt,1)\n" + "(sollt,1)\n" + "(klarheit,1)\n"
- + "(so,1)\n" + "(so,2)\n" + "(so,3)\n" + "(so,4)\n" + "(so,5)\n" + "(so,6)\n" + "(erfassen,1)\n" + "(liedchen,1)\n" + "(prolog,1)\n"
- + "(zur,1)\n" + "(fressen,1)\n" + "(zum,1)\n" + "(faust,1)\n" + "(faust,2)\n" + "(erzengel,1)\n" + "(erzengel,2)\n" + "(jahre,1)\n"
- + "(sonn,1)\n" + "(raphael,1)\n" + "(land,1)\n" + "(land,2)\n" + "(lang,1)\n" + "(gelange,1)\n" + "(lust,1)\n" + "(lust,2)\n"
- + "(welt,1)\n" + "(sehe,1)\n" + "(ihre,1)\n" + "(jedes,1)\n" + "(erfreut,1)\n" + "(seiner,1)\n" + "(denn,1)\n" + "(wandeln,1)\n"
- + "(wechselt,1)\n" + "(jeden,1)\n" + "(dort,1)\n" + "(schlecht,1)\n" + "(wenigsten,1)\n" + "(wuerd,1)\n" + "(schranken,1)\n"
- + "(bewusst,1)\n" + "(bewusst,2)\n" + "(seinem,1)\n" + "(seinem,2)\n" + "(gehasst,1)\n" + "(sein,1)\n" + "(meinem,1)\n"
- + "(meinen,1)\n" + "(pathos,1)\n" + "(herrn,1)\n" + "(lange,1)\n" + "(lange,2)\n" + "(herab,1)\n" + "(diesen,1)\n" + "(ihren,1)\n"
- + "(beruehmte,1)\n" + "(goethe,1)\n" + "(tag,1)\n" + "(tag,2)\n" + "(tag,3)\n" + "(tier,1)\n" + "(quark,1)\n" + "(dank,1)\n"
- + "(seine,1)\n" + "(teufel,1)\n" + "(teufel,2)\n" + "(zweck,1)\n" + "(wenn,1)\n" + "(wenn,2)\n" + "(wenn,3)\n" + "(wenn,4)\n"
- + "(wenn,5)\n" + "(wenn,6)\n" + "(wenn,7)\n" + "(soll,1)\n" + "(wirkung,1)\n" + "(erlaubnis,1)\n" + "(lebendig,1)\n" + "(uns,1)\n"
- + "(leicht,1)\n" + "(gewiss,1)\n" + "(schnell,1)\n" + "(und,1)\n" + "(und,2)\n" + "(und,3)\n" + "(und,4)\n" + "(und,5)\n" + "(und,6)\n"
- + "(und,7)\n" + "(und,8)\n" + "(und,9)\n" + "(und,10)\n" + "(und,11)\n" + "(und,12)\n" + "(und,13)\n" + "(und,14)\n" + "(und,15)\n"
- + "(und,16)\n" + "(und,17)\n" + "(und,18)\n" + "(und,19)\n" + "(und,20)\n" + "(und,21)\n" + "(und,22)\n" + "(und,23)\n" + "(und,24)\n"
- + "(und,25)\n" + "(und,26)\n" + "(und,27)\n" + "(und,28)\n" + "(und,29)\n" + "(gerne,1)\n" + "(rechten,1)\n" + "(umher,1)\n" + "(umher,2)\n"
- + "(vernunft,1)\n" + "(grase,1)\n" + "(nach,1)\n" + "(leben,1)\n" + "(gott,1)\n" + "(der,1)\n" + "(der,2)\n" + "(der,3)\n" + "(der,4)\n"
- + "(der,5)\n" + "(der,6)\n" + "(der,7)\n" + "(der,8)\n" + "(der,9)\n" + "(der,10)\n" + "(der,11)\n" + "(der,12)\n" + "(der,13)\n"
- + "(der,14)\n" + "(der,15)\n" + "(der,16)\n" + "(der,17)\n" + "(der,18)\n" + "(der,19)\n" + "(der,20)\n" + "(der,21)\n" + "(der,22)\n"
- + "(der,23)\n" + "(der,24)\n" + "(der,25)\n" + "(der,26)\n" + "(der,27)\n" + "(der,28)\n" + "(der,29)\n" + "(des,1)\n" + "(des,2)\n"
- + "(des,3)\n" + "(des,4)\n" + "(des,5)\n" + "(doktor,1)\n" + "(beschaemt,1)\n" + "(dreht,1)\n" + "(habe,1)\n" + "(sagen,1)\n" + "(sagen,2)\n"
- + "(bekennen,1)\n" + "(dunklen,1)\n" + "(wettet,1)\n" + "(den,1)\n" + "(den,2)\n" + "(den,3)\n" + "(den,4)\n" + "(den,5)\n" + "(den,6)\n"
- + "(den,7)\n" + "(den,8)\n" + "(den,9)\n" + "(mephistopheles,1)\n" + "(mephistopheles,2)\n" + "(mephistopheles,3)\n"
- + "(mephistopheles,4)\n" + "(mephistopheles,5)\n" + "(mephistopheles,6)\n" + "(mephistopheles,7)\n" + "(mephistopheles,8)\n"
- + "(mephistopheles,9)\n" + "(dem,1)\n" + "(dem,2)\n" + "(dem,3)\n" + "(dem,4)\n" + "(auch,1)\n" + "(auch,2)\n" + "(auch,3)\n" + "(auch,4)\n"
- + "(kann,1)\n" + "(kann,2)\n" + "(armen,1)\n" + "(mir,1)\n" + "(mir,2)\n" + "(mir,3)\n" + "(mir,4)\n" + "(mir,5)\n" + "(mir,6)\n" + "(mir,7)\n"
- + "(mir,8)\n" + "(mir,9)\n" + "(strebt,1)\n" + "(gut,1)\n" + "(gut,2)\n" + "(mit,1)\n" + "(mit,2)\n" + "(mit,3)\n" + "(mit,4)\n" + "(mit,5)\n"
- + "(mit,6)\n" + "(mit,7)\n" + "(mit,8)\n" + "(mit,9)\n" + "(mit,10)\n" + "(mit,11)\n" + "(bald,1)\n" + "(bald,2)\n" + "(himmlischen,1)\n"
- + "(himmel,1)\n" + "(himmel,2)\n" + "(himmel,3)\n" + "(noch,1)\n" + "(noch,2)\n" + "(noch,3)\n" + "(kannst,1)\n" + "(deinesgleichen,1)\n"
- + "(flammt,1)\n" + "(ergruenden,1)\n" + "(ergruenden,2)\n" + "(nacht,1)\n" + "(scheint,1)\n" + "(ferne,1)\n" + "(ferne,2)\n"
- + "(tragoedie,1)\n" + "(abgewoehnt,1)\n" + "(reizt,1)\n" + "(geistern,1)\n" + "(nicht,1)\n" + "(nicht,2)\n" + "(nicht,3)\n" + "(nicht,4)\n"
- + "(nicht,5)\n" + "(nicht,6)\n" + "(nicht,7)\n" + "(nicht,8)\n" + "(nicht,9)\n" + "(nicht,10)\n" + "(sacht,1)\n" + "(unbegreiflich,1)\n"
- + "(unbegreiflich,2)\n" + "(schnelle,1)\n" + "(einmal,1)\n" + "(werd,1)\n" + "(werke,1)\n" + "(werke,2)\n" + "(begraebt,1)\n"
- + "(knecht,1)\n" + "(rings,1)\n" + "(wird,1)\n" + "(katze,1)\n" + "(huete,1)\n" + "(fortgerissen,1)\n" + "(gebt,1)\n" + "(huebsch,1)\n"
- + "(hast,1)\n" + "(irrt,1)\n" + "(befinde,1)\n" + "(sind,1)\n" + "(sind,2)\n" + "(fuehren,1)\n" + "(fuehren,2)\n" + "(fliegt,1)\n"
- + "(ewig,1)\n" + "(ewig,2)\n" + "(ewig,3)\n" + "(brust,1)\n" + "(brust,2)\n" + "(sonne,1)\n" + "(sprechen,1)\n" + "(ein,1)\n" + "(ein,2)\n"
- + "(ein,3)\n" + "(strasse,1)\n" + "(von,1)\n" + "(von,2)\n" + "(von,3)\n" + "(von,4)\n" + "(von,5)\n" + "(von,6)\n" + "(von,7)\n" + "(von,8)\n"
- + "(ueberlassen,1)\n" + "(dir,1)\n" + "(dir,2)\n" + "(dir,3)\n" + "(dir,4)\n" + "(vom,1)\n" + "(vom,2)\n" + "(vom,3)\n" + "(zu,1)\n" + "(zu,2)\n"
- + "(zu,3)\n" + "(zu,4)\n" + "(zu,5)\n" + "(zu,6)\n" + "(zu,7)\n" + "(zu,8)\n" + "(zu,9)\n" + "(zu,10)\n" + "(zu,11)\n" + "(schwebt,1)\n"
- + "(die,1)\n" + "(die,2)\n" + "(die,3)\n" + "(die,4)\n" + "(die,5)\n" + "(die,6)\n" + "(die,7)\n" + "(die,8)\n" + "(die,9)\n" + "(die,10)\n"
- + "(die,11)\n" + "(die,12)\n" + "(die,13)\n" + "(die,14)\n" + "(die,15)\n" + "(die,16)\n" + "(die,17)\n" + "(die,18)\n" + "(die,19)\n"
- + "(die,20)\n" + "(die,21)\n" + "(die,22)\n" + "(vor,1)\n" + "(vor,2)\n" + "(wangen,1)\n" + "(wettgesang,1)\n" + "(donnerschlags,1)\n"
- + "(find,1)\n" + "(dich,1)\n" + "(dich,2)\n" + "(dich,3)\n" + "(umfass,1)\n" + "(verboten,1)\n" + "(laeg,1)\n" + "(nie,1)\n" + "(drei,1)\n"
- + "(drei,2)\n" + "(dauern,1)\n" + "(toren,1)\n" + "(dauert,1)\n" + "(verheeren,1)\n" + "(fliegend,1)\n" + "(aus,1)\n" + "(staub,1)\n"
- + "(fluessen,1)\n" + "(haus,1)\n" + "(auf,1)\n" + "(auf,2)\n" + "(auf,3)\n" + "(auf,4)\n" + "(auf,5)\n" + "(dient,1)\n" + "(dient,2)\n"
- + "(tiefer,1)\n" + "(naeh,1)\n" + "(zieren,1)\n";
-
- public static final String COUNTS_AS_TUPLES = "(machen,1)\n" + "(zeit,2)\n" + "(heerscharen,1)\n" + "(keiner,2)\n" + "(meine,3)\n"
- + "(fuehr,1)\n" + "(triumph,1)\n" + "(kommst,1)\n" + "(frei,1)\n" + "(schaffen,1)\n" + "(gesinde,1)\n"
- + "(langbeinigen,1)\n" + "(schalk,1)\n" + "(besser,1)\n" + "(solang,1)\n" + "(meer,4)\n" + "(fragst,1)\n"
- + "(gabriel,1)\n" + "(selbst,2)\n" + "(bin,1)\n" + "(sich,7)\n" + "(du,11)\n" + "(sogar,1)\n" + "(geht,1)\n"
- + "(immer,4)\n" + "(mensch,2)\n" + "(befestigt,1)\n" + "(lebt,2)\n" + "(mag,3)\n" + "(engeln,2)\n" + "(breiten,1)\n"
- + "(blitzendes,1)\n" + "(tags,1)\n" + "(sie,2)\n" + "(plagen,2)\n" + "(allzu,1)\n" + "(meisten,1)\n" + "(o,1)\n"
- + "(pfade,1)\n" + "(kennst,1)\n" + "(nichts,3)\n" + "(gedanken,1)\n" + "(befriedigt,1)\n" + "(mich,6)\n" + "(s,3)\n"
- + "(es,8)\n" + "(verneinen,1)\n" + "(er,13)\n" + "(gleich,1)\n" + "(baeumchen,1)\n" + "(donnergang,1)\n"
- + "(wunderlich,1)\n" + "(reise,1)\n" + "(urquell,1)\n" + "(doch,3)\n" + "(aufs,2)\n" + "(toten,1)\n" + "(niemals,1)\n"
- + "(eine,2)\n" + "(hab,1)\n" + "(darfst,1)\n" + "(da,5)\n" + "(gen,1)\n" + "(einem,2)\n" + "(teil,1)\n" + "(das,7)\n"
- + "(speise,1)\n" + "(wenig,1)\n" + "(sterne,1)\n" + "(geb,1)\n" + "(welten,1)\n" + "(alle,3)\n" + "(toent,1)\n"
- + "(gras,1)\n" + "(felsen,1)\n" + "(kette,1)\n" + "(ich,14)\n" + "(fuer,2)\n" + "(als,3)\n" + "(mein,1)\n"
- + "(schoene,1)\n" + "(verzeih,1)\n" + "(schwankender,1)\n" + "(wie,9)\n" + "(menschlich,1)\n" + "(gaertner,1)\n"
- + "(taetigkeit,1)\n" + "(bange,1)\n" + "(liebe,1)\n" + "(sei,2)\n" + "(seh,1)\n" + "(tollheit,1)\n" + "(am,6)\n"
- + "(michael,1)\n" + "(geist,1)\n" + "(ab,1)\n" + "(nahst,1)\n" + "(vollendet,1)\n" + "(liebt,1)\n" + "(brausen,1)\n"
- + "(nase,1)\n" + "(erlaubt,1)\n" + "(weiss,2)\n" + "(schnellem,1)\n" + "(deinem,1)\n" + "(gleichem,1)\n"
- + "(gaerung,1)\n" + "(dauernden,1)\n" + "(deines,1)\n" + "(vorgeschriebne,1)\n" + "(irdisch,1)\n" + "(worte,1)\n"
- + "(verehren,1)\n" + "(hohen,2)\n" + "(weise,2)\n" + "(kuenft,1)\n" + "(werdende,1)\n" + "(wette,2)\n" + "(wuetend,1)\n"
- + "(erscheinung,1)\n" + "(gar,2)\n" + "(verlieren,1)\n" + "(braucht,1)\n" + "(weiter,1)\n" + "(trank,1)\n"
- + "(tierischer,1)\n" + "(wohl,1)\n" + "(verteilen,1)\n" + "(verhoehnt,1)\n" + "(schaeumt,1)\n" + "(himmelslichts,1)\n"
- + "(unbedingte,1)\n" + "(herzlich,1)\n" + "(anblick,2)\n" + "(nennt,1)\n" + "(gruent,1)\n" + "(bluet,1)\n"
- + "(leichnam,1)\n" + "(erschlaffen,1)\n" + "(jammertagen,1)\n" + "(zieh,1)\n" + "(ihm,3)\n" + "(besondre,1)\n"
- + "(ihn,5)\n" + "(grossen,1)\n" + "(vollen,1)\n" + "(ihr,7)\n" + "(boten,1)\n" + "(voller,1)\n" + "(singt,1)\n"
- + "(muhme,1)\n" + "(schon,1)\n" + "(last,1)\n" + "(kleine,1)\n" + "(paradieseshelle,1)\n" + "(nein,1)\n" + "(echten,1)\n"
- + "(unter,1)\n" + "(bei,1)\n" + "(herr,11)\n" + "(gern,3)\n" + "(sphaerenlauf,1)\n" + "(stets,1)\n" + "(ganze,1)\n"
- + "(braechte,1)\n" + "(fordert,1)\n" + "(schoensten,1)\n" + "(herrlich,2)\n" + "(gegeben,1)\n" + "(allein,2)\n"
- + "(reichen,1)\n" + "(schauervoller,1)\n" + "(musst,1)\n" + "(recht,1)\n" + "(bleibt,1)\n" + "(pracht,1)\n"
- + "(treibt,1)\n" + "(befangen,1)\n" + "(was,2)\n" + "(menschen,3)\n" + "(jede,1)\n" + "(hohe,1)\n" + "(tiefsten,1)\n"
- + "(bilden,1)\n" + "(drum,1)\n" + "(gibt,2)\n" + "(guter,1)\n" + "(fuerwahr,1)\n" + "(im,3)\n" + "(grund,1)\n" + "(in,9)\n"
- + "(hoechste,1)\n" + "(schliesst,1)\n" + "(fels,1)\n" + "(steh,1)\n" + "(euer,1)\n" + "(erster,1)\n" + "(ersten,3)\n"
- + "(goettersoehne,1)\n" + "(brechen,1)\n" + "(tiefen,1)\n" + "(frucht,1)\n" + "(kreis,1)\n" + "(siehst,1)\n"
- + "(wege,1)\n" + "(ist,8)\n" + "(zikaden,1)\n" + "(frischen,1)\n" + "(ruh,1)\n" + "(deine,2)\n" + "(maus,1)\n"
- + "(brudersphaeren,1)\n" + "(nachher,1)\n" + "(euch,4)\n" + "(gnaden,1)\n" + "(anzuklagen,1)\n" + "(schlange,1)\n"
- + "(staerke,2)\n" + "(erde,4)\n" + "(verlaub,1)\n" + "(sanfte,1)\n" + "(holden,1)\n" + "(sonst,1)\n" + "(treten,1)\n"
- + "(sahst,1)\n" + "(alten,1)\n" + "(um,1)\n" + "(wieder,1)\n" + "(alter,1)\n" + "(altes,1)\n" + "(nun,1)\n" + "(lieb,1)\n"
- + "(gesellen,1)\n" + "(erscheinen,1)\n" + "(wirkt,2)\n" + "(haettst,2)\n" + "(nur,7)\n" + "(tiefbewegte,1)\n"
- + "(lachen,2)\n" + "(drange,1)\n" + "(schlag,1)\n" + "(schein,1)\n" + "(muss,1)\n" + "(verworren,1)\n" + "(weges,1)\n"
- + "(allen,1)\n" + "(gewoehnlich,1)\n" + "(alles,1)\n" + "(halb,1)\n" + "(stuerme,1)\n" + "(springt,1)\n" + "(sollt,1)\n"
- + "(klarheit,1)\n" + "(so,6)\n" + "(erfassen,1)\n" + "(liedchen,1)\n" + "(prolog,1)\n" + "(zur,1)\n" + "(fressen,1)\n"
- + "(zum,1)\n" + "(faust,2)\n" + "(erzengel,2)\n" + "(jahre,1)\n" + "(sonn,1)\n" + "(raphael,1)\n" + "(land,2)\n"
- + "(lang,1)\n" + "(gelange,1)\n" + "(lust,2)\n" + "(welt,1)\n" + "(sehe,1)\n" + "(ihre,1)\n" + "(jedes,1)\n"
- + "(erfreut,1)\n" + "(seiner,1)\n" + "(denn,1)\n" + "(wandeln,1)\n" + "(wechselt,1)\n" + "(jeden,1)\n" + "(dort,1)\n"
- + "(schlecht,1)\n" + "(wenigsten,1)\n" + "(wuerd,1)\n" + "(schranken,1)\n" + "(bewusst,2)\n" + "(seinem,2)\n"
- + "(gehasst,1)\n" + "(sein,1)\n" + "(meinem,1)\n" + "(meinen,1)\n" + "(pathos,1)\n" + "(herrn,1)\n" + "(lange,2)\n"
- + "(herab,1)\n" + "(diesen,1)\n" + "(ihren,1)\n" + "(beruehmte,1)\n" + "(goethe,1)\n" + "(tag,3)\n" + "(tier,1)\n"
- + "(quark,1)\n" + "(dank,1)\n" + "(seine,1)\n" + "(teufel,2)\n" + "(zweck,1)\n" + "(wenn,7)\n" + "(soll,1)\n"
- + "(wirkung,1)\n" + "(erlaubnis,1)\n" + "(lebendig,1)\n" + "(uns,1)\n" + "(leicht,1)\n" + "(gewiss,1)\n"
- + "(schnell,1)\n" + "(und,29)\n" + "(gerne,1)\n" + "(rechten,1)\n" + "(umher,2)\n" + "(vernunft,1)\n" + "(grase,1)\n"
- + "(nach,1)\n" + "(leben,1)\n" + "(gott,1)\n" + "(der,29)\n" + "(des,5)\n" + "(doktor,1)\n" + "(beschaemt,1)\n"
- + "(dreht,1)\n" + "(habe,1)\n" + "(sagen,2)\n" + "(bekennen,1)\n" + "(dunklen,1)\n" + "(wettet,1)\n" + "(den,9)\n"
- + "(mephistopheles,9)\n" + "(dem,4)\n" + "(auch,4)\n" + "(kann,2)\n" + "(armen,1)\n" + "(mir,9)\n" + "(strebt,1)\n"
- + "(gut,2)\n" + "(mit,11)\n" + "(bald,2)\n" + "(himmlischen,1)\n" + "(himmel,3)\n" + "(noch,3)\n" + "(kannst,1)\n"
- + "(deinesgleichen,1)\n" + "(flammt,1)\n" + "(ergruenden,2)\n" + "(nacht,1)\n" + "(scheint,1)\n" + "(ferne,2)\n"
- + "(tragoedie,1)\n" + "(abgewoehnt,1)\n" + "(reizt,1)\n" + "(geistern,1)\n" + "(nicht,10)\n" + "(sacht,1)\n"
- + "(unbegreiflich,2)\n" + "(schnelle,1)\n" + "(einmal,1)\n" + "(werd,1)\n" + "(werke,2)\n" + "(begraebt,1)\n"
- + "(knecht,1)\n" + "(rings,1)\n" + "(wird,1)\n" + "(katze,1)\n" + "(huete,1)\n" + "(fortgerissen,1)\n" + "(gebt,1)\n"
- + "(huebsch,1)\n" + "(hast,1)\n" + "(irrt,1)\n" + "(befinde,1)\n" + "(sind,2)\n" + "(fuehren,2)\n" + "(fliegt,1)\n"
- + "(ewig,3)\n" + "(brust,2)\n" + "(sonne,1)\n" + "(sprechen,1)\n" + "(ein,3)\n" + "(strasse,1)\n" + "(von,8)\n"
- + "(ueberlassen,1)\n" + "(dir,4)\n" + "(vom,3)\n" + "(zu,11)\n" + "(schwebt,1)\n" + "(die,22)\n" + "(vor,2)\n"
- + "(wangen,1)\n" + "(wettgesang,1)\n" + "(donnerschlags,1)\n" + "(find,1)\n" + "(dich,3)\n" + "(umfass,1)\n"
- + "(verboten,1)\n" + "(laeg,1)\n" + "(nie,1)\n" + "(drei,2)\n" + "(dauern,1)\n" + "(toren,1)\n" + "(dauert,1)\n"
- + "(verheeren,1)\n" + "(fliegend,1)\n" + "(aus,1)\n" + "(staub,1)\n" + "(fluessen,1)\n" + "(haus,1)\n" + "(auf,5)\n"
- + "(dient,2)\n" + "(tiefer,1)\n" + "(naeh,1)\n" + "(zieren,1)\n";
-
- private WordCountData() {
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-test-utils/src/test/java/org/apache/flink/test/util/AbstractTestBase.java
----------------------------------------------------------------------
diff --git a/flink-test-utils/src/test/java/org/apache/flink/test/util/AbstractTestBase.java b/flink-test-utils/src/test/java/org/apache/flink/test/util/AbstractTestBase.java
deleted file mode 100644
index c2da691..0000000
--- a/flink-test-utils/src/test/java/org/apache/flink/test/util/AbstractTestBase.java
+++ /dev/null
@@ -1,152 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.test.util;
-
-import com.google.common.base.Charsets;
-import com.google.common.io.Files;
-import org.apache.flink.configuration.Configuration;
-import scala.concurrent.duration.FiniteDuration;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Objects;
-
-import org.apache.flink.runtime.akka.AkkaUtils;
-
-/**
- * A base class for tests that run test programs in a Flink mini cluster.
- */
-public abstract class AbstractTestBase extends TestBaseUtils {
-
- /** Configuration to start the testing cluster with */
- protected final Configuration config;
-
- private final List<File> tempFiles;
-
- private final FiniteDuration timeout;
-
- protected int taskManagerNumSlots = 1;
-
- protected int numTaskManagers = 1;
-
- /** The mini cluster that runs the test programs */
- protected ForkableFlinkMiniCluster executor;
-
-
- public AbstractTestBase(Configuration config) {
- this.config = Objects.requireNonNull(config);
- this.tempFiles = new ArrayList<File>();
-
- timeout = AkkaUtils.getTimeout(config);
- }
-
- // --------------------------------------------------------------------------------------------
- // Local Test Cluster Life Cycle
- // --------------------------------------------------------------------------------------------
-
- public void startCluster() throws Exception {
- this.executor = startCluster(
- numTaskManagers,
- taskManagerNumSlots,
- false,
- false,
- true);
- }
-
- public void stopCluster() throws Exception {
- stopCluster(executor, timeout);
- deleteAllTempFiles();
- }
-
- //------------------
- // Accessors
- //------------------
-
- public int getTaskManagerNumSlots() {
- return taskManagerNumSlots;
- }
-
- public void setTaskManagerNumSlots(int taskManagerNumSlots) {
- this.taskManagerNumSlots = taskManagerNumSlots;
- }
-
- public int getNumTaskManagers() {
- return numTaskManagers;
- }
-
- public void setNumTaskManagers(int numTaskManagers) {
- this.numTaskManagers = numTaskManagers;
- }
-
-
- // --------------------------------------------------------------------------------------------
- // Temporary File Utilities
- // --------------------------------------------------------------------------------------------
-
- public String getTempDirPath(String dirName) throws IOException {
- File f = createAndRegisterTempFile(dirName);
- return f.toURI().toString();
- }
-
- public String getTempFilePath(String fileName) throws IOException {
- File f = createAndRegisterTempFile(fileName);
- return f.toURI().toString();
- }
-
- public String createTempFile(String fileName, String contents) throws IOException {
- File f = createAndRegisterTempFile(fileName);
- Files.write(contents, f, Charsets.UTF_8);
- return f.toURI().toString();
- }
-
- public File createAndRegisterTempFile(String fileName) throws IOException {
- File baseDir = new File(System.getProperty("java.io.tmpdir"));
- File f = new File(baseDir, this.getClass().getName() + "-" + fileName);
-
- if (f.exists()) {
- deleteRecursively(f);
- }
-
- File parentToDelete = f;
- while (true) {
- File parent = parentToDelete.getParentFile();
- if (parent == null) {
- throw new IOException("Missed temp dir while traversing parents of a temp file.");
- }
- if (parent.equals(baseDir)) {
- break;
- }
- parentToDelete = parent;
- }
-
- Files.createParentDirs(f);
- this.tempFiles.add(parentToDelete);
- return f;
- }
-
- private void deleteAllTempFiles() throws IOException {
- for (File f : this.tempFiles) {
- if (f.exists()) {
- deleteRecursively(f);
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-test-utils/src/test/java/org/apache/flink/test/util/CollectionTestEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-test-utils/src/test/java/org/apache/flink/test/util/CollectionTestEnvironment.java b/flink-test-utils/src/test/java/org/apache/flink/test/util/CollectionTestEnvironment.java
deleted file mode 100644
index e56c7e8..0000000
--- a/flink-test-utils/src/test/java/org/apache/flink/test/util/CollectionTestEnvironment.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.test.util;
-
-import org.apache.flink.api.common.JobExecutionResult;
-import org.apache.flink.api.java.CollectionEnvironment;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.ExecutionEnvironmentFactory;
-
-public class CollectionTestEnvironment extends CollectionEnvironment {
-
- private CollectionTestEnvironment lastEnv = null;
-
- @Override
- public JobExecutionResult getLastJobExecutionResult() {
- if (lastEnv == null) {
- return this.lastJobExecutionResult;
- }
- else {
- return lastEnv.getLastJobExecutionResult();
- }
- }
-
- @Override
- public JobExecutionResult execute() throws Exception {
- return execute("test job");
- }
-
- @Override
- public JobExecutionResult execute(String jobName) throws Exception {
- JobExecutionResult result = super.execute(jobName);
- this.lastJobExecutionResult = result;
- return result;
- }
-
- protected void setAsContext() {
- ExecutionEnvironmentFactory factory = new ExecutionEnvironmentFactory() {
- @Override
- public ExecutionEnvironment createExecutionEnvironment() {
- lastEnv = new CollectionTestEnvironment();
- return lastEnv;
- }
- };
-
- initializeContextEnvironment(factory);
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-test-utils/src/test/java/org/apache/flink/test/util/JavaProgramTestBase.java
----------------------------------------------------------------------
diff --git a/flink-test-utils/src/test/java/org/apache/flink/test/util/JavaProgramTestBase.java b/flink-test-utils/src/test/java/org/apache/flink/test/util/JavaProgramTestBase.java
deleted file mode 100644
index e639c80..0000000
--- a/flink-test-utils/src/test/java/org/apache/flink/test/util/JavaProgramTestBase.java
+++ /dev/null
@@ -1,279 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.test.util;
-
-import java.util.Comparator;
-
-import org.apache.flink.api.common.JobExecutionResult;
-import org.apache.flink.configuration.Configuration;
-import org.junit.Assert;
-import org.junit.Test;
-import org.apache.flink.api.java.tuple.Tuple;
-
-
-public abstract class JavaProgramTestBase extends AbstractTestBase {
-
- private static final int DEFAULT_PARALLELISM = 4;
-
- private JobExecutionResult latestExecutionResult;
-
- private int parallelism = DEFAULT_PARALLELISM;
-
- /**
- * The number of times a test should be repeated.
- *
- * <p> This is useful for runtime changes, which affect resource management. Running certain
- * tests repeatedly might help to discover resource leaks, race conditions etc.
- */
- private int numberOfTestRepetitions = 1;
-
- private boolean isCollectionExecution;
-
- public JavaProgramTestBase() {
- this(new Configuration());
- }
-
- public JavaProgramTestBase(Configuration config) {
- super(config);
- setTaskManagerNumSlots(parallelism);
- }
-
- public void setParallelism(int parallelism) {
- this.parallelism = parallelism;
- setTaskManagerNumSlots(parallelism);
- }
-
- public void setNumberOfTestRepetitions(int numberOfTestRepetitions) {
- this.numberOfTestRepetitions = numberOfTestRepetitions;
- }
-
- public int getParallelism() {
- return isCollectionExecution ? 1 : parallelism;
- }
-
- public JobExecutionResult getLatestExecutionResult() {
- return this.latestExecutionResult;
- }
-
- public boolean isCollectionExecution() {
- return isCollectionExecution;
- }
-
- // --------------------------------------------------------------------------------------------
- // Methods to create the test program and for pre- and post- test work
- // --------------------------------------------------------------------------------------------
-
- protected abstract void testProgram() throws Exception;
-
- protected void preSubmit() throws Exception {}
-
- protected void postSubmit() throws Exception {}
-
- protected boolean skipCollectionExecution() {
- return false;
- };
-
- // --------------------------------------------------------------------------------------------
- // Test entry point
- // --------------------------------------------------------------------------------------------
-
- @Test
- public void testJobWithObjectReuse() throws Exception {
- isCollectionExecution = false;
-
- startCluster();
- try {
- // pre-submit
- try {
- preSubmit();
- }
- catch (Exception e) {
- System.err.println(e.getMessage());
- e.printStackTrace();
- Assert.fail("Pre-submit work caused an error: " + e.getMessage());
- }
-
- // prepare the test environment
- TestEnvironment env = new TestEnvironment(this.executor, this.parallelism);
- env.getConfig().enableObjectReuse();
- env.setAsContext();
-
- // Possibly run the test multiple times
- for (int i = 0; i < numberOfTestRepetitions; i++) {
- // call the test program
- try {
- testProgram();
- this.latestExecutionResult = env.getLastJobExecutionResult();
- }
- catch (Exception e) {
- System.err.println(e.getMessage());
- e.printStackTrace();
- Assert.fail("Error while calling the test program: " + e.getMessage());
- }
-
- Assert.assertNotNull("The test program never triggered an execution.",
- this.latestExecutionResult);
- }
-
- // post-submit
- try {
- postSubmit();
- }
- catch (Exception e) {
- System.err.println(e.getMessage());
- e.printStackTrace();
- Assert.fail("Post-submit work caused an error: " + e.getMessage());
- }
- } finally {
- stopCluster();
- }
- }
-
- @Test
- public void testJobWithoutObjectReuse() throws Exception {
- isCollectionExecution = false;
-
- startCluster();
- try {
- // pre-submit
- try {
- preSubmit();
- }
- catch (Exception e) {
- System.err.println(e.getMessage());
- e.printStackTrace();
- Assert.fail("Pre-submit work caused an error: " + e.getMessage());
- }
-
- // prepare the test environment
- TestEnvironment env = new TestEnvironment(this.executor, this.parallelism);
- env.getConfig().disableObjectReuse();
- env.setAsContext();
-
- // Possibly run the test multiple times
- for (int i = 0; i < numberOfTestRepetitions; i++) {
- // call the test program
- try {
- testProgram();
- this.latestExecutionResult = env.getLastJobExecutionResult();
- }
- catch (Exception e) {
- System.err.println(e.getMessage());
- e.printStackTrace();
- Assert.fail("Error while calling the test program: " + e.getMessage());
- }
-
- Assert.assertNotNull("The test program never triggered an execution.",
- this.latestExecutionResult);
- }
-
- // post-submit
- try {
- postSubmit();
- }
- catch (Exception e) {
- System.err.println(e.getMessage());
- e.printStackTrace();
- Assert.fail("Post-submit work caused an error: " + e.getMessage());
- }
- } finally {
- stopCluster();
- }
- }
-
- @Test
- public void testJobCollectionExecution() throws Exception {
-
- // check if collection execution should be skipped.
- if(this.skipCollectionExecution()) {
- return;
- }
-
- isCollectionExecution = true;
-
- // pre-submit
- try {
- preSubmit();
- }
- catch (Exception e) {
- System.err.println(e.getMessage());
- e.printStackTrace();
- Assert.fail("Pre-submit work caused an error: " + e.getMessage());
- }
-
- // prepare the test environment
- CollectionTestEnvironment env = new CollectionTestEnvironment();
- env.setAsContext();
-
- // call the test program
- try {
- testProgram();
- this.latestExecutionResult = env.getLastJobExecutionResult();
- }
- catch (Exception e) {
- System.err.println(e.getMessage());
- e.printStackTrace();
- Assert.fail("Error while calling the test program: " + e.getMessage());
- }
-
- Assert.assertNotNull("The test program never triggered an execution.", this.latestExecutionResult);
-
- // post-submit
- try {
- postSubmit();
- }
- catch (Exception e) {
- System.err.println(e.getMessage());
- e.printStackTrace();
- Assert.fail("Post-submit work caused an error: " + e.getMessage());
- }
- }
-
- public static class TupleComparator<T extends Tuple> implements Comparator<T> {
-
- @SuppressWarnings("unchecked")
- @Override
- public int compare(T o1, T o2) {
- int a1 = o1.getArity();
- int a2 = o2.getArity();
-
- if (a1 < a2) {
- return -1;
- } else if (a2 < a1) {
- return 1;
- } else {
- for (int i = 0; i < a1; i++) {
- Object obj1 = o1.getField(i);
- Object obj2 = o2.getField(i);
-
- if (!(obj1 instanceof Comparable && obj2 instanceof Comparable)) {
- Assert.fail("Cannot compare tuple fields");
- }
-
- int cmp = ((Comparable<Object>) obj1).compareTo(obj2);
- if (cmp != 0) {
- return cmp;
- }
- }
-
- return 0;
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-test-utils/src/test/java/org/apache/flink/test/util/MultipleProgramsTestBase.java
----------------------------------------------------------------------
diff --git a/flink-test-utils/src/test/java/org/apache/flink/test/util/MultipleProgramsTestBase.java b/flink-test-utils/src/test/java/org/apache/flink/test/util/MultipleProgramsTestBase.java
deleted file mode 100644
index d7f09bd..0000000
--- a/flink-test-utils/src/test/java/org/apache/flink/test/util/MultipleProgramsTestBase.java
+++ /dev/null
@@ -1,126 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.test.util;
-
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.runners.Parameterized;
-
-import java.util.Arrays;
-import java.util.Collection;
-
-/**
- * Base class for unit tests that run multiple tests and want to reuse the same
- * Flink cluster. This saves a significant amount of time, since the startup and
- * shutdown of the Flink clusters (including actor systems, etc) usually dominates
- * the execution of the actual tests.
- *
- * To write a unit test against this test base, simply extend it and add
- * one or more regular test methods and retrieve the ExecutionEnvironment from
- * the context:
- *
- * <pre>{@code
- *
- * {@literal @}Test
- * public void someTest() {
- * ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- * // test code
- * env.execute();
- * }
- *
- * {@literal @}Test
- * public void anotherTest() {
- * ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- * // test code
- * env.execute();
- * }
- *
- * }</pre>
- */
-public class MultipleProgramsTestBase extends TestBaseUtils {
-
- /**
- * Enum that defines which execution environment to run the next test on:
- * An embedded local flink cluster, or the collection execution backend.
- */
- public enum TestExecutionMode {
- CLUSTER,
- COLLECTION
- }
-
- // ------------------------------------------------------------------------
- // The mini cluster that is shared across tests
- // ------------------------------------------------------------------------
-
- protected static final int DEFAULT_PARALLELISM = 4;
-
- protected static boolean startWebServer = false;
-
- protected static ForkableFlinkMiniCluster cluster = null;
-
- // ------------------------------------------------------------------------
-
- protected final TestExecutionMode mode;
-
-
- public MultipleProgramsTestBase(TestExecutionMode mode) {
- this.mode = mode;
-
- switch(mode){
- case CLUSTER:
- TestEnvironment clusterEnv = new TestEnvironment(cluster, 4);
- clusterEnv.setAsContext();
- break;
- case COLLECTION:
- CollectionTestEnvironment collectionEnv = new CollectionTestEnvironment();
- collectionEnv.setAsContext();
- break;
- }
- }
-
- // ------------------------------------------------------------------------
- // Cluster setup & teardown
- // ------------------------------------------------------------------------
-
- @BeforeClass
- public static void setup() throws Exception {
- cluster = TestBaseUtils.startCluster(
- 1,
- DEFAULT_PARALLELISM,
- startWebServer,
- false,
- true);
- }
-
- @AfterClass
- public static void teardown() throws Exception {
- stopCluster(cluster, TestBaseUtils.DEFAULT_TIMEOUT);
- }
-
- // ------------------------------------------------------------------------
- // Parametrization lets the tests run in cluster and collection mode
- // ------------------------------------------------------------------------
-
- @Parameterized.Parameters(name = "Execution mode = {0}")
- public static Collection<Object[]> executionModes() {
- return Arrays.asList(
- new Object[] { TestExecutionMode.CLUSTER },
- new Object[] { TestExecutionMode.COLLECTION });
- }
-}
[03/15] flink git commit: [FLINK-3995] [build] Properly structure
test scopes and dependencies
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/TestBaseUtils.java
----------------------------------------------------------------------
diff --git a/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/TestBaseUtils.java b/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/TestBaseUtils.java
new file mode 100644
index 0000000..6acc653
--- /dev/null
+++ b/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/TestBaseUtils.java
@@ -0,0 +1,673 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.test.util;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import akka.actor.ActorRef;
+import akka.dispatch.Futures;
+import akka.pattern.Patterns;
+import akka.util.Timeout;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+
+import org.apache.flink.api.java.tuple.Tuple;
+import org.apache.flink.configuration.ConfigConstants;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.runtime.testingUtils.TestingTaskManagerMessages;
+import org.apache.flink.util.TestLogger;
+
+import org.apache.hadoop.fs.FileSystem;
+
+import org.junit.Assert;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import scala.concurrent.Await;
+import scala.concurrent.ExecutionContext;
+import scala.concurrent.ExecutionContext$;
+import scala.concurrent.Future;
+import scala.concurrent.duration.FiniteDuration;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileReader;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.io.InputStream;
+import java.lang.reflect.Field;
+import java.net.HttpURLConnection;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class TestBaseUtils extends TestLogger {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TestBaseUtils.class);
+
+ protected static final int MINIMUM_HEAP_SIZE_MB = 192;
+
+ protected static final long TASK_MANAGER_MEMORY_SIZE = 80;
+
+ protected static final long DEFAULT_AKKA_ASK_TIMEOUT = 1000;
+
+ protected static final String DEFAULT_AKKA_STARTUP_TIMEOUT = "60 s";
+
+ protected static FiniteDuration DEFAULT_TIMEOUT = new FiniteDuration(DEFAULT_AKKA_ASK_TIMEOUT, TimeUnit.SECONDS);
+
+ // ------------------------------------------------------------------------
+
+ protected static File logDir;
+
+ protected TestBaseUtils(){
+ verifyJvmOptions();
+ }
+
+ private static void verifyJvmOptions() {
+ long heap = Runtime.getRuntime().maxMemory() >> 20;
+ Assert.assertTrue("Insufficient java heap space " + heap + "mb - set JVM option: -Xmx" + MINIMUM_HEAP_SIZE_MB
+ + "m", heap > MINIMUM_HEAP_SIZE_MB - 50);
+ }
+
+
+ public static ForkableFlinkMiniCluster startCluster(
+ int numTaskManagers,
+ int taskManagerNumSlots,
+ boolean startWebserver,
+ boolean startZooKeeper,
+ boolean singleActorSystem) throws Exception {
+
+ Configuration config = new Configuration();
+
+ config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, numTaskManagers);
+ config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, taskManagerNumSlots);
+
+ config.setBoolean(ConfigConstants.LOCAL_START_WEBSERVER, startWebserver);
+
+ if (startZooKeeper) {
+ config.setInteger(ConfigConstants.LOCAL_NUMBER_JOB_MANAGER, 3);
+ config.setString(ConfigConstants.RECOVERY_MODE, "zookeeper");
+ }
+
+ return startCluster(config, singleActorSystem);
+ }
+
+ public static ForkableFlinkMiniCluster startCluster(
+ Configuration config,
+ boolean singleActorSystem) throws Exception {
+
+ logDir = File.createTempFile("TestBaseUtils-logdir", null);
+ Assert.assertTrue("Unable to delete temp file", logDir.delete());
+ Assert.assertTrue("Unable to create temp directory", logDir.mkdir());
+ Path logFile = Files.createFile(new File(logDir, "jobmanager.log").toPath());
+ Files.createFile(new File(logDir, "jobmanager.out").toPath());
+
+ config.setLong(ConfigConstants.TASK_MANAGER_MEMORY_SIZE_KEY, TASK_MANAGER_MEMORY_SIZE);
+ config.setBoolean(ConfigConstants.FILESYSTEM_DEFAULT_OVERWRITE_KEY, true);
+
+ config.setString(ConfigConstants.AKKA_ASK_TIMEOUT, DEFAULT_AKKA_ASK_TIMEOUT + "s");
+ config.setString(ConfigConstants.AKKA_STARTUP_TIMEOUT, DEFAULT_AKKA_STARTUP_TIMEOUT);
+
+ config.setInteger(ConfigConstants.JOB_MANAGER_WEB_PORT_KEY, 8081);
+ config.setString(ConfigConstants.JOB_MANAGER_WEB_LOG_PATH_KEY, logFile.toString());
+
+ config.setString(ConfigConstants.TASK_MANAGER_LOG_PATH_KEY, logFile.toString());
+
+ ForkableFlinkMiniCluster cluster = new ForkableFlinkMiniCluster(config, singleActorSystem);
+
+ cluster.start();
+
+ return cluster;
+ }
+
+
+ public static void stopCluster(ForkableFlinkMiniCluster executor, FiniteDuration timeout) throws Exception {
+ if (logDir != null) {
+ FileUtils.deleteDirectory(logDir);
+ }
+ if (executor != null) {
+ int numUnreleasedBCVars = 0;
+ int numActiveConnections = 0;
+
+ if (executor.running()) {
+ List<ActorRef> tms = executor.getTaskManagersAsJava();
+ List<Future<Object>> bcVariableManagerResponseFutures = new ArrayList<>();
+ List<Future<Object>> numActiveConnectionsResponseFutures = new ArrayList<>();
+
+ for (ActorRef tm : tms) {
+ bcVariableManagerResponseFutures.add(Patterns.ask(tm, TestingTaskManagerMessages
+ .RequestBroadcastVariablesWithReferences$.MODULE$, new Timeout(timeout)));
+
+ numActiveConnectionsResponseFutures.add(Patterns.ask(tm, TestingTaskManagerMessages
+ .RequestNumActiveConnections$.MODULE$, new Timeout(timeout)));
+ }
+
+ Future<Iterable<Object>> bcVariableManagerFutureResponses = Futures.sequence(
+ bcVariableManagerResponseFutures, defaultExecutionContext());
+
+ Iterable<Object> responses = Await.result(bcVariableManagerFutureResponses, timeout);
+
+ for (Object response : responses) {
+ numUnreleasedBCVars += ((TestingTaskManagerMessages
+ .ResponseBroadcastVariablesWithReferences) response).number();
+ }
+
+ Future<Iterable<Object>> numActiveConnectionsFutureResponses = Futures.sequence(
+ numActiveConnectionsResponseFutures, defaultExecutionContext());
+
+ responses = Await.result(numActiveConnectionsFutureResponses, timeout);
+
+ for (Object response : responses) {
+ numActiveConnections += ((TestingTaskManagerMessages
+ .ResponseNumActiveConnections) response).number();
+ }
+ }
+
+ executor.stop();
+ FileSystem.closeAll();
+ System.gc();
+
+ Assert.assertEquals("Not all broadcast variables were released.", 0, numUnreleasedBCVars);
+ Assert.assertEquals("Not all TCP connections were released.", 0, numActiveConnections);
+ }
+
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // Result Checking
+ // --------------------------------------------------------------------------------------------
+
+ public static BufferedReader[] getResultReader(String resultPath) throws IOException {
+ return getResultReader(resultPath, new String[]{}, false);
+ }
+
+ public static BufferedReader[] getResultReader(String resultPath, String[] excludePrefixes,
+ boolean inOrderOfFiles) throws IOException {
+ File[] files = getAllInvolvedFiles(resultPath, excludePrefixes);
+
+ if (inOrderOfFiles) {
+ // sort the files after their name (1, 2, 3, 4)...
+ // we cannot sort by path, because strings sort by prefix
+ Arrays.sort(files, new Comparator<File>() {
+
+ @Override
+ public int compare(File o1, File o2) {
+ try {
+ int f1 = Integer.parseInt(o1.getName());
+ int f2 = Integer.parseInt(o2.getName());
+ return f1 < f2 ? -1 : (f1 > f2 ? 1 : 0);
+ }
+ catch (NumberFormatException e) {
+ throw new RuntimeException("The file names are no numbers and cannot be ordered: " +
+ o1.getName() + "/" + o2.getName());
+ }
+ }
+ });
+ }
+
+ BufferedReader[] readers = new BufferedReader[files.length];
+ for (int i = 0; i < files.length; i++) {
+ readers[i] = new BufferedReader(new FileReader(files[i]));
+ }
+ return readers;
+ }
+
+ public static BufferedInputStream[] getResultInputStream(String resultPath) throws IOException {
+ return getResultInputStream(resultPath, new String[]{});
+ }
+
+ public static BufferedInputStream[] getResultInputStream(String resultPath, String[]
+ excludePrefixes) throws IOException {
+ File[] files = getAllInvolvedFiles(resultPath, excludePrefixes);
+ BufferedInputStream[] inStreams = new BufferedInputStream[files.length];
+ for (int i = 0; i < files.length; i++) {
+ inStreams[i] = new BufferedInputStream(new FileInputStream(files[i]));
+ }
+ return inStreams;
+ }
+
+ public static void readAllResultLines(List<String> target, String resultPath) throws IOException {
+ readAllResultLines(target, resultPath, new String[]{});
+ }
+
+ public static void readAllResultLines(List<String> target, String resultPath, String[] excludePrefixes)
+ throws IOException {
+
+ readAllResultLines(target, resultPath, excludePrefixes, false);
+ }
+
+ public static void readAllResultLines(List<String> target, String resultPath,
+ String[] excludePrefixes, boolean inOrderOfFiles) throws IOException {
+
+ final BufferedReader[] readers = getResultReader(resultPath, excludePrefixes, inOrderOfFiles);
+ try {
+ for (BufferedReader reader : readers) {
+ String s;
+ while ((s = reader.readLine()) != null) {
+ target.add(s);
+ }
+ }
+ }
+ finally {
+ for (BufferedReader reader : readers) {
+ try {
+ reader.close();
+ }
+ catch (Exception e) {
+ // ignore, this is best-effort cleanup
+ }
+ }
+ }
+ }
+
+ public static void compareResultsByLinesInMemory(String expectedResultStr, String resultPath) throws Exception {
+ compareResultsByLinesInMemory(expectedResultStr, resultPath, new String[0]);
+ }
+
+ public static void compareResultsByLinesInMemory(String expectedResultStr, String resultPath,
+ String[] excludePrefixes) throws Exception {
+ ArrayList<String> list = new ArrayList<>();
+ readAllResultLines(list, resultPath, excludePrefixes, false);
+
+ String[] result = list.toArray(new String[list.size()]);
+ Arrays.sort(result);
+
+ String[] expected = expectedResultStr.isEmpty() ? new String[0] : expectedResultStr.split("\n");
+ Arrays.sort(expected);
+
+ Assert.assertEquals("Different number of lines in expected and obtained result.", expected.length, result.length);
+ Assert.assertArrayEquals(expected, result);
+ }
+
+ public static void compareResultsByLinesInMemoryWithStrictOrder(String expectedResultStr,
+ String resultPath) throws Exception {
+ compareResultsByLinesInMemoryWithStrictOrder(expectedResultStr, resultPath, new String[]{});
+ }
+
+ public static void compareResultsByLinesInMemoryWithStrictOrder(String expectedResultStr,
+ String resultPath, String[] excludePrefixes) throws Exception {
+ ArrayList<String> list = new ArrayList<>();
+ readAllResultLines(list, resultPath, excludePrefixes, true);
+
+ String[] result = list.toArray(new String[list.size()]);
+
+ String[] expected = expectedResultStr.split("\n");
+
+ Assert.assertEquals("Different number of lines in expected and obtained result.", expected.length, result.length);
+ Assert.assertArrayEquals(expected, result);
+ }
+
+ public static void checkLinesAgainstRegexp(String resultPath, String regexp){
+ Pattern pattern = Pattern.compile(regexp);
+ Matcher matcher = pattern.matcher("");
+
+ ArrayList<String> list = new ArrayList<>();
+ try {
+ readAllResultLines(list, resultPath, new String[]{}, false);
+ } catch (IOException e1) {
+ Assert.fail("Error reading the result");
+ }
+
+ for (String line : list){
+ matcher.reset(line);
+ if (!matcher.find()){
+ String msg = "Line is not well-formed: " + line;
+ Assert.fail(msg);
+ }
+ }
+ }
+
+ public static void compareKeyValuePairsWithDelta(String expectedLines, String resultPath,
+ String delimiter, double maxDelta) throws Exception {
+ compareKeyValuePairsWithDelta(expectedLines, resultPath, new String[]{}, delimiter, maxDelta);
+ }
+
+ public static void compareKeyValuePairsWithDelta(String expectedLines, String resultPath,
+ String[] excludePrefixes, String delimiter, double maxDelta) throws Exception {
+ ArrayList<String> list = new ArrayList<>();
+ readAllResultLines(list, resultPath, excludePrefixes, false);
+
+ String[] result = list.toArray(new String[list.size()]);
+ String[] expected = expectedLines.isEmpty() ? new String[0] : expectedLines.split("\n");
+
+ Assert.assertEquals("Wrong number of result lines.", expected.length, result.length);
+
+ Arrays.sort(result);
+ Arrays.sort(expected);
+
+ for (int i = 0; i < expected.length; i++) {
+ String[] expectedFields = expected[i].split(delimiter);
+ String[] resultFields = result[i].split(delimiter);
+
+ double expectedPayLoad = Double.parseDouble(expectedFields[1]);
+ double resultPayLoad = Double.parseDouble(resultFields[1]);
+
+ Assert.assertTrue("Values differ by more than the permissible delta", Math.abs(expectedPayLoad - resultPayLoad) < maxDelta);
+ }
+ }
+
+ public static <X> void compareResultCollections(List<X> expected, List<X> actual,
+ Comparator<X> comparator) {
+ Assert.assertEquals(expected.size(), actual.size());
+
+ Collections.sort(expected, comparator);
+ Collections.sort(actual, comparator);
+
+ for (int i = 0; i < expected.size(); i++) {
+ Assert.assertEquals(expected.get(i), actual.get(i));
+ }
+ }
+
+ private static File[] getAllInvolvedFiles(String resultPath, String[] excludePrefixes) {
+ final String[] exPrefs = excludePrefixes;
+ File result = asFile(resultPath);
+ if (!result.exists()) {
+ Assert.fail("Result file was not written");
+ }
+ if (result.isDirectory()) {
+ return result.listFiles(new FilenameFilter() {
+
+ @Override
+ public boolean accept(File dir, String name) {
+ for(String p: exPrefs) {
+ if(name.startsWith(p)) {
+ return false;
+ }
+ }
+ return true;
+ }
+ });
+ } else {
+ return new File[] { result };
+ }
+ }
+
+ protected static File asFile(String path) {
+ try {
+ URI uri = new URI(path);
+ if (uri.getScheme().equals("file")) {
+ return new File(uri.getPath());
+ } else {
+ throw new IllegalArgumentException("This path does not denote a local file.");
+ }
+ } catch (URISyntaxException | NullPointerException e) {
+ throw new IllegalArgumentException("This path does not describe a valid local file URI.");
+ }
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // Comparison methods for tests using collect()
+ // --------------------------------------------------------------------------------------------
+
+ public static <T> void compareResultAsTuples(List<T> result, String expected) {
+ compareResult(result, expected, true, true);
+ }
+
+ public static <T> void compareResultAsText(List<T> result, String expected) {
+ compareResult(result, expected,
+ false, true);
+ }
+
+ public static <T> void compareOrderedResultAsText(List<T> result, String expected) {
+ compareResult(result, expected, false, false);
+ }
+
+ public static <T> void compareOrderedResultAsText(List<T> result, String expected, boolean asTuples) {
+ compareResult(result, expected, asTuples, false);
+ }
+
+ private static <T> void compareResult(List<T> result, String expected, boolean asTuples, boolean sort) {
+ String[] expectedStrings = expected.split("\n");
+ String[] resultStrings = new String[result.size()];
+
+ for (int i = 0; i < resultStrings.length; i++) {
+ T val = result.get(i);
+
+ if (asTuples) {
+ if (val instanceof Tuple) {
+ Tuple t = (Tuple) val;
+ Object first = t.getField(0);
+ StringBuilder bld = new StringBuilder(first == null ? "null" : first.toString());
+ for (int pos = 1; pos < t.getArity(); pos++) {
+ Object next = t.getField(pos);
+ bld.append(',').append(next == null ? "null" : next.toString());
+ }
+ resultStrings[i] = bld.toString();
+ }
+ else {
+ throw new IllegalArgumentException(val + " is no tuple");
+ }
+ }
+ else {
+ resultStrings[i] = (val == null) ? "null" : val.toString();
+ }
+ }
+
+ assertEquals("Wrong number of elements result", expectedStrings.length, resultStrings.length);
+
+ if (sort) {
+ Arrays.sort(expectedStrings);
+ Arrays.sort(resultStrings);
+ }
+
+ for (int i = 0; i < expectedStrings.length; i++) {
+ assertEquals(expectedStrings[i], resultStrings[i]);
+ }
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // Comparison methods for tests using sample
+ // --------------------------------------------------------------------------------------------
+
+ /**
+ * The expected string contains all expected results separate with line break, check whether all elements in result
+ * are contained in the expected string.
+ * @param result The test result.
+ * @param expected The expected string value combination.
+ * @param <T> The result type.
+ */
+ public static <T> void containsResultAsText(List<T> result, String expected) {
+ String[] expectedStrings = expected.split("\n");
+ List<String> resultStrings = new ArrayList<>();
+
+ for (T val : result) {
+ String str = (val == null) ? "null" : val.toString();
+ resultStrings.add(str);
+ }
+
+ List<String> expectedStringList = Arrays.asList(expectedStrings);
+
+ for (String element : resultStrings) {
+ assertTrue(expectedStringList.contains(element));
+ }
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // Miscellaneous helper methods
+ // --------------------------------------------------------------------------------------------
+
+ protected static Collection<Object[]> toParameterList(Configuration ... testConfigs) {
+ ArrayList<Object[]> configs = new ArrayList<>();
+ for (Configuration testConfig : testConfigs) {
+ Object[] c = { testConfig };
+ configs.add(c);
+ }
+ return configs;
+ }
+
+ protected static Collection<Object[]> toParameterList(List<Configuration> testConfigs) {
+ LinkedList<Object[]> configs = new LinkedList<>();
+ for (Configuration testConfig : testConfigs) {
+ Object[] c = { testConfig };
+ configs.add(c);
+ }
+ return configs;
+ }
+
+ // This code is taken from: http://stackoverflow.com/a/7201825/568695
+ // it changes the environment variables of this JVM. Use only for testing purposes!
+ @SuppressWarnings("unchecked")
+ public static void setEnv(Map<String, String> newenv) {
+ try {
+ Class<?> processEnvironmentClass = Class.forName("java.lang.ProcessEnvironment");
+ Field theEnvironmentField = processEnvironmentClass.getDeclaredField("theEnvironment");
+ theEnvironmentField.setAccessible(true);
+ Map<String, String> env = (Map<String, String>) theEnvironmentField.get(null);
+ env.putAll(newenv);
+ Field theCaseInsensitiveEnvironmentField = processEnvironmentClass.getDeclaredField("theCaseInsensitiveEnvironment");
+ theCaseInsensitiveEnvironmentField.setAccessible(true);
+ Map<String, String> cienv = (Map<String, String>) theCaseInsensitiveEnvironmentField.get(null);
+ cienv.putAll(newenv);
+ } catch (NoSuchFieldException e) {
+ try {
+ Class<?>[] classes = Collections.class.getDeclaredClasses();
+ Map<String, String> env = System.getenv();
+ for (Class<?> cl : classes) {
+ if ("java.util.Collections$UnmodifiableMap".equals(cl.getName())) {
+ Field field = cl.getDeclaredField("m");
+ field.setAccessible(true);
+ Object obj = field.get(env);
+ Map<String, String> map = (Map<String, String>) obj;
+ map.clear();
+ map.putAll(newenv);
+ }
+ }
+ } catch (Exception e2) {
+ throw new RuntimeException(e2);
+ }
+ } catch (Exception e1) {
+ throw new RuntimeException(e1);
+ }
+ }
+
+ private static ExecutionContext defaultExecutionContext() {
+ return ExecutionContext$.MODULE$.global();
+ }
+ // --------------------------------------------------------------------------------------------
+ // File helper methods
+ // --------------------------------------------------------------------------------------------
+
+ protected static void deleteRecursively(File f) throws IOException {
+ if (f.isDirectory()) {
+ FileUtils.deleteDirectory(f);
+ } else if (!f.delete()) {
+ System.err.println("Failed to delete file " + f.getAbsolutePath());
+ }
+ }
+
+ public static String constructTestPath(Class<?> forClass, String folder) {
+ // we create test path that depends on class to prevent name clashes when two tests
+ // create temp files with the same name
+ String path = System.getProperty("java.io.tmpdir");
+ if (!(path.endsWith("/") || path.endsWith("\\")) ) {
+ path += System.getProperty("file.separator");
+ }
+ path += (forClass.getName() + "-" + folder);
+ return path;
+ }
+
+ public static String constructTestURI(Class<?> forClass, String folder) {
+ return new File(constructTestPath(forClass, folder)).toURI().toString();
+ }
+
+ //---------------------------------------------------------------------------------------------
+ // Web utils
+ //---------------------------------------------------------------------------------------------
+
+ public static String getFromHTTP(String url) throws Exception {
+ URL u = new URL(url);
+ LOG.info("Accessing URL "+url+" as URL: "+u);
+ HttpURLConnection connection = (HttpURLConnection) u.openConnection();
+ connection.setConnectTimeout(100000);
+ connection.connect();
+ InputStream is;
+ if(connection.getResponseCode() >= 400) {
+ // error!
+ LOG.warn("HTTP Response code when connecting to {} was {}", url, connection.getResponseCode());
+ is = connection.getErrorStream();
+ } else {
+ is = connection.getInputStream();
+ }
+
+ return IOUtils.toString(is, connection.getContentEncoding() != null ? connection.getContentEncoding() : "UTF-8");
+ }
+
+ public static class TupleComparator<T extends Tuple> implements Comparator<T> {
+
+ @Override
+ public int compare(T o1, T o2) {
+ if (o1 == null || o2 == null) {
+ throw new IllegalArgumentException("Cannot compare null tuples");
+ }
+ else if (o1.getArity() != o2.getArity()) {
+ return o1.getArity() - o2.getArity();
+ }
+ else {
+ for (int i = 0; i < o1.getArity(); i++) {
+ Object val1 = o1.getField(i);
+ Object val2 = o2.getField(i);
+
+ int cmp;
+ if (val1 != null && val2 != null) {
+ cmp = compareValues(val1, val2);
+ }
+ else {
+ cmp = val1 == null ? (val2 == null ? 0 : -1) : 1;
+ }
+
+ if (cmp != 0) {
+ return cmp;
+ }
+ }
+
+ return 0;
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ private static <X extends Comparable<X>> int compareValues(Object o1, Object o2) {
+ if (o1 instanceof Comparable && o2 instanceof Comparable) {
+ X c1 = (X) o1;
+ X c2 = (X) o2;
+ return c1.compareTo(c2);
+ }
+ else {
+ throw new IllegalArgumentException("Cannot compare tuples with non comparable elements");
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/TestEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/TestEnvironment.java b/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/TestEnvironment.java
new file mode 100644
index 0000000..7cb88be
--- /dev/null
+++ b/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/TestEnvironment.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.test.util;
+
+import org.apache.flink.api.common.JobExecutionResult;
+import org.apache.flink.api.common.Plan;
+import org.apache.flink.api.common.CodeAnalysisMode;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.ExecutionEnvironmentFactory;
+import org.apache.flink.optimizer.DataStatistics;
+import org.apache.flink.optimizer.Optimizer;
+import org.apache.flink.optimizer.plan.OptimizedPlan;
+import org.apache.flink.optimizer.plandump.PlanJSONDumpGenerator;
+import org.apache.flink.optimizer.plantranslate.JobGraphGenerator;
+import org.apache.flink.runtime.jobgraph.JobGraph;
+
+public class TestEnvironment extends ExecutionEnvironment {
+
+ private final ForkableFlinkMiniCluster executor;
+
+ private TestEnvironment lastEnv = null;
+
+ @Override
+ public JobExecutionResult getLastJobExecutionResult() {
+ if (lastEnv == null) {
+ return this.lastJobExecutionResult;
+ }
+ else {
+ return lastEnv.getLastJobExecutionResult();
+ }
+ }
+
+ public TestEnvironment(ForkableFlinkMiniCluster executor, int parallelism) {
+ this.executor = executor;
+ setParallelism(parallelism);
+
+ // disabled to improve build time
+ getConfig().setCodeAnalysisMode(CodeAnalysisMode.DISABLE);
+ }
+
+ public TestEnvironment(ForkableFlinkMiniCluster executor, int parallelism, boolean isObjectReuseEnabled) {
+ this(executor, parallelism);
+
+ if (isObjectReuseEnabled) {
+ getConfig().enableObjectReuse();
+ } else {
+ getConfig().disableObjectReuse();
+ }
+ }
+
+ @Override
+ public void startNewSession() throws Exception {
+ }
+
+ @Override
+ public JobExecutionResult execute(String jobName) throws Exception {
+ OptimizedPlan op = compileProgram(jobName);
+
+ JobGraphGenerator jgg = new JobGraphGenerator();
+ JobGraph jobGraph = jgg.compileJobGraph(op);
+
+ this.lastJobExecutionResult = executor.submitJobAndWait(jobGraph, false);
+ return this.lastJobExecutionResult;
+ }
+
+
+ @Override
+ public String getExecutionPlan() throws Exception {
+ OptimizedPlan op = compileProgram("unused");
+
+ PlanJSONDumpGenerator jsonGen = new PlanJSONDumpGenerator();
+ return jsonGen.getOptimizerPlanAsJSON(op);
+ }
+
+
+ private OptimizedPlan compileProgram(String jobName) {
+ Plan p = createProgramPlan(jobName);
+
+ Optimizer pc = new Optimizer(new DataStatistics(), this.executor.configuration());
+ return pc.compile(p);
+ }
+
+ public void setAsContext() {
+ ExecutionEnvironmentFactory factory = new ExecutionEnvironmentFactory() {
+ @Override
+ public ExecutionEnvironment createExecutionEnvironment() {
+ lastEnv = new TestEnvironment(executor, getParallelism(), getConfig().isObjectReuseEnabled());
+ return lastEnv;
+ }
+ };
+
+ initializeContextEnvironment(factory);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-test-utils-parent/flink-test-utils/src/main/scala/org/apache/flink/test/util/ForkableFlinkMiniCluster.scala
----------------------------------------------------------------------
diff --git a/flink-test-utils-parent/flink-test-utils/src/main/scala/org/apache/flink/test/util/ForkableFlinkMiniCluster.scala b/flink-test-utils-parent/flink-test-utils/src/main/scala/org/apache/flink/test/util/ForkableFlinkMiniCluster.scala
new file mode 100644
index 0000000..79c5a25
--- /dev/null
+++ b/flink-test-utils-parent/flink-test-utils/src/main/scala/org/apache/flink/test/util/ForkableFlinkMiniCluster.scala
@@ -0,0 +1,308 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.test.util
+
+import java.util.concurrent.TimeoutException
+
+import akka.actor.{ActorRef, ActorSystem}
+import akka.pattern.Patterns._
+import akka.pattern.ask
+
+import org.apache.curator.test.TestingCluster
+import org.apache.flink.configuration.{ConfigConstants, Configuration}
+import org.apache.flink.runtime.akka.AkkaUtils
+import org.apache.flink.runtime.clusterframework.FlinkResourceManager
+import org.apache.flink.runtime.clusterframework.types.ResourceID
+import org.apache.flink.runtime.jobmanager.{JobManager, RecoveryMode}
+import org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster
+import org.apache.flink.runtime.taskmanager.TaskManager
+import org.apache.flink.runtime.testingUtils.TestingTaskManagerMessages.NotifyWhenRegisteredAtJobManager
+import org.apache.flink.runtime.testingUtils.{TestingJobManager, TestingMemoryArchivist, TestingTaskManager}
+import org.apache.flink.runtime.testutils.TestingResourceManager
+
+import scala.concurrent.{Await, Future}
+import scala.concurrent.duration._
+
+/**
+ * A forkable mini cluster is a special case of the mini cluster, used for parallel test execution
+ * on build servers. If multiple tests run in parallel, the cluster picks up the fork number and
+ * uses it to avoid port conflicts.
+ *
+ * @param userConfiguration Configuration object with the user provided configuration values
+ * @param singleActorSystem true, if all actors (JobManager and TaskManager) shall be run in the
+ * same [[ActorSystem]], otherwise false.
+ */
+class ForkableFlinkMiniCluster(
+ userConfiguration: Configuration,
+ singleActorSystem: Boolean)
+ extends LocalFlinkMiniCluster(userConfiguration, singleActorSystem) {
+
+ def this(userConfiguration: Configuration) = this(userConfiguration, true)
+
+ // --------------------------------------------------------------------------
+
+ var zookeeperCluster: Option[TestingCluster] = None
+
+ override def generateConfiguration(userConfiguration: Configuration): Configuration = {
+ val forkNumberString = System.getProperty("forkNumber")
+
+ val forkNumber = try {
+ Integer.parseInt(forkNumberString)
+ }
+ catch {
+ case e: NumberFormatException => -1
+ }
+
+ val config = userConfiguration.clone()
+
+ if (forkNumber != -1) {
+ val jobManagerRPC = 1024 + forkNumber*400
+ val taskManagerRPC = 1024 + forkNumber*400 + 100
+ val taskManagerData = 1024 + forkNumber*400 + 200
+ val resourceManagerRPC = 1024 + forkNumber*400 + 300
+
+ config.setInteger(ConfigConstants.JOB_MANAGER_IPC_PORT_KEY, jobManagerRPC)
+ config.setInteger(ConfigConstants.TASK_MANAGER_IPC_PORT_KEY, taskManagerRPC)
+ config.setInteger(ConfigConstants.TASK_MANAGER_DATA_PORT_KEY, taskManagerData)
+ config.setInteger(ConfigConstants.RESOURCE_MANAGER_IPC_PORT_KEY, resourceManagerRPC)
+ }
+
+ super.generateConfiguration(config)
+ }
+
+ override def startJobManager(index: Int, actorSystem: ActorSystem): ActorRef = {
+ val config = configuration.clone()
+
+ val jobManagerName = getJobManagerName(index)
+ val archiveName = getArchiveName(index)
+
+ val jobManagerPort = config.getInteger(
+ ConfigConstants.JOB_MANAGER_IPC_PORT_KEY,
+ ConfigConstants.DEFAULT_JOB_MANAGER_IPC_PORT)
+
+ if (jobManagerPort > 0) {
+ config.setInteger(ConfigConstants.JOB_MANAGER_IPC_PORT_KEY, jobManagerPort + index)
+ }
+
+ val (jobManager, _) = JobManager.startJobManagerActors(
+ config,
+ actorSystem,
+ Some(jobManagerName),
+ Some(archiveName),
+ classOf[TestingJobManager],
+ classOf[TestingMemoryArchivist])
+
+ jobManager
+ }
+
+ override def startResourceManager(index: Int, system: ActorSystem): ActorRef = {
+ val config = configuration.clone()
+
+ val resourceManagerName = getResourceManagerName(index)
+
+ val resourceManagerPort = config.getInteger(
+ ConfigConstants.RESOURCE_MANAGER_IPC_PORT_KEY,
+ ConfigConstants.DEFAULT_RESOURCE_MANAGER_IPC_PORT)
+
+ if (resourceManagerPort > 0) {
+ config.setInteger(ConfigConstants.RESOURCE_MANAGER_IPC_PORT_KEY, resourceManagerPort + index)
+ }
+
+ val resourceManager = FlinkResourceManager.startResourceManagerActors(
+ config,
+ system,
+ createLeaderRetrievalService(),
+ classOf[TestingResourceManager],
+ resourceManagerName)
+
+ resourceManager
+ }
+
+ override def startTaskManager(index: Int, system: ActorSystem): ActorRef = {
+ val config = configuration.clone()
+
+ val rpcPort = config.getInteger(
+ ConfigConstants.TASK_MANAGER_IPC_PORT_KEY,
+ ConfigConstants.DEFAULT_TASK_MANAGER_IPC_PORT)
+
+ val dataPort = config.getInteger(
+ ConfigConstants.TASK_MANAGER_DATA_PORT_KEY,
+ ConfigConstants.DEFAULT_TASK_MANAGER_DATA_PORT)
+
+ if (rpcPort > 0) {
+ config.setInteger(ConfigConstants.TASK_MANAGER_IPC_PORT_KEY, rpcPort + index)
+ }
+ if (dataPort > 0) {
+ config.setInteger(ConfigConstants.TASK_MANAGER_DATA_PORT_KEY, dataPort + index)
+ }
+
+ val localExecution = numTaskManagers == 1
+
+ TaskManager.startTaskManagerComponentsAndActor(
+ config,
+ ResourceID.generate(),
+ system,
+ hostname,
+ Some(TaskManager.TASK_MANAGER_NAME + index),
+ Some(createLeaderRetrievalService()),
+ localExecution,
+ classOf[TestingTaskManager])
+ }
+
+ def restartLeadingJobManager(): Unit = {
+ this.synchronized {
+ (jobManagerActorSystems, jobManagerActors) match {
+ case (Some(jmActorSystems), Some(jmActors)) =>
+ val leader = getLeaderGateway(AkkaUtils.getTimeout(configuration))
+ val index = getLeaderIndex(AkkaUtils.getTimeout(configuration))
+
+ clearLeader()
+
+ val stopped = gracefulStop(leader.actor(), ForkableFlinkMiniCluster.MAX_RESTART_DURATION)
+ Await.result(stopped, ForkableFlinkMiniCluster.MAX_RESTART_DURATION)
+
+ if(!singleActorSystem) {
+ jmActorSystems(index).shutdown()
+ jmActorSystems(index).awaitTermination()
+ }
+
+ val newJobManagerActorSystem = if(!singleActorSystem) {
+ startJobManagerActorSystem(index)
+ } else {
+ jmActorSystems.head
+ }
+
+ val newJobManagerActor = startJobManager(index, newJobManagerActorSystem)
+
+ jobManagerActors = Some(jmActors.patch(index, Seq(newJobManagerActor), 1))
+ jobManagerActorSystems = Some(jmActorSystems.patch(
+ index,
+ Seq(newJobManagerActorSystem),
+ 1))
+
+ val lrs = createLeaderRetrievalService()
+
+ jobManagerLeaderRetrievalService = Some(lrs)
+ lrs.start(this)
+
+ case _ => throw new Exception("The JobManager of the ForkableFlinkMiniCluster have not " +
+ "been started properly.")
+ }
+ }
+ }
+
+
+ def restartTaskManager(index: Int): Unit = {
+ (taskManagerActorSystems, taskManagerActors) match {
+ case (Some(tmActorSystems), Some(tmActors)) =>
+ val stopped = gracefulStop(tmActors(index), ForkableFlinkMiniCluster.MAX_RESTART_DURATION)
+ Await.result(stopped, ForkableFlinkMiniCluster.MAX_RESTART_DURATION)
+
+ if(!singleActorSystem) {
+ tmActorSystems(index).shutdown()
+ tmActorSystems(index).awaitTermination()
+ }
+
+ val taskManagerActorSystem = if(!singleActorSystem) {
+ startTaskManagerActorSystem(index)
+ } else {
+ tmActorSystems.head
+ }
+
+ val taskManagerActor = startTaskManager(index, taskManagerActorSystem)
+
+ taskManagerActors = Some(tmActors.patch(index, Seq(taskManagerActor), 1))
+ taskManagerActorSystems = Some(tmActorSystems.patch(index, Seq(taskManagerActorSystem), 1))
+
+ case _ => throw new Exception("The TaskManager of the ForkableFlinkMiniCluster have not " +
+ "been started properly.")
+ }
+ }
+
+ override def start(): Unit = {
+ val zookeeperURL = configuration.getString(ConfigConstants.ZOOKEEPER_QUORUM_KEY, "")
+
+ zookeeperCluster = if(recoveryMode == RecoveryMode.ZOOKEEPER && zookeeperURL.equals("")) {
+ LOG.info("Starting ZooKeeper cluster.")
+
+ val testingCluster = new TestingCluster(1)
+
+ configuration.setString(ConfigConstants.ZOOKEEPER_QUORUM_KEY, testingCluster.getConnectString)
+
+ testingCluster.start()
+
+ Some(testingCluster)
+ } else {
+ None
+ }
+
+ super.start()
+ }
+
+ override def stop(): Unit = {
+ super.stop()
+
+ zookeeperCluster.foreach{
+ LOG.info("Stopping ZooKeeper cluster.")
+ _.close()
+ }
+ }
+
+ def waitForTaskManagersToBeRegisteredAtJobManager(jobManager: ActorRef): Unit = {
+ val futures = taskManagerActors.map {
+ _.map {
+ tm => (tm ? NotifyWhenRegisteredAtJobManager(jobManager))(timeout)
+ }
+ }.getOrElse(Seq())
+
+ try {
+ Await.ready(Future.sequence(futures), timeout)
+ } catch {
+ case t: TimeoutException =>
+ throw new Exception("Timeout while waiting for TaskManagers to register at " +
+ s"${jobManager.path}")
+ }
+
+ }
+}
+
+object ForkableFlinkMiniCluster {
+
+ val MAX_RESTART_DURATION = 2 minute
+
+ val DEFAULT_MINICLUSTER_AKKA_ASK_TIMEOUT = "200 s"
+
+ def startCluster(
+ numSlots: Int,
+ numTaskManagers: Int,
+ timeout: String = DEFAULT_MINICLUSTER_AKKA_ASK_TIMEOUT)
+ : ForkableFlinkMiniCluster = {
+
+ val config = new Configuration()
+ config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, numSlots)
+ config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, numTaskManagers)
+ config.setString(ConfigConstants.AKKA_ASK_TIMEOUT, timeout)
+
+ val cluster = new ForkableFlinkMiniCluster(config)
+
+ cluster.start()
+
+ cluster
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-test-utils-parent/pom.xml
----------------------------------------------------------------------
diff --git a/flink-test-utils-parent/pom.xml b/flink-test-utils-parent/pom.xml
new file mode 100644
index 0000000..03747e8
--- /dev/null
+++ b/flink-test-utils-parent/pom.xml
@@ -0,0 +1,43 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-parent</artifactId>
+ <version>1.1-SNAPSHOT</version>
+ <relativePath>..</relativePath>
+ </parent>
+
+ <artifactId>flink-test-utils-parent</artifactId>
+ <name>flink-test-utils-parent</name>
+
+ <packaging>pom</packaging>
+
+ <modules>
+ <module>flink-test-utils-junit</module>
+ <module>flink-test-utils</module>
+ </modules>
+
+</project>
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-test-utils/pom.xml
----------------------------------------------------------------------
diff --git a/flink-test-utils/pom.xml b/flink-test-utils/pom.xml
deleted file mode 100644
index 1038531..0000000
--- a/flink-test-utils/pom.xml
+++ /dev/null
@@ -1,246 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
-
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-parent</artifactId>
- <version>1.1-SNAPSHOT</version>
- <relativePath>..</relativePath>
- </parent>
-
- <artifactId>flink-test-utils_2.10</artifactId>
- <name>flink-test-utils</name>
-
- <packaging>jar</packaging>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-optimizer_2.10</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-core</artifactId>
- <version>${project.version}</version>
- <type>test-jar</type>
- </dependency>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-runtime_2.10</artifactId>
- <version>${project.version}</version>
- <type>test-jar</type>
- </dependency>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-clients_2.10</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>4.11</version>
- </dependency>
-
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- <version>${guava.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.scalatest</groupId>
- <artifactId>scalatest_${scala.binary.version}</artifactId>
- <scope>compile</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.curator</groupId>
- <artifactId>curator-test</artifactId>
- <version>${curator.version}</version>
- <scope>compile</scope>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <!-- Scala Compiler -->
- <plugin>
- <groupId>net.alchim31.maven</groupId>
- <artifactId>scala-maven-plugin</artifactId>
- <version>3.1.4</version>
- <executions>
- <!-- Run scala compiler in the process-resources phase, so that dependencies
- on scala classes can be resolved later in the (Java) compile phase -->
- <execution>
- <id>scala-compile-first</id>
- <phase>process-resources</phase>
- <goals>
- <goal>compile</goal>
- </goals>
- </execution>
-
- <!-- Run scala compiler in the process-test-resources phase, so that
- dependencies on scala classes can be resolved later in the (Java) test-compile
- phase -->
- <execution>
- <id>scala-test-compile</id>
- <phase>process-test-resources</phase>
- <goals>
- <goal>testCompile</goal>
- </goals>
- </execution>
- </executions>
- <configuration>
- <jvmArgs>
- <jvmArg>-Xms128m</jvmArg>
- <jvmArg>-Xmx512m</jvmArg>
- </jvmArgs>
- </configuration>
- </plugin>
-
- <!-- Eclipse Integration -->
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-eclipse-plugin</artifactId>
- <version>2.8</version>
- <configuration>
- <downloadSources>true</downloadSources>
- <projectnatures>
- <projectnature>org.scala-ide.sdt.core.scalanature</projectnature>
- <projectnature>org.eclipse.jdt.core.javanature</projectnature>
- </projectnatures>
- <buildcommands>
- <buildcommand>org.scala-ide.sdt.core.scalabuilder</buildcommand>
- </buildcommands>
- <classpathContainers>
- <classpathContainer>org.scala-ide.sdt.launching.SCALA_CONTAINER</classpathContainer>
- <classpathContainer>org.eclipse.jdt.launching.JRE_CONTAINER</classpathContainer>
- </classpathContainers>
- <excludes>
- <exclude>org.scala-lang:scala-library</exclude>
- <exclude>org.scala-lang:scala-compiler</exclude>
- </excludes>
- <sourceIncludes>
- <sourceInclude>**/*.scala</sourceInclude>
- <sourceInclude>**/*.java</sourceInclude>
- </sourceIncludes>
- </configuration>
- </plugin>
-
- <!-- Adding scala source directories to build path -->
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <version>1.7</version>
- <executions>
- <!-- Add src/main/scala to eclipse build path -->
- <execution>
- <id>add-source</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>add-source</goal>
- </goals>
- <configuration>
- <sources>
- <source>src/main/scala</source>
- </sources>
- </configuration>
- </execution>
- <!-- Add src/test/scala to eclipse build path -->
- <execution>
- <id>add-test-source</id>
- <phase>generate-test-sources</phase>
- <goals>
- <goal>add-test-source</goal>
- </goals>
- <configuration>
- <sources>
- <source>src/test/scala</source>
- </sources>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- <!-- Scala Code Style, most of the configuration done via plugin management -->
- <plugin>
- <groupId>org.scalastyle</groupId>
- <artifactId>scalastyle-maven-plugin</artifactId>
- <configuration>
- <configLocation>${project.basedir}/../tools/maven/scalastyle-config.xml</configLocation>
- </configuration>
- </plugin>
- <!--Plugin to manage test as test-jar apart from the main source code -->
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <executions>
- <execution>
- <goals>
- <goal>test-jar</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
-
- </plugins>
- <pluginManagement>
- <plugins>
- <!--This plugin's configuration is used to store Eclipse m2e settings only. It has no influence on the Maven build itself.-->
- <plugin>
- <groupId>org.eclipse.m2e</groupId>
- <artifactId>lifecycle-mapping</artifactId>
- <version>1.0.0</version>
- <configuration>
- <lifecycleMappingMetadata>
- <pluginExecutions>
- <pluginExecution>
- <pluginExecutionFilter>
- <groupId>
- net.alchim31.maven
- </groupId>
- <artifactId>
- scala-maven-plugin
- </artifactId>
- <versionRange>
- [3.1.4,)
- </versionRange>
- <goals>
- <goal>compile</goal>
- <goal>testCompile</goal>
- </goals>
- </pluginExecutionFilter>
- <action>
- <ignore></ignore>
- </action>
- </pluginExecution>
- </pluginExecutions>
- </lifecycleMappingMetadata>
- </configuration>
- </plugin>
- </plugins>
- </pluginManagement>
- </build>
-</project>
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-test-utils/src/test/java/org/apache/flink/test/testdata/ConnectedComponentsData.java
----------------------------------------------------------------------
diff --git a/flink-test-utils/src/test/java/org/apache/flink/test/testdata/ConnectedComponentsData.java b/flink-test-utils/src/test/java/org/apache/flink/test/testdata/ConnectedComponentsData.java
deleted file mode 100644
index 60cd319..0000000
--- a/flink-test-utils/src/test/java/org/apache/flink/test/testdata/ConnectedComponentsData.java
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.test.testdata;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.util.List;
-import java.util.Random;
-import java.util.regex.Pattern;
-
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.junit.Assert;
-
-
-public class ConnectedComponentsData {
-
- public static final String getEnumeratingVertices(int num) {
- if (num < 1 || num > 1000000) {
- throw new IllegalArgumentException();
- }
-
- StringBuilder bld = new StringBuilder(3 * num);
- for (int i = 1; i <= num; i++) {
- bld.append(i);
- bld.append('\n');
- }
- return bld.toString();
- }
-
- /**
- * Creates random edges such that even numbered vertices are connected with even numbered vertices
- * and odd numbered vertices only with other odd numbered ones.
- */
- public static final String getRandomOddEvenEdges(int numEdges, int numVertices, long seed) {
- if (numVertices < 2 || numVertices > 1000000 || numEdges < numVertices || numEdges > 1000000) {
- throw new IllegalArgumentException();
- }
-
- StringBuilder bld = new StringBuilder(5 * numEdges);
-
- // first create the linear edge sequence even -> even and odd -> odd to make sure they are
- // all in the same component
- for (int i = 3; i <= numVertices; i++) {
- bld.append(i - 2).append(' ').append(i).append('\n');
- }
-
- numEdges -= numVertices - 2;
- Random r = new Random(seed);
-
- for (int i = 1; i <= numEdges; i++) {
- int evenOdd = r.nextBoolean() ? 1 : 0;
-
- int source = r.nextInt(numVertices) + 1;
- if (source % 2 != evenOdd) {
- source--;
- if (source < 1) {
- source = 2;
- }
- }
-
- int target = r.nextInt(numVertices) + 1;
- if (target % 2 != evenOdd) {
- target--;
- if (target < 1) {
- target = 2;
- }
- }
-
- bld.append(source).append(' ').append(target).append('\n');
- }
- return bld.toString();
- }
-
- public static void checkOddEvenResult(BufferedReader result) throws IOException {
- Pattern split = Pattern.compile(" ");
- String line;
- while ((line = result.readLine()) != null) {
- String[] res = split.split(line);
- Assert.assertEquals("Malformed result: Wrong number of tokens in line.", 2, res.length);
- try {
- int vertex = Integer.parseInt(res[0]);
- int component = Integer.parseInt(res[1]);
-
- int should = vertex % 2;
- if (should == 0) {
- should = 2;
- }
- Assert.assertEquals("Vertex is in wrong component.", should, component);
- } catch (NumberFormatException e) {
- Assert.fail("Malformed result.");
- }
- }
- }
-
- public static void checkOddEvenResult(List<Tuple2<Long, Long>> lines) throws IOException {
- for (Tuple2<Long, Long> line : lines) {
- try {
- long vertex = line.f0;
- long component = line.f1;
- long should = vertex % 2;
- if (should == 0) {
- should = 2;
- }
- Assert.assertEquals("Vertex is in wrong component.", should, component);
- } catch (NumberFormatException e) {
- Assert.fail("Malformed result.");
- }
- }
- }
-
- private ConnectedComponentsData() {}
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-test-utils/src/test/java/org/apache/flink/test/testdata/EnumTriangleData.java
----------------------------------------------------------------------
diff --git a/flink-test-utils/src/test/java/org/apache/flink/test/testdata/EnumTriangleData.java b/flink-test-utils/src/test/java/org/apache/flink/test/testdata/EnumTriangleData.java
deleted file mode 100644
index b170404..0000000
--- a/flink-test-utils/src/test/java/org/apache/flink/test/testdata/EnumTriangleData.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.test.testdata;
-
-public class EnumTriangleData {
-
- public static final String EDGES =
- "1 2\n" +
- "1 3\n" +
- "1 4\n" +
- "1 5\n" +
- "2 3\n" +
- "2 5\n" +
- "3 4\n" +
- "3 7\n" +
- "5 6\n" +
- "3 8\n" +
- "7 8\n";
-
- public static final String TRIANGLES_BY_ID =
- "1,2,3\n" +
- "1,3,4\n" +
- "1,2,5\n" +
- "3,7,8\n";
-
- public static final String TRIANGLES_BY_DEGREE =
- "2,1,3\n" +
- "4,1,3\n" +
- "2,1,5\n" +
- "7,3,8\n";
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-test-utils/src/test/java/org/apache/flink/test/testdata/KMeansData.java
----------------------------------------------------------------------
diff --git a/flink-test-utils/src/test/java/org/apache/flink/test/testdata/KMeansData.java b/flink-test-utils/src/test/java/org/apache/flink/test/testdata/KMeansData.java
deleted file mode 100644
index 1319e84..0000000
--- a/flink-test-utils/src/test/java/org/apache/flink/test/testdata/KMeansData.java
+++ /dev/null
@@ -1,257 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.test.testdata;
-
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.List;
-import java.util.StringTokenizer;
-
-import org.junit.Assert;
-
-
-public class KMeansData {
-
- // --------------------------------------------------------------------------------------------
- // 3-dimensional data
- // --------------------------------------------------------------------------------------------
-
- public final static String DATAPOINTS = "0|50.90|16.20|72.08|\n" + "1|73.65|61.76|62.89|\n" + "2|61.73|49.95|92.74|\n"
- + "3|1.60|70.11|16.32|\n" + "4|2.43|19.81|89.56|\n" + "5|67.99|9.00|14.48|\n" + "6|87.80|84.49|55.83|\n"
- + "7|90.26|42.99|53.29|\n" + "8|51.36|6.16|9.35|\n" + "9|12.43|9.52|12.54|\n" + "10|80.01|8.78|29.74|\n"
- + "11|92.76|2.93|80.07|\n" + "12|46.32|100.00|22.98|\n" + "13|34.11|45.61|58.60|\n"
- + "14|68.82|16.36|96.60|\n" + "15|81.47|76.45|28.40|\n" + "16|65.55|40.21|43.43|\n"
- + "17|84.22|88.56|13.31|\n" + "18|36.99|68.36|57.12|\n" + "19|28.87|37.69|91.04|\n"
- + "20|31.56|13.22|86.00|\n" + "21|18.49|34.45|54.52|\n" + "22|13.33|94.02|92.07|\n"
- + "23|91.19|81.62|55.06|\n" + "24|85.78|39.02|25.58|\n" + "25|94.41|47.07|78.23|\n"
- + "26|90.62|10.43|80.20|\n" + "27|31.52|85.81|39.79|\n" + "28|24.65|77.98|26.35|\n"
- + "29|69.34|75.79|63.96|\n" + "30|22.56|78.61|66.66|\n" + "31|91.74|83.82|73.92|\n"
- + "32|76.64|89.53|44.66|\n" + "33|36.02|73.01|92.32|\n" + "34|87.86|18.94|10.74|\n"
- + "35|91.94|34.61|5.20|\n" + "36|12.52|47.01|95.29|\n" + "37|44.01|26.19|78.50|\n"
- + "38|26.20|73.36|10.08|\n" + "39|15.21|17.37|54.33|\n" + "40|27.96|94.81|44.41|\n"
- + "41|26.44|44.81|70.88|\n" + "42|53.29|26.69|2.40|\n" + "43|23.94|11.50|1.71|\n"
- + "44|19.00|25.48|50.80|\n" + "45|82.26|1.88|58.08|\n" + "46|47.56|82.54|82.73|\n"
- + "47|51.54|35.10|32.95|\n" + "48|86.71|55.51|19.08|\n" + "49|54.16|23.68|32.41|\n"
- + "50|71.81|32.83|46.66|\n" + "51|20.70|14.19|64.96|\n" + "52|57.17|88.56|55.23|\n"
- + "53|91.39|49.38|70.55|\n" + "54|47.90|62.07|76.03|\n" + "55|55.70|37.77|30.15|\n"
- + "56|87.87|74.62|25.95|\n" + "57|95.70|45.04|15.27|\n" + "58|41.61|89.37|24.45|\n"
- + "59|82.19|20.84|11.13|\n" + "60|49.88|2.62|18.62|\n" + "61|16.42|53.30|74.13|\n"
- + "62|38.37|72.62|35.16|\n" + "63|43.26|49.59|92.56|\n" + "64|28.96|2.36|78.49|\n"
- + "65|88.41|91.43|92.55|\n" + "66|98.61|79.58|33.03|\n" + "67|4.94|18.65|30.78|\n"
- + "68|75.89|79.30|63.90|\n" + "69|93.18|76.26|9.50|\n" + "70|73.43|70.50|76.49|\n"
- + "71|78.64|90.87|34.49|\n" + "72|58.47|63.07|8.82|\n" + "73|69.74|54.36|64.43|\n"
- + "74|38.47|36.60|33.39|\n" + "75|51.07|14.75|2.54|\n" + "76|24.18|16.85|15.00|\n"
- + "77|7.56|50.72|93.45|\n" + "78|64.28|97.01|57.31|\n" + "79|85.30|24.13|76.57|\n"
- + "80|72.78|30.78|13.11|\n" + "81|18.42|17.45|32.20|\n" + "82|87.44|74.98|87.90|\n"
- + "83|38.30|17.77|37.33|\n" + "84|63.62|7.90|34.23|\n" + "85|8.84|67.87|30.65|\n"
- + "86|76.12|51.83|80.12|\n" + "87|32.30|74.79|4.39|\n" + "88|41.73|45.34|18.66|\n"
- + "89|58.13|18.43|83.38|\n" + "90|98.10|33.46|83.07|\n" + "91|17.76|4.10|88.51|\n"
- + "92|60.58|18.15|59.96|\n" + "93|50.11|33.25|85.64|\n" + "94|97.74|60.93|38.97|\n"
- + "95|76.31|52.50|95.43|\n" + "96|7.71|85.85|36.26|\n" + "97|9.32|72.21|42.17|\n"
- + "98|71.29|51.88|57.62|\n" + "99|31.39|7.27|88.74|";
-
- public static final String INITIAL_CENTERS =
- "0|1.96|65.04|20.82|\n" +
- "1|53.99|84.23|81.59|\n" +
- "2|97.28|74.50|40.32|\n" +
- "3|63.57|24.53|87.07|\n" +
- "4|28.10|43.27|86.53|\n" +
- "5|99.51|62.70|64.48|\n" +
- "6|30.31|30.36|80.46|";
-
- public static final String CENTERS_AFTER_ONE_STEP =
- "0|28.47|54.80|21.88|\n" +
- "1|52.74|80.10|73.03|\n" +
- "2|83.92|60.45|25.17|\n" +
- "3|70.73|20.18|67.06|\n" +
- "4|22.51|47.19|86.23|\n" +
- "5|82.70|53.79|68.68|\n" +
- "6|29.74|19.17|59.16|";
-
-
- public static final String CENTERS_AFTER_ONE_STEP_SINGLE_DIGIT =
- "0|28.5|54.8|21.9|\n" +
- "1|52.7|80.1|73.0|\n" +
- "2|83.9|60.5|25.2|\n" +
- "3|70.7|20.2|67.0|\n" +
- "4|22.5|47.2|86.2|\n" +
- "5|82.7|53.8|68.7|\n" +
- "6|29.7|19.2|59.2|";
-
-
- public static final String CENTERS_AFTER_20_ITERATIONS_SINGLE_DIGIT =
- "0|38.3|54.5|19.3|\n" +
- "1|32.1|83.0|50.4|\n" +
- "2|87.5|56.6|20.3|\n" +
- "3|75.4|18.6|67.5|\n" +
- "4|24.9|29.2|77.6|\n" +
- "5|78.7|66.1|70.8|\n" +
- "6|39.5|14.0|18.7|\n";
-
- public static final String CENTERS_AFTER_20_ITERATIONS_DOUBLE_DIGIT =
- "0|38.25|54.52|19.34|\n" +
- "1|32.14|83.04|50.35|\n" +
- "2|87.48|56.57|20.27|\n" +
- "3|75.40|18.65|67.49|\n" +
- "4|24.93|29.25|77.56|\n" +
- "5|78.67|66.07|70.82|\n" +
- "6|39.51|14.04|18.74|\n";
-
- // --------------------------------------------------------------------------------------------
- // 2-dimensional data
- // --------------------------------------------------------------------------------------------
-
- public final static String DATAPOINTS_2D = "0|50.90|16.20|\n" + "1|73.65|61.76|\n" + "2|61.73|49.95|\n"
- + "3|1.60|70.11|\n" + "4|2.43|19.81|\n" + "5|67.99|9.00|\n" + "6|87.80|84.49|\n"
- + "7|90.26|42.99|\n" + "8|51.36|6.16|\n" + "9|12.43|9.52|\n" + "10|80.01|8.78|\n"
- + "11|92.76|2.93|\n" + "12|46.32|100.00|\n"+ "13|34.11|45.61|\n"
- + "14|68.82|16.36|\n" + "15|81.47|76.45|\n" + "16|65.55|40.21|\n"
- + "17|84.22|88.56|\n" + "18|36.99|68.36|\n" + "19|28.87|37.69|\n"
- + "20|31.56|13.22|\n" + "21|18.49|34.45|\n" + "22|13.33|94.02|\n"
- + "23|91.19|81.62|\n" + "24|85.78|39.02|\n" + "25|94.41|47.07|\n"
- + "26|90.62|10.43|\n" + "27|31.52|85.81|\n" + "28|24.65|77.98|\n"
- + "29|69.34|75.79|\n" + "30|22.56|78.61|\n" + "31|91.74|83.82|\n"
- + "32|76.64|89.53|\n" + "33|36.02|73.01|\n" + "34|87.86|18.94|\n"
- + "35|91.94|34.61|\n" + "36|12.52|47.01|\n" + "37|44.01|26.19|\n"
- + "38|26.20|73.36|\n" + "39|15.21|17.37|\n" + "40|27.96|94.81|\n"
- + "41|26.44|44.81|\n" + "42|53.29|26.69|\n" + "43|23.94|11.50|n"
- + "44|19.00|25.48|\n" + "45|82.26|1.88|\n" + "46|47.56|82.54|\n"
- + "47|51.54|35.10|\n" + "48|86.71|55.51|\n" + "49|54.16|23.68|\n"
- + "50|71.81|32.83|\n" + "51|20.70|14.19|\n" + "52|57.17|88.56|\n"
- + "53|91.39|49.38|\n" + "54|47.90|62.07|\n" + "55|55.70|37.77|\n"
- + "56|87.87|74.62|\n" + "57|95.70|45.04|\n" + "58|41.61|89.37|\n"
- + "59|82.19|20.84|\n" + "60|49.88|2.62|\n" + "61|16.42|53.30|\n"
- + "62|38.37|72.62|\n" + "63|43.26|49.59|\n" + "64|28.96|2.36|\n"
- + "65|88.41|91.43|\n" + "66|98.61|79.58|\n" + "67|4.94|18.65|\n"
- + "68|75.89|79.30|\n" + "69|93.18|76.26|\n" + "70|73.43|70.50|\n"
- + "71|78.64|90.87|\n" + "72|58.47|63.07|\n" + "73|69.74|54.36|\n"
- + "74|38.47|36.60|\n" + "75|51.07|14.75|\n" + "76|24.18|16.85|\n"
- + "77|7.56|50.72|\n" + "78|64.28|97.01|\n" + "79|85.30|24.13|\n"
- + "80|72.78|30.78|\n" + "81|18.42|17.45|\n" + "82|87.44|74.98|\n"
- + "83|38.30|17.77|\n" + "84|63.62|7.90|\n" + "85|8.84|67.87|\n"
- + "86|76.12|51.83|\n" + "87|32.30|74.79|n" + "88|41.73|45.34|\n"
- + "89|58.13|18.43|\n" + "90|98.10|33.46|\n" + "91|17.76|4.10|\n"
- + "92|60.58|18.15|\n" + "93|50.11|33.25|\n" + "94|97.74|60.93|\n"
- + "95|76.31|52.50|\n" + "96|7.71|85.85|\n" + "97|9.32|72.21|\n"
- + "98|71.29|51.88|\n" + "99|31.39|7.27|";
-
- public static final String INITIAL_CENTERS_2D =
- "0|1.96|65.04|\n" +
- "1|53.99|84.23|\n" +
- "2|97.28|74.50|\n" +
- "3|63.57|24.53|\n" +
- "4|28.10|43.27|\n" +
- "5|99.51|62.70|\n" +
- "6|30.31|30.36|";
-
- public static final String CENTERS_2D_AFTER_SINGLE_ITERATION_DOUBLE_DIGIT =
- "0|13.53|74.53|\n" +
- "1|49.12|80.49|\n" +
- "2|87.20|81.83|\n" +
- "3|67.39|23.32|\n" +
- "4|26.94|46.34|\n" +
- "5|88.39|48.64|\n" +
- "6|23.20|16.71|";
-
- public static final String CENTERS_2D_AFTER_20_ITERATIONS_DOUBLE_DIGIT =
- "0|15.80|79.42|\n" +
- "1|43.11|78.20|\n" +
- "2|83.13|82.18|\n" +
- "3|65.70|18.49|\n" +
- "4|25.13|44.42|\n" +
- "5|82.90|48.16|\n" +
- "6|20.79|13.08|";
-
- // --------------------------------------------------------------------------------------------
- // testing / verification
- // --------------------------------------------------------------------------------------------
-
- public static void checkResultsWithDelta(String expectedResults, List<String> resultLines, final double maxDelta) {
-
- Comparator<String> deltaComp = new Comparator<String>() {
-
- @Override
- public int compare(String o1, String o2) {
-
- StringTokenizer st1 = new StringTokenizer(o1, "|");
- StringTokenizer st2 = new StringTokenizer(o2, "|");
-
- if(st1.countTokens() != st2.countTokens()) {
- return st1.countTokens() - st2.countTokens();
- }
-
- // first token is ID
- String t1 = st1.nextToken();
- String t2 = st2.nextToken();
- if(!t1.equals(t2)) {
- return t1.compareTo(t2);
- }
-
- while(st1.hasMoreTokens()) {
- t1 = st1.nextToken();
- t2 = st2.nextToken();
-
- double d1 = Double.parseDouble(t1);
- double d2 = Double.parseDouble(t2);
-
- if (Math.abs(d1-d2) > maxDelta) {
- return d1 < d2 ? -1 : 1;
- }
- }
-
- return 0;
- }
- };
-
- // ------- Test results -----------
-
- Collections.sort(resultLines, deltaComp);
-
- final String[] should = expectedResults.split("\n");
- final String[] is = (String[]) resultLines.toArray(new String[resultLines.size()]);
-
- Assert.assertEquals("Wrong number of result lines.", should.length, is.length);
-
- for (int i = 0; i < should.length; i++) {
- StringTokenizer shouldRecord = new StringTokenizer(should[i], "|");
- StringTokenizer isRecord = new StringTokenizer(is[i], "|");
-
- Assert.assertEquals("Records don't match.", shouldRecord.countTokens(), isRecord.countTokens());
-
- // first token is ID
- String shouldToken = shouldRecord.nextToken();
- String isToken = isRecord.nextToken();
-
- Assert.assertEquals("Records don't match.", shouldToken, isToken);
-
- while (shouldRecord.hasMoreTokens()) {
- shouldToken = shouldRecord.nextToken();
- isToken = isRecord.nextToken();
-
- double shouldDouble = Double.parseDouble(shouldToken);
- double isDouble = Double.parseDouble(isToken);
-
- Assert.assertTrue("Value " + isDouble + " is out of range of " + shouldDouble + " +/- " + maxDelta, shouldDouble - maxDelta <= isDouble && shouldDouble + maxDelta >= isDouble);
- }
- }
- }
-
-
- private KMeansData() {}
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-test-utils/src/test/java/org/apache/flink/test/testdata/PageRankData.java
----------------------------------------------------------------------
diff --git a/flink-test-utils/src/test/java/org/apache/flink/test/testdata/PageRankData.java b/flink-test-utils/src/test/java/org/apache/flink/test/testdata/PageRankData.java
deleted file mode 100644
index f91e611..0000000
--- a/flink-test-utils/src/test/java/org/apache/flink/test/testdata/PageRankData.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.test.testdata;
-
-public class PageRankData {
-
- public static final int NUM_VERTICES = 5;
-
- public static final String VERTICES = "1\n" +
- "2\n" +
- "5\n" +
- "3\n" +
- "4";
-
- public static final String EDGES = "2 1\n" +
- "5 2\n" +
- "5 4\n" +
- "4 3\n" +
- "4 2\n" +
- "1 4\n" +
- "1 2\n" +
- "1 3\n" +
- "3 5\n";
-
-
- public static final String RANKS_AFTER_3_ITERATIONS = "1 0.237\n" +
- "2 0.248\n" +
- "3 0.173\n" +
- "4 0.175\n" +
- "5 0.165";
-
-
- public static final String RANKS_AFTER_EPSILON_0_0001_CONVERGENCE = "1 0.238\n" +
- "2 0.244\n" +
- "3 0.170\n" +
- "4 0.171\n" +
- "5 0.174";
-
- private PageRankData() {}
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-test-utils/src/test/java/org/apache/flink/test/testdata/TransitiveClosureData.java
----------------------------------------------------------------------
diff --git a/flink-test-utils/src/test/java/org/apache/flink/test/testdata/TransitiveClosureData.java b/flink-test-utils/src/test/java/org/apache/flink/test/testdata/TransitiveClosureData.java
deleted file mode 100644
index 652b195..0000000
--- a/flink-test-utils/src/test/java/org/apache/flink/test/testdata/TransitiveClosureData.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.test.testdata;
-
-import org.junit.Assert;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.util.regex.Pattern;
-
-public class TransitiveClosureData {
-
- public static void checkOddEvenResult(BufferedReader result) throws IOException {
- Pattern split = Pattern.compile(" ");
- String line;
- while ((line = result.readLine()) != null) {
- String[] res = split.split(line);
- Assert.assertEquals("Malformed result: Wrong number of tokens in line.", 2, res.length);
- try {
- int from = Integer.parseInt(res[0]);
- int to = Integer.parseInt(res[1]);
-
- Assert.assertEquals("Vertex should not be reachable.", from % 2, to % 2);
- } catch (NumberFormatException e) {
- Assert.fail("Malformed result.");
- }
- }
- }
-
- private TransitiveClosureData() {}
-}
[07/15] flink git commit: [FLINK-3995] [build] Properly structure
test scopes and dependencies
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-runtime/src/main/scala/org/apache/flink/runtime/testingUtils/TestingJobManagerMessages.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/testingUtils/TestingJobManagerMessages.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/testingUtils/TestingJobManagerMessages.scala
new file mode 100644
index 0000000..ec513e3
--- /dev/null
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/testingUtils/TestingJobManagerMessages.scala
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.testingUtils
+
+import java.util.Map
+
+import akka.actor.ActorRef
+import org.apache.flink.api.common.JobID
+import org.apache.flink.api.common.accumulators.Accumulator
+import org.apache.flink.runtime.accumulators.AccumulatorRegistry
+import org.apache.flink.runtime.checkpoint.CompletedCheckpoint
+import org.apache.flink.runtime.executiongraph.{ExecutionAttemptID, ExecutionGraph}
+import org.apache.flink.runtime.instance.ActorGateway
+import org.apache.flink.runtime.jobgraph.JobStatus
+
+object TestingJobManagerMessages {
+
+ case class RequestExecutionGraph(jobID: JobID)
+
+ sealed trait ResponseExecutionGraph {
+ def jobID: JobID
+ }
+
+ case class ExecutionGraphFound(jobID: JobID, executionGraph: ExecutionGraph) extends
+ ResponseExecutionGraph
+
+ case class ExecutionGraphNotFound(jobID: JobID) extends ResponseExecutionGraph
+
+ case class WaitForAllVerticesToBeRunning(jobID: JobID)
+ case class WaitForAllVerticesToBeRunningOrFinished(jobID: JobID)
+ case class AllVerticesRunning(jobID: JobID)
+
+ case class NotifyWhenJobRemoved(jobID: JobID)
+
+ case class RequestWorkingTaskManager(jobID: JobID)
+ case class WorkingTaskManager(gatewayOption: Option[ActorGateway])
+
+ case class NotifyWhenJobStatus(jobID: JobID, state: JobStatus)
+ case class JobStatusIs(jobID: JobID, state: JobStatus)
+
+ case object NotifyListeners
+
+ case class NotifyWhenTaskManagerTerminated(taskManager: ActorRef)
+ case class TaskManagerTerminated(taskManager: ActorRef)
+
+ /**
+ * Registers a listener to receive a message when accumulators changed.
+ * The change must be explicitly triggered by the TestingTaskManager which can receive an
+ * [[org.apache.flink.runtime.testingUtils.TestingTaskManagerMessages.AccumulatorsChanged]]
+ * message by a task that changed the accumulators. This message is then
+ * forwarded to the JobManager which will send the accumulators in the [[UpdatedAccumulators]]
+ * message when the next Heartbeat occurs.
+ */
+ case class NotifyWhenAccumulatorChange(jobID: JobID)
+
+ /**
+ * Reports updated accumulators back to the listener.
+ */
+ case class UpdatedAccumulators(jobID: JobID,
+ flinkAccumulators: Map[ExecutionAttemptID, Map[AccumulatorRegistry.Metric, Accumulator[_,_]]],
+ userAccumulators: Map[String, Accumulator[_,_]])
+
+ /** Notifies the sender when the [[TestingJobManager]] has been elected as the leader
+ *
+ */
+ case object NotifyWhenLeader
+
+ /**
+ * Registers to be notified by an [[org.apache.flink.runtime.messages.Messages.Acknowledge]]
+ * message when at least numRegisteredTaskManager have registered at the JobManager.
+ *
+ * @param numRegisteredTaskManager minimum number of registered TMs before the sender is notified
+ */
+ case class NotifyWhenAtLeastNumTaskManagerAreRegistered(numRegisteredTaskManager: Int)
+
+ /** Disables the post stop method of the [[TestingJobManager]].
+ *
+ * Only the leaderElectionService is stopped in the postStop method call to revoke the leadership
+ */
+ case object DisablePostStop
+
+ /**
+ * Requests a savepoint from the job manager.
+ *
+ * @param savepointPath The path of the savepoint to request.
+ */
+ case class RequestSavepoint(savepointPath: String)
+
+ /**
+ * Response to a savepoint request.
+ *
+ * @param savepoint The requested savepoint or null if none available.
+ */
+ case class ResponseSavepoint(savepoint: CompletedCheckpoint)
+
+ def getNotifyWhenLeader(): AnyRef = NotifyWhenLeader
+ def getDisablePostStop(): AnyRef = DisablePostStop
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-runtime/src/main/scala/org/apache/flink/runtime/testingUtils/TestingMemoryArchivist.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/testingUtils/TestingMemoryArchivist.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/testingUtils/TestingMemoryArchivist.scala
new file mode 100644
index 0000000..48a1ddd
--- /dev/null
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/testingUtils/TestingMemoryArchivist.scala
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.testingUtils
+
+import org.apache.flink.runtime.jobmanager.MemoryArchivist
+import org.apache.flink.runtime.testingUtils.TestingJobManagerMessages.{ExecutionGraphFound, ExecutionGraphNotFound, RequestExecutionGraph}
+
+/** Memory archivist extended by testing messages
+ *
+ * @param maxEntries number of maximum number of archived jobs
+ */
+class TestingMemoryArchivist(maxEntries: Int) extends MemoryArchivist(maxEntries) {
+
+ override def handleMessage: Receive = {
+ handleTestingMessage orElse super.handleMessage
+ }
+
+ def handleTestingMessage: Receive = {
+ case RequestExecutionGraph(jobID) =>
+ val executionGraph = graphs.get(jobID)
+
+ executionGraph match {
+ case Some(graph) => sender ! decorateMessage(ExecutionGraphFound(jobID, graph))
+ case None => sender ! decorateMessage(ExecutionGraphNotFound(jobID))
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-runtime/src/main/scala/org/apache/flink/runtime/testingUtils/TestingMessages.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/testingUtils/TestingMessages.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/testingUtils/TestingMessages.scala
new file mode 100644
index 0000000..91d169a
--- /dev/null
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/testingUtils/TestingMessages.scala
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.testingUtils
+
+import akka.actor.ActorRef
+import org.apache.flink.api.common.JobID
+
+object TestingMessages {
+
+ case class CheckIfJobRemoved(jobID: JobID)
+
+ case object DisableDisconnect
+
+ case object Alive
+
+ def getAlive: AnyRef = Alive
+
+ def getDisableDisconnect: AnyRef = DisableDisconnect
+
+ case object NotifyOfComponentShutdown
+ case class ComponentShutdown(ref: ActorRef)
+
+ def getNotifyOfComponentShutdown(): AnyRef = NotifyOfComponentShutdown
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-runtime/src/main/scala/org/apache/flink/runtime/testingUtils/TestingTaskManager.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/testingUtils/TestingTaskManager.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/testingUtils/TestingTaskManager.scala
new file mode 100644
index 0000000..2597753
--- /dev/null
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/testingUtils/TestingTaskManager.scala
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.testingUtils
+
+import org.apache.flink.runtime.clusterframework.types.ResourceID
+import org.apache.flink.runtime.instance.InstanceConnectionInfo
+import org.apache.flink.runtime.io.disk.iomanager.IOManager
+import org.apache.flink.runtime.io.network.NetworkEnvironment
+import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService
+import org.apache.flink.runtime.memory.MemoryManager
+import org.apache.flink.runtime.taskmanager.{TaskManager, TaskManagerConfiguration}
+
+import scala.language.postfixOps
+
+/** Subclass of the [[TaskManager]] to support testing messages
+ */
+class TestingTaskManager(
+ config: TaskManagerConfiguration,
+ resourceID: ResourceID,
+ connectionInfo: InstanceConnectionInfo,
+ memoryManager: MemoryManager,
+ ioManager: IOManager,
+ network: NetworkEnvironment,
+ numberOfSlots: Int,
+ leaderRetrievalService: LeaderRetrievalService)
+ extends TaskManager(
+ config,
+ resourceID,
+ connectionInfo,
+ memoryManager,
+ ioManager,
+ network,
+ numberOfSlots,
+ leaderRetrievalService)
+ with TestingTaskManagerLike {
+
+ def this(
+ config: TaskManagerConfiguration,
+ connectionInfo: InstanceConnectionInfo,
+ memoryManager: MemoryManager,
+ ioManager: IOManager,
+ network: NetworkEnvironment,
+ numberOfSlots: Int,
+ leaderRetrievalService: LeaderRetrievalService) {
+ this(
+ config,
+ ResourceID.generate(),
+ connectionInfo,
+ memoryManager,
+ ioManager,
+ network,
+ numberOfSlots,
+ leaderRetrievalService)
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-runtime/src/main/scala/org/apache/flink/runtime/testingUtils/TestingTaskManagerLike.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/testingUtils/TestingTaskManagerLike.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/testingUtils/TestingTaskManagerLike.scala
new file mode 100644
index 0000000..b41db31
--- /dev/null
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/testingUtils/TestingTaskManagerLike.scala
@@ -0,0 +1,248 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.testingUtils
+
+import akka.actor.{ActorRef, Terminated}
+import org.apache.flink.api.common.JobID
+import org.apache.flink.runtime.FlinkActor
+import org.apache.flink.runtime.execution.ExecutionState
+import org.apache.flink.runtime.executiongraph.ExecutionAttemptID
+import org.apache.flink.runtime.messages.JobManagerMessages.{RequestLeaderSessionID, ResponseLeaderSessionID}
+import org.apache.flink.runtime.messages.Messages.{Acknowledge, Disconnect}
+import org.apache.flink.runtime.messages.RegistrationMessages.{AcknowledgeRegistration, AlreadyRegistered}
+import org.apache.flink.runtime.messages.TaskMessages.{SubmitTask, TaskInFinalState, UpdateTaskExecutionState}
+import org.apache.flink.runtime.taskmanager.TaskManager
+import org.apache.flink.runtime.testingUtils.TestingJobManagerMessages.NotifyWhenJobRemoved
+import org.apache.flink.runtime.testingUtils.TestingMessages._
+import org.apache.flink.runtime.testingUtils.TestingTaskManagerMessages._
+
+import scala.concurrent.duration._
+import scala.language.postfixOps
+
+/** This mixin can be used to decorate a TaskManager with messages for testing purposes. */
+trait TestingTaskManagerLike extends FlinkActor {
+ that: TaskManager =>
+
+ import scala.collection.JavaConverters._
+
+ val waitForRemoval = scala.collection.mutable.HashMap[ExecutionAttemptID, Set[ActorRef]]()
+ val waitForJobManagerToBeTerminated = scala.collection.mutable.HashMap[String, Set[ActorRef]]()
+ val waitForRegisteredAtResourceManager =
+ scala.collection.mutable.HashMap[ActorRef, Set[ActorRef]]()
+ val waitForRunning = scala.collection.mutable.HashMap[ExecutionAttemptID, Set[ActorRef]]()
+ val unregisteredTasks = scala.collection.mutable.HashSet[ExecutionAttemptID]()
+
+ /** Map of registered task submit listeners */
+ val registeredSubmitTaskListeners = scala.collection.mutable.HashMap[JobID, ActorRef]()
+
+ val waitForShutdown = scala.collection.mutable.HashSet[ActorRef]()
+
+ var disconnectDisabled = false
+
+ /**
+ * Handler for testing related messages
+ */
+ abstract override def handleMessage: Receive = {
+ handleTestingMessage orElse super.handleMessage
+ }
+
+ def handleTestingMessage: Receive = {
+ case Alive => sender() ! Acknowledge
+
+ case NotifyWhenTaskIsRunning(executionID) =>
+ Option(runningTasks.get(executionID)) match {
+ case Some(task) if task.getExecutionState == ExecutionState.RUNNING =>
+ sender ! decorateMessage(true)
+
+ case _ =>
+ val listeners = waitForRunning.getOrElse(executionID, Set())
+ waitForRunning += (executionID -> (listeners + sender))
+ }
+
+ case RequestRunningTasks =>
+ sender ! decorateMessage(ResponseRunningTasks(runningTasks.asScala.toMap))
+
+ case NotifyWhenTaskRemoved(executionID) =>
+ Option(runningTasks.get(executionID)) match {
+ case Some(_) =>
+ val set = waitForRemoval.getOrElse(executionID, Set())
+ waitForRemoval += (executionID -> (set + sender))
+ case None =>
+ if(unregisteredTasks.contains(executionID)) {
+ sender ! decorateMessage(true)
+ } else {
+ val set = waitForRemoval.getOrElse(executionID, Set())
+ waitForRemoval += (executionID -> (set + sender))
+ }
+ }
+
+ case TaskInFinalState(executionID) =>
+ super.handleMessage(TaskInFinalState(executionID))
+ waitForRemoval.remove(executionID) match {
+ case Some(actors) => for(actor <- actors) actor ! decorateMessage(true)
+ case None =>
+ }
+
+ unregisteredTasks += executionID
+
+ case RequestBroadcastVariablesWithReferences =>
+ sender ! decorateMessage(
+ ResponseBroadcastVariablesWithReferences(
+ bcVarManager.getNumberOfVariablesWithReferences)
+ )
+
+ case RequestNumActiveConnections =>
+ val numActive = if (network.isAssociated) {
+ network.getConnectionManager.getNumberOfActiveConnections
+ } else {
+ 0
+ }
+ sender ! decorateMessage(ResponseNumActiveConnections(numActive))
+
+ case NotifyWhenJobRemoved(jobID) =>
+ if(runningTasks.values.asScala.exists(_.getJobID == jobID)){
+ context.system.scheduler.scheduleOnce(
+ 200 milliseconds,
+ self,
+ decorateMessage(CheckIfJobRemoved(jobID)))(
+ context.dispatcher,
+ sender()
+ )
+ }else{
+ sender ! decorateMessage(true)
+ }
+
+ case CheckIfJobRemoved(jobID) =>
+ if(runningTasks.values.asScala.forall(_.getJobID != jobID)){
+ sender ! decorateMessage(true)
+ } else {
+ context.system.scheduler.scheduleOnce(
+ 200 milliseconds,
+ self,
+ decorateMessage(CheckIfJobRemoved(jobID)))(
+ context.dispatcher,
+ sender()
+ )
+ }
+
+ case NotifyWhenJobManagerTerminated(jobManager) =>
+ val waiting = waitForJobManagerToBeTerminated.getOrElse(jobManager.path.name, Set())
+ waitForJobManagerToBeTerminated += jobManager.path.name -> (waiting + sender)
+
+ case RegisterSubmitTaskListener(jobId) =>
+ registeredSubmitTaskListeners.put(jobId, sender())
+
+ case msg@SubmitTask(tdd) =>
+ registeredSubmitTaskListeners.get(tdd.getJobID) match {
+ case Some(listenerRef) =>
+ listenerRef ! ResponseSubmitTaskListener(tdd)
+ case None =>
+ // Nothing to do
+ }
+
+ super.handleMessage(msg)
+
+ /**
+ * Message from task manager that accumulator values changed and need to be reported immediately
+ * instead of lazily through the
+ * [[org.apache.flink.runtime.messages.TaskManagerMessages.Heartbeat]] message. We forward this
+ * message to the job manager that it knows it should report to the listeners.
+ */
+ case msg: AccumulatorsChanged =>
+ currentJobManager match {
+ case Some(jobManager) =>
+ jobManager.forward(msg)
+ sendHeartbeatToJobManager()
+ sender ! true
+ case None =>
+ }
+
+ case msg@Terminated(jobManager) =>
+ super.handleMessage(msg)
+
+ waitForJobManagerToBeTerminated.remove(jobManager.path.name) foreach {
+ _ foreach {
+ _ ! decorateMessage(JobManagerTerminated(jobManager))
+ }
+ }
+
+ case msg:Disconnect =>
+ if (!disconnectDisabled) {
+ super.handleMessage(msg)
+
+ val jobManager = sender()
+
+ waitForJobManagerToBeTerminated.remove(jobManager.path.name) foreach {
+ _ foreach {
+ _ ! decorateMessage(JobManagerTerminated(jobManager))
+ }
+ }
+ }
+
+ case DisableDisconnect =>
+ disconnectDisabled = true
+
+ case NotifyOfComponentShutdown =>
+ waitForShutdown += sender()
+
+ case msg @ UpdateTaskExecutionState(taskExecutionState) =>
+ super.handleMessage(msg)
+
+ if(taskExecutionState.getExecutionState == ExecutionState.RUNNING) {
+ waitForRunning.get(taskExecutionState.getID) foreach {
+ _ foreach (_ ! decorateMessage(true))
+ }
+ }
+
+ case RequestLeaderSessionID =>
+ sender() ! ResponseLeaderSessionID(leaderSessionID.orNull)
+
+ case NotifyWhenRegisteredAtJobManager(jobManager: ActorRef) =>
+ if(isConnected && jobManager == currentJobManager.get) {
+ sender() ! true
+ } else {
+ val list = waitForRegisteredAtResourceManager.getOrElse(
+ jobManager,
+ Set[ActorRef]())
+
+ waitForRegisteredAtResourceManager += jobManager -> (list + sender())
+ }
+
+ case msg @ (_: AcknowledgeRegistration | _: AlreadyRegistered) =>
+ super.handleMessage(msg)
+
+ val jm = sender()
+
+ waitForRegisteredAtResourceManager.remove(jm).foreach {
+ listeners => listeners.foreach{
+ listener =>
+ listener ! true
+ }
+ }
+ }
+
+ /**
+ * No killing of the VM for testing.
+ */
+ override protected def shutdown(): Unit = {
+ log.info("Shutting down TestingJobManager.")
+ waitForShutdown.foreach(_ ! ComponentShutdown(self))
+ waitForShutdown.clear()
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-runtime/src/main/scala/org/apache/flink/runtime/testingUtils/TestingTaskManagerMessages.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/testingUtils/TestingTaskManagerMessages.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/testingUtils/TestingTaskManagerMessages.scala
new file mode 100644
index 0000000..974e4e8
--- /dev/null
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/testingUtils/TestingTaskManagerMessages.scala
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.testingUtils
+
+import akka.actor.ActorRef
+import org.apache.flink.api.common.JobID
+import org.apache.flink.runtime.deployment.TaskDeploymentDescriptor
+import org.apache.flink.runtime.executiongraph.ExecutionAttemptID
+import org.apache.flink.runtime.taskmanager.Task
+
+/**
+ * Additional messages that the [[TestingTaskManager]] understands.
+ */
+object TestingTaskManagerMessages {
+
+ case class NotifyWhenTaskRemoved(executionID: ExecutionAttemptID)
+
+ case class NotifyWhenTaskIsRunning(executionID: ExecutionAttemptID)
+
+ case class ResponseRunningTasks(tasks: Map[ExecutionAttemptID, Task]){
+ import collection.JavaConverters._
+ def asJava: java.util.Map[ExecutionAttemptID, Task] = tasks.asJava
+ }
+
+ case class ResponseBroadcastVariablesWithReferences(number: Int)
+
+ case object RequestNumActiveConnections
+ case class ResponseNumActiveConnections(number: Int)
+
+ case object RequestRunningTasks
+
+ case object RequestBroadcastVariablesWithReferences
+
+ case class NotifyWhenJobManagerTerminated(jobManager: ActorRef)
+
+ case class JobManagerTerminated(jobManager: ActorRef)
+
+ case class NotifyWhenRegisteredAtJobManager(resourceManager: ActorRef)
+
+ /**
+ * Message to give a hint to the task manager that accumulator values were updated in the task.
+ * This message is forwarded to the job manager which knows that it needs to notify listeners
+ * of accumulator updates.
+ */
+ case class AccumulatorsChanged(jobID: JobID)
+
+ /**
+ * Registers a listener for all [[org.apache.flink.runtime.messages.TaskMessages.SubmitTask]]
+ * messages of the given job.
+ *
+ * If a task is submitted with the given job ID the task deployment
+ * descriptor is forwarded to the listener.
+ *
+ * @param jobId The job ID to listen for.
+ */
+ case class RegisterSubmitTaskListener(jobId: JobID)
+
+ /**
+ * A response to a listened job ID containing the submitted task deployment descriptor.
+ *
+ * @param tdd The submitted task deployment descriptor.
+ */
+ case class ResponseSubmitTaskListener(tdd: TaskDeploymentDescriptor)
+
+ // --------------------------------------------------------------------------
+ // Utility methods to allow simpler case object access from Java
+ // --------------------------------------------------------------------------
+
+ def getRequestRunningTasksMessage: AnyRef = {
+ RequestRunningTasks
+ }
+
+ def getRequestBroadcastVariablesWithReferencesMessage: AnyRef = {
+ RequestBroadcastVariablesWithReferences
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ExecutionGraphCheckpointCoordinatorTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ExecutionGraphCheckpointCoordinatorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ExecutionGraphCheckpointCoordinatorTest.java
index a801348..dcc98c9 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ExecutionGraphCheckpointCoordinatorTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ExecutionGraphCheckpointCoordinatorTest.java
@@ -19,7 +19,8 @@
package org.apache.flink.runtime.checkpoint;
import akka.actor.ActorSystem;
-import org.apache.flink.api.common.ExecutionConfigTest;
+
+import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.api.common.JobID;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.akka.AkkaUtils;
@@ -30,7 +31,10 @@ import org.apache.flink.runtime.executiongraph.ExecutionJobVertex;
import org.apache.flink.runtime.executiongraph.restart.NoRestartStrategy;
import org.apache.flink.runtime.jobmanager.RecoveryMode;
import org.apache.flink.runtime.testingUtils.TestingUtils;
+import org.apache.flink.util.SerializedValue;
+
import org.junit.Test;
+
import scala.concurrent.duration.FiniteDuration;
import java.lang.reflect.Field;
@@ -42,7 +46,7 @@ import java.util.concurrent.TimeUnit;
import static org.junit.Assert.assertEquals;
public class ExecutionGraphCheckpointCoordinatorTest {
-
+
@Test
public void testCheckpointAndSavepointCoordinatorShareCheckpointIDCounter() throws Exception {
ExecutionGraph executionGraph = new ExecutionGraph(
@@ -50,7 +54,7 @@ public class ExecutionGraphCheckpointCoordinatorTest {
new JobID(),
"test",
new Configuration(),
- ExecutionConfigTest.getSerializedConfig(),
+ new SerializedValue<>(new ExecutionConfig()),
new FiniteDuration(1, TimeUnit.DAYS),
new NoRestartStrategy(),
Collections.<BlobKey>emptyList(),
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-runtime/src/test/java/org/apache/flink/runtime/deployment/TaskDeploymentDescriptorTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/deployment/TaskDeploymentDescriptorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/deployment/TaskDeploymentDescriptorTest.java
index f0bf694..56da9c8 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/deployment/TaskDeploymentDescriptorTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/deployment/TaskDeploymentDescriptorTest.java
@@ -27,7 +27,6 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.flink.api.common.ExecutionConfig;
-import org.apache.flink.api.common.ExecutionConfigTest;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.blob.BlobKey;
import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
@@ -37,6 +36,7 @@ import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
import org.apache.flink.runtime.operators.BatchTask;
import org.apache.flink.core.testutils.CommonTestUtils;
import org.apache.flink.util.SerializedValue;
+
import org.junit.Test;
public class TaskDeploymentDescriptorTest {
@@ -58,7 +58,7 @@ public class TaskDeploymentDescriptorTest {
final List<InputGateDeploymentDescriptor> inputGates = new ArrayList<InputGateDeploymentDescriptor>(0);
final List<BlobKey> requiredJars = new ArrayList<BlobKey>(0);
final List<URL> requiredClasspaths = new ArrayList<URL>(0);
- final SerializedValue<ExecutionConfig> executionConfig = ExecutionConfigTest.getSerializedConfig();
+ final SerializedValue<ExecutionConfig> executionConfig = new SerializedValue<>(new ExecutionConfig());
final TaskDeploymentDescriptor orig = new TaskDeploymentDescriptor(jobID, jobName, vertexID, execId,
executionConfig, taskName, indexInSubtaskGroup, currentNumberOfSubtasks, attemptNumber,
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-runtime/src/test/java/org/apache/flink/runtime/event/task/TaskEventTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/event/task/TaskEventTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/event/task/TaskEventTest.java
index d659b45..314471e 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/event/task/TaskEventTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/event/task/TaskEventTest.java
@@ -26,7 +26,8 @@ import java.io.IOException;
import java.util.Iterator;
import org.apache.flink.runtime.event.AbstractEvent;
-import org.apache.flink.core.testutils.CommonTestUtils;
+import org.apache.flink.util.InstantiationUtil;
+
import org.junit.Test;
/**
@@ -43,7 +44,7 @@ public class TaskEventTest {
try {
final IntegerTaskEvent orig = new IntegerTaskEvent(11);
- final IntegerTaskEvent copy = (IntegerTaskEvent) CommonTestUtils.createCopyWritable(orig);
+ final IntegerTaskEvent copy = InstantiationUtil.createCopyWritable(orig);
assertEquals(orig.getInteger(), copy.getInteger());
assertEquals(orig.hashCode(), copy.hashCode());
@@ -63,7 +64,7 @@ public class TaskEventTest {
try {
final StringTaskEvent orig = new StringTaskEvent("Test");
- final StringTaskEvent copy = (StringTaskEvent) CommonTestUtils.createCopyWritable(orig);
+ final StringTaskEvent copy = InstantiationUtil.createCopyWritable(orig);
assertEquals(orig.getString(), copy.getString());
assertEquals(orig.hashCode(), copy.hashCode());
@@ -85,7 +86,7 @@ public class TaskEventTest {
final EventList orig = new EventList();
orig.add(new StringTaskEvent("Test 2"));
orig.add(new IntegerTaskEvent(70));
- final EventList copy = (EventList) CommonTestUtils.createCopyWritable(orig);
+ final EventList copy = InstantiationUtil.createCopyWritable(orig);
assertEquals(orig.size(), copy.size());
final Iterator<AbstractEvent> origIt = orig.iterator();
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphConstructionTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphConstructionTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphConstructionTest.java
index 8eebe66..6740293 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphConstructionTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphConstructionTest.java
@@ -26,11 +26,13 @@ import static org.junit.Assert.fail;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
-import org.apache.flink.api.common.ExecutionConfigTest;
+import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.runtime.akka.AkkaUtils;
import org.apache.flink.runtime.executiongraph.restart.NoRestartStrategy;
import org.apache.flink.runtime.io.network.partition.ResultPartitionType;
import org.apache.flink.runtime.testingUtils.TestingUtils;
+import org.apache.flink.util.SerializedValue;
+
import org.junit.Test;
import org.mockito.Matchers;
@@ -61,7 +63,7 @@ import org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup;
* This class contains test concerning the correct conversion from {@link JobGraph} to {@link ExecutionGraph} objects.
*/
public class ExecutionGraphConstructionTest {
-
+
/**
* Creates a JobGraph of the following form:
*
@@ -76,7 +78,7 @@ public class ExecutionGraphConstructionTest {
* </pre>
*/
@Test
- public void testCreateSimpleGraphBipartite() {
+ public void testCreateSimpleGraphBipartite() throws Exception {
final JobID jobId = new JobID();
final String jobName = "Test Job Sample Name";
@@ -107,7 +109,7 @@ public class ExecutionGraphConstructionTest {
jobId,
jobName,
cfg,
- ExecutionConfigTest.getSerializedConfig(),
+ new SerializedValue<>(new ExecutionConfig()),
AkkaUtils.getDefaultTimeout(),
new NoRestartStrategy());
try {
@@ -122,7 +124,7 @@ public class ExecutionGraphConstructionTest {
}
@Test
- public void testAttachViaDataSets() {
+ public void testAttachViaDataSets() throws Exception {
final JobID jobId = new JobID();
final String jobName = "Test Job Sample Name";
final Configuration cfg = new Configuration();
@@ -151,8 +153,8 @@ public class ExecutionGraphConstructionTest {
TestingUtils.defaultExecutionContext(),
jobId,
jobName,
- cfg,
- ExecutionConfigTest.getSerializedConfig(),
+ cfg,
+ new SerializedValue<>(new ExecutionConfig()),
AkkaUtils.getDefaultTimeout(),
new NoRestartStrategy());
try {
@@ -190,7 +192,7 @@ public class ExecutionGraphConstructionTest {
}
@Test
- public void testAttachViaIds() {
+ public void testAttachViaIds() throws Exception {
final JobID jobId = new JobID();
final String jobName = "Test Job Sample Name";
final Configuration cfg = new Configuration();
@@ -219,8 +221,8 @@ public class ExecutionGraphConstructionTest {
TestingUtils.defaultExecutionContext(),
jobId,
jobName,
- cfg,
- ExecutionConfigTest.getSerializedConfig(),
+ cfg,
+ new SerializedValue<>(new ExecutionConfig()),
AkkaUtils.getDefaultTimeout(),
new NoRestartStrategy());
try {
@@ -459,7 +461,7 @@ public class ExecutionGraphConstructionTest {
}
@Test
- public void testCannotConnectMissingId() {
+ public void testCannotConnectMissingId() throws Exception {
final JobID jobId = new JobID();
final String jobName = "Test Job Sample Name";
final Configuration cfg = new Configuration();
@@ -474,8 +476,8 @@ public class ExecutionGraphConstructionTest {
TestingUtils.defaultExecutionContext(),
jobId,
jobName,
- cfg,
- ExecutionConfigTest.getSerializedConfig(),
+ cfg,
+ new SerializedValue<>(new ExecutionConfig()),
AkkaUtils.getDefaultTimeout(),
new NoRestartStrategy());
try {
@@ -502,7 +504,7 @@ public class ExecutionGraphConstructionTest {
}
@Test
- public void testCannotConnectWrongOrder() {
+ public void testCannotConnectWrongOrder() throws Exception {
final JobID jobId = new JobID();
final String jobName = "Test Job Sample Name";
final Configuration cfg = new Configuration();
@@ -531,8 +533,8 @@ public class ExecutionGraphConstructionTest {
TestingUtils.defaultExecutionContext(),
jobId,
jobName,
- cfg,
- ExecutionConfigTest.getSerializedConfig(),
+ cfg,
+ new SerializedValue<>(new ExecutionConfig()),
AkkaUtils.getDefaultTimeout(),
new NoRestartStrategy());
try {
@@ -593,8 +595,8 @@ public class ExecutionGraphConstructionTest {
TestingUtils.defaultExecutionContext(),
jobId,
jobName,
- cfg,
- ExecutionConfigTest.getSerializedConfig(),
+ cfg,
+ new SerializedValue<>(new ExecutionConfig()),
AkkaUtils.getDefaultTimeout(),
new NoRestartStrategy());
try {
@@ -639,8 +641,8 @@ public class ExecutionGraphConstructionTest {
TestingUtils.defaultExecutionContext(),
jobId,
jobName,
- cfg,
- ExecutionConfigTest.getSerializedConfig(),
+ cfg,
+ new SerializedValue<>(new ExecutionConfig()),
AkkaUtils.getDefaultTimeout(),
new NoRestartStrategy());
@@ -711,8 +713,8 @@ public class ExecutionGraphConstructionTest {
TestingUtils.defaultExecutionContext(),
jobId,
jobName,
- cfg,
- ExecutionConfigTest.getSerializedConfig(),
+ cfg,
+ new SerializedValue<>(new ExecutionConfig()),
AkkaUtils.getDefaultTimeout(),
new NoRestartStrategy());
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphDeploymentTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphDeploymentTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphDeploymentTest.java
index d126acb..a599f42 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphDeploymentTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphDeploymentTest.java
@@ -30,7 +30,7 @@ import java.util.Comparator;
import java.util.List;
import java.util.Map;
-import org.apache.flink.api.common.ExecutionConfigTest;
+import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.akka.AkkaUtils;
import org.apache.flink.runtime.deployment.InputGateDeploymentDescriptor;
@@ -47,6 +47,8 @@ import org.apache.flink.runtime.jobgraph.JobVertexID;
import org.apache.flink.runtime.jobmanager.scheduler.Scheduler;
import org.apache.flink.runtime.operators.BatchTask;
import org.apache.flink.runtime.testingUtils.TestingUtils;
+import org.apache.flink.util.SerializedValue;
+
import org.junit.Test;
public class ExecutionGraphDeploymentTest {
@@ -85,7 +87,7 @@ public class ExecutionGraphDeploymentTest {
jobId,
"some job",
new Configuration(),
- ExecutionConfigTest.getSerializedConfig(),
+ new SerializedValue<>(new ExecutionConfig()),
AkkaUtils.getDefaultTimeout(),
new NoRestartStrategy());
@@ -288,8 +290,8 @@ public class ExecutionGraphDeploymentTest {
TestingUtils.directExecutionContext(),
jobId,
"some job",
- new Configuration(),
- ExecutionConfigTest.getSerializedConfig(),
+ new Configuration(),
+ new SerializedValue<>(new ExecutionConfig()),
AkkaUtils.getDefaultTimeout(),
new NoRestartStrategy());
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphRestartTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphRestartTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphRestartTest.java
index 26ba04f..c12f346 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphRestartTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphRestartTest.java
@@ -20,7 +20,6 @@ package org.apache.flink.runtime.executiongraph;
import akka.dispatch.Futures;
import org.apache.flink.api.common.ExecutionConfig;
-import org.apache.flink.api.common.ExecutionConfigTest;
import org.apache.flink.api.common.JobID;
import org.apache.flink.api.common.restartstrategy.RestartStrategies;
import org.apache.flink.configuration.Configuration;
@@ -39,7 +38,9 @@ import org.apache.flink.runtime.jobmanager.scheduler.CoLocationConstraint;
import org.apache.flink.runtime.jobmanager.scheduler.Scheduler;
import org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup;
import org.apache.flink.runtime.testingUtils.TestingUtils;
+import org.apache.flink.util.SerializedValue;
import org.apache.flink.util.TestLogger;
+
import org.junit.Test;
import scala.concurrent.Await;
import scala.concurrent.Future;
@@ -86,7 +87,7 @@ public class ExecutionGraphRestartTest extends TestLogger {
new JobID(),
"test job",
new Configuration(),
- ExecutionConfigTest.getSerializedConfig(),
+ new SerializedValue<>(new ExecutionConfig()),
AkkaUtils.getDefaultTimeout(),
new NoRestartStrategy());
eg.attachJobGraph(jobGraph.getVerticesSortedTopologicallyFromSources());
@@ -141,7 +142,7 @@ public class ExecutionGraphRestartTest extends TestLogger {
new JobID(),
"test job",
new Configuration(),
- ExecutionConfigTest.getSerializedConfig(),
+ new SerializedValue<>(new ExecutionConfig()),
AkkaUtils.getDefaultTimeout(),
new FixedDelayRestartStrategy(1, 0L));
eg.attachJobGraph(jobGraph.getVerticesSortedTopologicallyFromSources());
@@ -197,7 +198,7 @@ public class ExecutionGraphRestartTest extends TestLogger {
new JobID(),
"Test job",
new Configuration(),
- ExecutionConfigTest.getSerializedConfig(),
+ new SerializedValue<>(new ExecutionConfig()),
AkkaUtils.getDefaultTimeout(),
new FixedDelayRestartStrategy(1, 1000));
eg.attachJobGraph(jobGraph.getVerticesSortedTopologicallyFromSources());
@@ -226,7 +227,7 @@ public class ExecutionGraphRestartTest extends TestLogger {
new JobID(),
"TestJob",
new Configuration(),
- ExecutionConfigTest.getSerializedConfig(),
+ new SerializedValue<>(new ExecutionConfig()),
AkkaUtils.getDefaultTimeout(),
// We want to manually control the restart and delay
new FixedDelayRestartStrategy(Integer.MAX_VALUE, Long.MAX_VALUE));
@@ -344,7 +345,7 @@ public class ExecutionGraphRestartTest extends TestLogger {
new JobID(),
"TestJob",
new Configuration(),
- ExecutionConfigTest.getSerializedConfig(),
+ new SerializedValue<>(new ExecutionConfig()),
AkkaUtils.getDefaultTimeout(),
// We want to manually control the restart and delay
new FixedDelayRestartStrategy(Integer.MAX_VALUE, Long.MAX_VALUE));
@@ -427,7 +428,7 @@ public class ExecutionGraphRestartTest extends TestLogger {
new JobID(),
"Test job",
new Configuration(),
- ExecutionConfigTest.getSerializedConfig(),
+ new SerializedValue<>(new ExecutionConfig()),
AkkaUtils.getDefaultTimeout(),
new FixedDelayRestartStrategy(1, 1000)));
@@ -498,7 +499,7 @@ public class ExecutionGraphRestartTest extends TestLogger {
new JobID(),
"test job",
new Configuration(),
- ExecutionConfigTest.getSerializedConfig(),
+ new SerializedValue<>(new ExecutionConfig()),
AkkaUtils.getDefaultTimeout(),
new FixedDelayRestartStrategy(1, 1000));
@@ -594,7 +595,7 @@ public class ExecutionGraphRestartTest extends TestLogger {
new JobID(),
"test job",
new Configuration(),
- ExecutionConfigTest.getSerializedConfig(),
+ new SerializedValue<>(new ExecutionConfig()),
AkkaUtils.getDefaultTimeout(),
new FixedDelayRestartStrategy(1, 1000000));
@@ -648,7 +649,7 @@ public class ExecutionGraphRestartTest extends TestLogger {
new JobID(),
"test job",
new Configuration(),
- ExecutionConfigTest.getSerializedConfig(),
+ new SerializedValue<>(new ExecutionConfig()),
AkkaUtils.getDefaultTimeout(),
new FixedDelayRestartStrategy(1, 1000000));
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphSignalsTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphSignalsTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphSignalsTest.java
index bcee5a1..59f2a9b 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphSignalsTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphSignalsTest.java
@@ -22,7 +22,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
-import org.apache.flink.api.common.ExecutionConfigTest;
+import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.api.common.JobID;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.StoppingException;
@@ -32,6 +32,8 @@ import org.apache.flink.runtime.jobgraph.JobVertex;
import org.apache.flink.runtime.jobgraph.DistributionPattern;
import org.apache.flink.runtime.jobgraph.JobStatus;
import org.apache.flink.runtime.testingUtils.TestingUtils;
+import org.apache.flink.util.SerializedValue;
+
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -128,7 +130,7 @@ public class ExecutionGraphSignalsTest {
jobId,
jobName,
cfg,
- ExecutionConfigTest.getSerializedConfig(),
+ new SerializedValue<>(new ExecutionConfig()),
AkkaUtils.getDefaultTimeout(),
new NoRestartStrategy());
eg.attachJobGraph(ordered);
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphTestUtils.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphTestUtils.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphTestUtils.java
index 92a7402..903d5f9 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphTestUtils.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphTestUtils.java
@@ -22,13 +22,11 @@ import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
-import java.io.IOException;
import java.lang.reflect.Field;
import java.net.InetAddress;
-import org.apache.flink.api.common.ExecutionConfigTest;
+import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.configuration.Configuration;
-import org.apache.flink.runtime.JobException;
import org.apache.flink.runtime.akka.AkkaUtils;
import org.apache.flink.runtime.clusterframework.types.ResourceID;
import org.apache.flink.runtime.deployment.TaskDeploymentDescriptor;
@@ -52,6 +50,7 @@ import org.apache.flink.runtime.messages.TaskMessages.FailIntermediateResultPart
import org.apache.flink.runtime.messages.TaskMessages.CancelTask;
import org.apache.flink.runtime.messages.TaskMessages.TaskOperationResult;
import org.apache.flink.runtime.testingUtils.TestingUtils;
+import org.apache.flink.util.SerializedValue;
import org.mockito.Matchers;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
@@ -168,7 +167,7 @@ public class ExecutionGraphTestUtils {
public static final String ERROR_MESSAGE = "test_failure_error_message";
- public static ExecutionJobVertex getExecutionVertex(JobVertexID id, ExecutionContext executionContext) throws JobException {
+ public static ExecutionJobVertex getExecutionVertex(JobVertexID id, ExecutionContext executionContext) throws Exception {
JobVertex ajv = new JobVertex("TestVertex", id);
ajv.setInvokableClass(mock(AbstractInvokable.class).getClass());
@@ -176,8 +175,8 @@ public class ExecutionGraphTestUtils {
executionContext,
new JobID(),
"test job",
- new Configuration(),
- ExecutionConfigTest.getSerializedConfig(),
+ new Configuration(),
+ new SerializedValue<>(new ExecutionConfig()),
AkkaUtils.getDefaultTimeout(),
new NoRestartStrategy());
@@ -198,7 +197,7 @@ public class ExecutionGraphTestUtils {
return ejv;
}
- public static ExecutionJobVertex getExecutionVertex(JobVertexID id) throws JobException, IOException {
+ public static ExecutionJobVertex getExecutionVertex(JobVertexID id) throws Exception {
return getExecutionVertex(id, TestingUtils.defaultExecutionContext());
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionStateProgressTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionStateProgressTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionStateProgressTest.java
index 9e4aa6d..f3743c1 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionStateProgressTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionStateProgressTest.java
@@ -22,9 +22,9 @@ import static org.apache.flink.runtime.executiongraph.ExecutionGraphTestUtils.*;
import static org.junit.Assert.*;
import static org.mockito.Mockito.mock;
-import java.util.Arrays;
+import java.util.Collections;
-import org.apache.flink.api.common.ExecutionConfigTest;
+import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.akka.AkkaUtils;
import org.apache.flink.runtime.executiongraph.restart.NoRestartStrategy;
@@ -35,6 +35,7 @@ import org.apache.flink.runtime.jobgraph.JobStatus;
import org.apache.flink.runtime.jobgraph.JobVertexID;
import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
import org.apache.flink.runtime.testingUtils.TestingUtils;
+import org.apache.flink.util.SerializedValue;
import org.junit.Test;
public class ExecutionStateProgressTest {
@@ -54,10 +55,10 @@ public class ExecutionStateProgressTest {
jid,
"test job",
new Configuration(),
- ExecutionConfigTest.getSerializedConfig(),
+ new SerializedValue<>(new ExecutionConfig()),
AkkaUtils.getDefaultTimeout(),
new NoRestartStrategy());
- graph.attachJobGraph(Arrays.asList(ajv));
+ graph.attachJobGraph(Collections.singletonList(ajv));
setGraphStatus(graph, JobStatus.RUNNING);
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/LocalInputSplitsTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/LocalInputSplitsTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/LocalInputSplitsTest.java
index d7ce0ba..f03370c 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/LocalInputSplitsTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/LocalInputSplitsTest.java
@@ -25,7 +25,7 @@ import static org.mockito.Mockito.when;
import java.net.InetAddress;
import java.util.concurrent.TimeUnit;
-import org.apache.flink.api.common.ExecutionConfigTest;
+import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.api.common.io.StrictlyLocalAssignment;
import org.apache.flink.core.io.InputSplitAssigner;
import org.apache.flink.core.io.InputSplitSource;
@@ -42,6 +42,7 @@ import org.apache.flink.runtime.jobgraph.JobGraph;
import org.apache.flink.runtime.jobmanager.scheduler.Scheduler;
import org.apache.flink.runtime.operators.testutils.DummyInvokable;
import org.apache.flink.runtime.testingUtils.TestingUtils;
+import org.apache.flink.util.SerializedValue;
import org.junit.Test;
import scala.concurrent.duration.FiniteDuration;
@@ -273,7 +274,7 @@ public class LocalInputSplitsTest {
jobGraph.getJobID(),
jobGraph.getName(),
jobGraph.getJobConfiguration(),
- ExecutionConfigTest.getSerializedConfig(),
+ new SerializedValue<>(new ExecutionConfig()),
TIMEOUT,
new NoRestartStrategy());
@@ -338,7 +339,7 @@ public class LocalInputSplitsTest {
jobGraph.getJobID(),
jobGraph.getName(),
jobGraph.getJobConfiguration(),
- ExecutionConfigTest.getSerializedConfig(),
+ new SerializedValue<>(new ExecutionConfig()),
TIMEOUT,
new NoRestartStrategy());
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/PointwisePatternTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/PointwisePatternTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/PointwisePatternTest.java
index 1b369db..0eb0607 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/PointwisePatternTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/PointwisePatternTest.java
@@ -22,10 +22,12 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
-import org.apache.flink.api.common.ExecutionConfigTest;
+import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.runtime.akka.AkkaUtils;
import org.apache.flink.runtime.executiongraph.restart.NoRestartStrategy;
import org.apache.flink.runtime.testingUtils.TestingUtils;
+import org.apache.flink.util.SerializedValue;
+
import org.junit.Test;
import java.util.ArrayList;
@@ -46,7 +48,7 @@ public class PointwisePatternTest {
private final Configuration cfg = new Configuration();
@Test
- public void testNToN() {
+ public void testNToN() throws Exception {
final int N = 23;
JobVertex v1 = new JobVertex("vertex1");
@@ -64,7 +66,7 @@ public class PointwisePatternTest {
jobId,
jobName,
cfg,
- ExecutionConfigTest.getSerializedConfig(),
+ new SerializedValue<>(new ExecutionConfig()),
AkkaUtils.getDefaultTimeout(),
new NoRestartStrategy());
try {
@@ -88,7 +90,7 @@ public class PointwisePatternTest {
}
@Test
- public void test2NToN() {
+ public void test2NToN() throws Exception {
final int N = 17;
JobVertex v1 = new JobVertex("vertex1");
@@ -106,7 +108,7 @@ public class PointwisePatternTest {
jobId,
jobName,
cfg,
- ExecutionConfigTest.getSerializedConfig(),
+ new SerializedValue<>(new ExecutionConfig()),
AkkaUtils.getDefaultTimeout(),
new NoRestartStrategy());
try {
@@ -131,7 +133,7 @@ public class PointwisePatternTest {
}
@Test
- public void test3NToN() {
+ public void test3NToN() throws Exception {
final int N = 17;
JobVertex v1 = new JobVertex("vertex1");
@@ -149,7 +151,7 @@ public class PointwisePatternTest {
jobId,
jobName,
cfg,
- ExecutionConfigTest.getSerializedConfig(),
+ new SerializedValue<>(new ExecutionConfig()),
AkkaUtils.getDefaultTimeout(),
new NoRestartStrategy());
try {
@@ -175,7 +177,7 @@ public class PointwisePatternTest {
}
@Test
- public void testNTo2N() {
+ public void testNTo2N() throws Exception {
final int N = 41;
JobVertex v1 = new JobVertex("vertex1");
@@ -193,7 +195,7 @@ public class PointwisePatternTest {
jobId,
jobName,
cfg,
- ExecutionConfigTest.getSerializedConfig(),
+ new SerializedValue<>(new ExecutionConfig()),
AkkaUtils.getDefaultTimeout(),
new NoRestartStrategy());
try {
@@ -217,7 +219,7 @@ public class PointwisePatternTest {
}
@Test
- public void testNTo7N() {
+ public void testNTo7N() throws Exception {
final int N = 11;
JobVertex v1 = new JobVertex("vertex1");
@@ -235,7 +237,7 @@ public class PointwisePatternTest {
jobId,
jobName,
cfg,
- ExecutionConfigTest.getSerializedConfig(),
+ new SerializedValue<>(new ExecutionConfig()),
AkkaUtils.getDefaultTimeout(),
new NoRestartStrategy());
try {
@@ -259,7 +261,7 @@ public class PointwisePatternTest {
}
@Test
- public void testLowHighIrregular() {
+ public void testLowHighIrregular() throws Exception {
testLowToHigh(3, 16);
testLowToHigh(19, 21);
testLowToHigh(15, 20);
@@ -267,14 +269,14 @@ public class PointwisePatternTest {
}
@Test
- public void testHighLowIrregular() {
+ public void testHighLowIrregular() throws Exception {
testHighToLow(16, 3);
testHighToLow(21, 19);
testHighToLow(20, 15);
testHighToLow(31, 11);
}
- private void testLowToHigh(int lowDop, int highDop) {
+ private void testLowToHigh(int lowDop, int highDop) throws Exception {
if (highDop < lowDop) {
throw new IllegalArgumentException();
}
@@ -297,7 +299,7 @@ public class PointwisePatternTest {
jobId,
jobName,
cfg,
- ExecutionConfigTest.getSerializedConfig(),
+ new SerializedValue<>(new ExecutionConfig()),
AkkaUtils.getDefaultTimeout(),
new NoRestartStrategy());
try {
@@ -327,7 +329,7 @@ public class PointwisePatternTest {
}
}
- private void testHighToLow(int highDop, int lowDop) {
+ private void testHighToLow(int highDop, int lowDop) throws Exception {
if (highDop < lowDop) {
throw new IllegalArgumentException();
}
@@ -350,7 +352,7 @@ public class PointwisePatternTest {
jobId,
jobName,
cfg,
- ExecutionConfigTest.getSerializedConfig(),
+ new SerializedValue<>(new ExecutionConfig()),
AkkaUtils.getDefaultTimeout(),
new NoRestartStrategy());
try {
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/TerminalStateDeadlockTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/TerminalStateDeadlockTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/TerminalStateDeadlockTest.java
index 8bc474b..2a690d9 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/TerminalStateDeadlockTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/TerminalStateDeadlockTest.java
@@ -18,7 +18,7 @@
package org.apache.flink.runtime.executiongraph;
-import org.apache.flink.api.common.ExecutionConfigTest;
+import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.api.common.JobID;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.clusterframework.types.ResourceID;
@@ -35,12 +35,14 @@ import org.apache.flink.runtime.jobgraph.JobStatus;
import org.apache.flink.runtime.jobgraph.JobVertexID;
import org.apache.flink.runtime.jobmanager.scheduler.Scheduler;
import org.apache.flink.runtime.operators.testutils.DummyInvokable;
-
import org.apache.flink.runtime.testingUtils.TestingUtils;
+import org.apache.flink.util.SerializedValue;
+
import org.junit.Test;
import scala.concurrent.duration.FiniteDuration;
+import java.io.IOException;
import java.lang.reflect.Field;
import java.net.InetAddress;
import java.util.Arrays;
@@ -105,9 +107,6 @@ public class TerminalStateDeadlockTest {
final JobID jobId = resource.getJobID();
final JobVertexID vid1 = new JobVertexID();
final JobVertexID vid2 = new JobVertexID();
-
-
- final Configuration jobConfig = new Configuration();
final List<JobVertex> vertices;
{
@@ -182,13 +181,13 @@ public class TerminalStateDeadlockTest {
private volatile boolean done;
- TestExecGraph(JobID jobId) {
+ TestExecGraph(JobID jobId) throws IOException {
super(
TestingUtils.defaultExecutionContext(),
jobId,
"test graph",
EMPTY_CONFIG,
- ExecutionConfigTest.getSerializedConfig(),
+ new SerializedValue<>(new ExecutionConfig()),
TIMEOUT,
new FixedDelayRestartStrategy(1, 0));
}
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/VertexLocationConstraintTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/VertexLocationConstraintTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/VertexLocationConstraintTest.java
index c483f41..4ee06b36 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/VertexLocationConstraintTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/VertexLocationConstraintTest.java
@@ -26,7 +26,7 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.concurrent.TimeUnit;
-import org.apache.flink.api.common.ExecutionConfigTest;
+import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.runtime.executiongraph.restart.NoRestartStrategy;
import org.apache.flink.runtime.clusterframework.types.ResourceID;
import org.apache.flink.runtime.instance.DummyActorGateway;
@@ -43,6 +43,7 @@ import org.apache.flink.runtime.jobmanager.scheduler.Scheduler;
import org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup;
import org.apache.flink.runtime.operators.testutils.DummyInvokable;
import org.apache.flink.runtime.testingUtils.TestingUtils;
+import org.apache.flink.util.SerializedValue;
import org.junit.Test;
import scala.concurrent.duration.FiniteDuration;
@@ -83,7 +84,7 @@ public class VertexLocationConstraintTest {
jg.getJobID(),
jg.getName(),
jg.getJobConfiguration(),
- ExecutionConfigTest.getSerializedConfig(),
+ new SerializedValue<>(new ExecutionConfig()),
timeout,
new NoRestartStrategy());
eg.attachJobGraph(Collections.singletonList(jobVertex));
@@ -156,7 +157,7 @@ public class VertexLocationConstraintTest {
jg.getJobID(),
jg.getName(),
jg.getJobConfiguration(),
- ExecutionConfigTest.getSerializedConfig(),
+ new SerializedValue<>(new ExecutionConfig()),
timeout,
new NoRestartStrategy());
eg.attachJobGraph(Collections.singletonList(jobVertex));
@@ -233,7 +234,7 @@ public class VertexLocationConstraintTest {
jg.getJobID(),
jg.getName(),
jg.getJobConfiguration(),
- ExecutionConfigTest.getSerializedConfig(),
+ new SerializedValue<>(new ExecutionConfig()),
timeout,
new NoRestartStrategy());
eg.attachJobGraph(Arrays.asList(jobVertex1, jobVertex2));
@@ -301,7 +302,7 @@ public class VertexLocationConstraintTest {
jg.getJobID(),
jg.getName(),
jg.getJobConfiguration(),
- ExecutionConfigTest.getSerializedConfig(),
+ new SerializedValue<>(new ExecutionConfig()),
timeout,
new NoRestartStrategy());
eg.attachJobGraph(Collections.singletonList(jobVertex));
@@ -371,7 +372,7 @@ public class VertexLocationConstraintTest {
jg.getJobID(),
jg.getName(),
jg.getJobConfiguration(),
- ExecutionConfigTest.getSerializedConfig(),
+ new SerializedValue<>(new ExecutionConfig()),
timeout,
new NoRestartStrategy());
eg.attachJobGraph(Arrays.asList(jobVertex1, jobVertex2));
@@ -411,7 +412,7 @@ public class VertexLocationConstraintTest {
jg.getJobID(),
jg.getName(),
jg.getJobConfiguration(),
- ExecutionConfigTest.getSerializedConfig(),
+ new SerializedValue<>(new ExecutionConfig()),
timeout,
new NoRestartStrategy());
eg.attachJobGraph(Collections.singletonList(vertex));
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/VertexSlotSharingTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/VertexSlotSharingTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/VertexSlotSharingTest.java
index 7a23e26..6314969 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/VertexSlotSharingTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/VertexSlotSharingTest.java
@@ -24,7 +24,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
-import org.apache.flink.api.common.ExecutionConfigTest;
+import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.akka.AkkaUtils;
import org.apache.flink.runtime.executiongraph.restart.NoRestartStrategy;
@@ -33,6 +33,7 @@ import org.apache.flink.runtime.jobgraph.DistributionPattern;
import org.apache.flink.api.common.JobID;
import org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup;
import org.apache.flink.runtime.testingUtils.TestingUtils;
+import org.apache.flink.util.SerializedValue;
import org.junit.Test;
public class VertexSlotSharingTest {
@@ -76,7 +77,7 @@ public class VertexSlotSharingTest {
new JobID(),
"test job",
new Configuration(),
- ExecutionConfigTest.getSerializedConfig(),
+ new SerializedValue<>(new ExecutionConfig()),
AkkaUtils.getDefaultTimeout(),
new NoRestartStrategy());
eg.attachJobGraph(vertices);
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-runtime/src/test/java/org/apache/flink/runtime/instance/InstanceConnectionInfoTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/instance/InstanceConnectionInfoTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/instance/InstanceConnectionInfoTest.java
index bc8cd63..3a9488d 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/instance/InstanceConnectionInfoTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/instance/InstanceConnectionInfoTest.java
@@ -25,7 +25,8 @@ import static org.junit.Assert.fail;
import java.net.InetAddress;
-import org.apache.flink.core.testutils.CommonTestUtils;
+import org.apache.flink.util.InstantiationUtil;
+
import org.junit.Assert;
import org.junit.Test;
@@ -97,10 +98,10 @@ public class InstanceConnectionInfoTest {
{
InstanceConnectionInfo original = new InstanceConnectionInfo(InetAddress.getByName("1.2.3.4"), 8888);
- InstanceConnectionInfo copy = CommonTestUtils.createCopyWritable(original);
+ InstanceConnectionInfo copy = InstantiationUtil.createCopyWritable(original);
assertEquals(original, copy);
- InstanceConnectionInfo serCopy = CommonTestUtils.createCopySerializable(original);
+ InstanceConnectionInfo serCopy = InstantiationUtil.clone(original);
assertEquals(original, serCopy);
}
@@ -109,10 +110,10 @@ public class InstanceConnectionInfoTest {
InstanceConnectionInfo original = new InstanceConnectionInfo(InetAddress.getByName("127.0.0.1"), 19871);
original.getFQDNHostname();
- InstanceConnectionInfo copy = CommonTestUtils.createCopyWritable(original);
+ InstanceConnectionInfo copy = InstantiationUtil.createCopyWritable(original);
assertEquals(original, copy);
- InstanceConnectionInfo serCopy = CommonTestUtils.createCopySerializable(original);
+ InstanceConnectionInfo serCopy = InstantiationUtil.clone(original);
assertEquals(original, serCopy);
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/MultiShotLatch.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/MultiShotLatch.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/MultiShotLatch.java
deleted file mode 100644
index 7c55419..0000000
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/MultiShotLatch.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.taskmanager;
-
-/**
- * Latch for synchronizing parts of code in tests. In contrast to
- * {@link org.apache.flink.runtime.taskmanager.OneShotLatch} this will reset the state once
- * {@link #await()} returns.
- *
- * <p>
- * A part of the code that should only run after other code calls {@link #await()}. The call
- * will only return once the other part is finished and calls {@link #trigger()}.
- */
-public final class MultiShotLatch {
-
- private final Object lock = new Object();
-
- private volatile boolean triggered;
-
- /**
- * Fires the latch. Code that is blocked on {@link #await()} will now return.
- */
- public void trigger() {
- synchronized (lock) {
- triggered = true;
- lock.notifyAll();
- }
- }
-
- /**
- * Waits until {@link #trigger())} is called.
- */
- public void await() throws InterruptedException {
- synchronized (lock) {
- while (!triggered) {
- lock.wait();
- }
- triggered = false;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/OneShotLatch.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/OneShotLatch.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/OneShotLatch.java
deleted file mode 100644
index 504ccca..0000000
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/OneShotLatch.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.taskmanager;
-
-/**
- * Latch for synchronizing parts of code in tests. Once the latch has fired once calls to
- * {@link #await()} will return immediately in the future.
- *
- * <p>
- * A part of the code that should only run after other code calls {@link #await()}. The call
- * will only return once the other part is finished and calls {@link #trigger()}.
- */
-public final class OneShotLatch {
-
- private final Object lock = new Object();
-
- private boolean triggered;
-
- /**
- * Fires the latch. Code that is blocked on {@link #await()} will now return.
- */
- public void trigger() {
- synchronized (lock) {
- triggered = true;
- lock.notifyAll();
- }
- }
-
- /**
- * Waits until {@link #trigger())} is called. Once {@code trigger()} has been called this
- * call will always return immediately.
- */
- public void await() throws InterruptedException {
- synchronized (lock) {
- while (!triggered) {
- lock.wait();
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskAsyncCallTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskAsyncCallTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskAsyncCallTest.java
index e1f551c..7b55987 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskAsyncCallTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskAsyncCallTest.java
@@ -18,9 +18,10 @@
package org.apache.flink.runtime.taskmanager;
-import org.apache.flink.api.common.ExecutionConfigTest;
+import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.api.common.JobID;
import org.apache.flink.configuration.Configuration;
+import org.apache.flink.core.testutils.OneShotLatch;
import org.apache.flink.metrics.groups.TaskMetricGroup;
import org.apache.flink.runtime.blob.BlobKey;
import org.apache.flink.runtime.broadcast.BroadcastVariableManager;
@@ -43,6 +44,7 @@ import org.apache.flink.runtime.jobgraph.tasks.StatefulTask;
import org.apache.flink.runtime.memory.MemoryManager;
import org.apache.flink.runtime.state.StateHandle;
+import org.apache.flink.util.SerializedValue;
import org.junit.Before;
import org.junit.Test;
@@ -137,7 +139,7 @@ public class TaskAsyncCallTest {
}
}
- private static Task createTask() {
+ private static Task createTask() throws Exception {
LibraryCacheManager libCache = mock(LibraryCacheManager.class);
when(libCache.getClassLoader(any(JobID.class))).thenReturn(ClassLoader.getSystemClassLoader());
@@ -150,7 +152,7 @@ public class TaskAsyncCallTest {
TaskDeploymentDescriptor tdd = new TaskDeploymentDescriptor(
new JobID(), "Job Name", new JobVertexID(), new ExecutionAttemptID(),
- ExecutionConfigTest.getSerializedConfig(),
+ new SerializedValue<>(new ExecutionConfig()),
"Test Task", 0, 1, 0,
new Configuration(), new Configuration(),
CheckpointsInOrderInvokable.class.getName(),
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerConfigurationTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerConfigurationTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerConfigurationTest.java
index f367799..335f788 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerConfigurationTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerConfigurationTest.java
@@ -33,6 +33,7 @@ import java.io.IOException;
import java.io.PrintWriter;
import java.lang.reflect.Field;
import java.net.*;
+import java.util.UUID;
import static org.junit.Assert.*;
@@ -109,7 +110,7 @@ public class TaskManagerConfigurationTest {
@Test
public void testDefaultFsParameterLoading() {
final File tmpDir = getTmpDir();
- final File confFile = new File(tmpDir.getAbsolutePath() + File.separator + CommonTestUtils.getRandomDirectoryName() + ".yaml");
+ final File confFile = new File(tmpDir, UUID.randomUUID().toString() + ".yaml");
try {
final URI defaultFS = new URI("otherFS", null, "localhost", 1234, null, null, null);
@@ -146,13 +147,7 @@ public class TaskManagerConfigurationTest {
try {
InetAddress localhostAddress = InetAddress.getByName(hostname);
server = new ServerSocket(0, 50, localhostAddress);
- }
- catch (UnknownHostException e) {
- // may happen if disconnected. skip test.
- System.err.println("Skipping 'testNetworkInterfaceSelection' test.");
- return;
- }
- catch (IOException e) {
+ } catch (IOException e) {
// may happen in certain test setups, skip test.
System.err.println("Skipping 'testNetworkInterfaceSelection' test.");
return;
@@ -181,10 +176,8 @@ public class TaskManagerConfigurationTest {
}
private File getTmpDir() {
- File tmpDir = new File(CommonTestUtils.getTempDir() + File.separator
- + CommonTestUtils.getRandomDirectoryName() + File.separator);
- tmpDir.mkdirs();
-
+ File tmpDir = new File(CommonTestUtils.getTempDir(), UUID.randomUUID().toString());
+ assertTrue("could not create temp directory", tmpDir.mkdirs());
return tmpDir;
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerTest.java
index b3ad589..ce88c09 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerTest.java
@@ -24,7 +24,6 @@ import akka.actor.Props;
import akka.japi.Creator;
import akka.testkit.JavaTestKit;
import org.apache.flink.api.common.ExecutionConfig;
-import org.apache.flink.api.common.ExecutionConfigTest;
import org.apache.flink.api.common.JobID;
import org.apache.flink.configuration.ConfigConstants;
import org.apache.flink.configuration.Configuration;
@@ -160,7 +159,7 @@ public class TaskManagerTest extends TestLogger {
final JobID jid = new JobID();
final JobVertexID vid = new JobVertexID();
final ExecutionAttemptID eid = new ExecutionAttemptID();
- final SerializedValue<ExecutionConfig> executionConfig = ExecutionConfigTest.getSerializedConfig();
+ final SerializedValue<ExecutionConfig> executionConfig = new SerializedValue<>(new ExecutionConfig());
final TaskDeploymentDescriptor tdd = new TaskDeploymentDescriptor(jid, "TestJob", vid, eid, executionConfig,
"TestTask", 2, 7, 0, new Configuration(), new Configuration(),
@@ -263,15 +262,19 @@ public class TaskManagerTest extends TestLogger {
final ExecutionAttemptID eid1 = new ExecutionAttemptID();
final ExecutionAttemptID eid2 = new ExecutionAttemptID();
- final TaskDeploymentDescriptor tdd1 = new TaskDeploymentDescriptor(jid1, "TestJob1", vid1, eid1,
- ExecutionConfigTest.getSerializedConfig(), "TestTask1", 1, 5, 0,
+ final TaskDeploymentDescriptor tdd1 = new TaskDeploymentDescriptor(
+ jid1, "TestJob1", vid1, eid1,
+ new SerializedValue<>(new ExecutionConfig()),
+ "TestTask1", 1, 5, 0,
new Configuration(), new Configuration(), TestInvokableBlockingCancelable.class.getName(),
Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
Collections.<InputGateDeploymentDescriptor>emptyList(),
new ArrayList<BlobKey>(), Collections.<URL>emptyList(), 0);
- final TaskDeploymentDescriptor tdd2 = new TaskDeploymentDescriptor(jid2, "TestJob2", vid2, eid2,
- ExecutionConfigTest.getSerializedConfig(), "TestTask2", 2, 7, 0,
+ final TaskDeploymentDescriptor tdd2 = new TaskDeploymentDescriptor(
+ jid2, "TestJob2", vid2, eid2,
+ new SerializedValue<>(new ExecutionConfig()),
+ "TestTask2", 2, 7, 0,
new Configuration(), new Configuration(), TestInvokableBlockingCancelable.class.getName(),
Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
Collections.<InputGateDeploymentDescriptor>emptyList(),
@@ -367,7 +370,7 @@ public class TaskManagerTest extends TestLogger {
}
@Test
- public void testJobSubmissionAndStop() {
+ public void testJobSubmissionAndStop() throws Exception {
new JavaTestKit(system){{
ActorGateway jobManager = null;
@@ -397,7 +400,7 @@ public class TaskManagerTest extends TestLogger {
final ExecutionAttemptID eid1 = new ExecutionAttemptID();
final ExecutionAttemptID eid2 = new ExecutionAttemptID();
- final SerializedValue<ExecutionConfig> executionConfig = ExecutionConfigTest.getSerializedConfig();
+ final SerializedValue<ExecutionConfig> executionConfig = new SerializedValue<>(new ExecutionConfig());
final TaskDeploymentDescriptor tdd1 = new TaskDeploymentDescriptor(jid1, "TestJob", vid1, eid1, executionConfig,
"TestTask1", 1, 5, 0, new Configuration(), new Configuration(), StoppableInvokable.class.getName(),
@@ -525,15 +528,19 @@ public class TaskManagerTest extends TestLogger {
final ExecutionAttemptID eid1 = new ExecutionAttemptID();
final ExecutionAttemptID eid2 = new ExecutionAttemptID();
- final TaskDeploymentDescriptor tdd1 = new TaskDeploymentDescriptor(jid, "TestJob", vid1, eid1,
- ExecutionConfigTest.getSerializedConfig(), "Sender", 0, 1, 0,
+ final TaskDeploymentDescriptor tdd1 = new TaskDeploymentDescriptor(
+ jid, "TestJob", vid1, eid1,
+ new SerializedValue<>(new ExecutionConfig()),
+ "Sender", 0, 1, 0,
new Configuration(), new Configuration(), Tasks.Sender.class.getName(),
Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
Collections.<InputGateDeploymentDescriptor>emptyList(),
new ArrayList<BlobKey>(), Collections.<URL>emptyList(), 0);
- final TaskDeploymentDescriptor tdd2 = new TaskDeploymentDescriptor(jid, "TestJob", vid2, eid2,
- ExecutionConfigTest.getSerializedConfig(), "Receiver", 2, 7, 0,
+ final TaskDeploymentDescriptor tdd2 = new TaskDeploymentDescriptor(
+ jid, "TestJob", vid2, eid2,
+ new SerializedValue<>(new ExecutionConfig()),
+ "Receiver", 2, 7, 0,
new Configuration(), new Configuration(), Tasks.Receiver.class.getName(),
Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
Collections.<InputGateDeploymentDescriptor>emptyList(),
@@ -626,14 +633,18 @@ public class TaskManagerTest extends TestLogger {
}
);
- final TaskDeploymentDescriptor tdd1 = new TaskDeploymentDescriptor(jid, "TestJob", vid1, eid1,
- ExecutionConfigTest.getSerializedConfig(), "Sender", 0, 1, 0,
+ final TaskDeploymentDescriptor tdd1 = new TaskDeploymentDescriptor(
+ jid, "TestJob", vid1, eid1,
+ new SerializedValue<>(new ExecutionConfig()),
+ "Sender", 0, 1, 0,
new Configuration(), new Configuration(), Tasks.Sender.class.getName(),
irpdd, Collections.<InputGateDeploymentDescriptor>emptyList(), new ArrayList<BlobKey>(),
Collections.<URL>emptyList(), 0);
- final TaskDeploymentDescriptor tdd2 = new TaskDeploymentDescriptor(jid, "TestJob", vid2, eid2,
- ExecutionConfigTest.getSerializedConfig(), "Receiver", 2, 7, 0,
+ final TaskDeploymentDescriptor tdd2 = new TaskDeploymentDescriptor(
+ jid, "TestJob", vid2, eid2,
+ new SerializedValue<>(new ExecutionConfig()),
+ "Receiver", 2, 7, 0,
new Configuration(), new Configuration(), Tasks.Receiver.class.getName(),
Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
Collections.singletonList(ircdd),
@@ -767,14 +778,18 @@ public class TaskManagerTest extends TestLogger {
}
);
- final TaskDeploymentDescriptor tdd1 = new TaskDeploymentDescriptor(jid, "TestJob", vid1, eid1,
- ExecutionConfigTest.getSerializedConfig(), "Sender", 0, 1, 0,
+ final TaskDeploymentDescriptor tdd1 = new TaskDeploymentDescriptor(
+ jid, "TestJob", vid1, eid1,
+ new SerializedValue<>(new ExecutionConfig()),
+ "Sender", 0, 1, 0,
new Configuration(), new Configuration(), Tasks.Sender.class.getName(),
irpdd, Collections.<InputGateDeploymentDescriptor>emptyList(),
new ArrayList<BlobKey>(), Collections.<URL>emptyList(), 0);
- final TaskDeploymentDescriptor tdd2 = new TaskDeploymentDescriptor(jid, "TestJob", vid2, eid2,
- ExecutionConfigTest.getSerializedConfig(), "Receiver", 2, 7, 0,
+ final TaskDeploymentDescriptor tdd2 = new TaskDeploymentDescriptor(
+ jid, "TestJob", vid2, eid2,
+ new SerializedValue<>(new ExecutionConfig()),
+ "Receiver", 2, 7, 0,
new Configuration(), new Configuration(), Tasks.BlockingReceiver.class.getName(),
Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
Collections.singletonList(ircdd),
@@ -913,7 +928,8 @@ public class TaskManagerTest extends TestLogger {
final TaskDeploymentDescriptor tdd = new TaskDeploymentDescriptor(
jid, "TestJob", vid, eid,
- ExecutionConfigTest.getSerializedConfig(), "Receiver", 0, 1, 0,
+ new SerializedValue<>(new ExecutionConfig()),
+ "Receiver", 0, 1, 0,
new Configuration(), new Configuration(),
Tasks.AgnosticReceiver.class.getName(),
Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
@@ -1007,7 +1023,9 @@ public class TaskManagerTest extends TestLogger {
new InputGateDeploymentDescriptor(resultId, 0, icdd);
final TaskDeploymentDescriptor tdd = new TaskDeploymentDescriptor(
- jid, "TestJob", vid, eid, ExecutionConfigTest.getSerializedConfig(), "Receiver", 0, 1, 0,
+ jid, "TestJob", vid, eid,
+ new SerializedValue<>(new ExecutionConfig()),
+ "Receiver", 0, 1, 0,
new Configuration(), new Configuration(),
Tasks.AgnosticReceiver.class.getName(),
Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
@@ -1084,7 +1102,7 @@ public class TaskManagerTest extends TestLogger {
"Job",
new JobVertexID(),
new ExecutionAttemptID(),
- ExecutionConfigTest.getSerializedConfig(),
+ new SerializedValue<>(new ExecutionConfig()),
"Task",
0,
1,
[13/15] flink git commit: [FLINK-3995] [build] flink-test-utils also
contains the streaming test utilities.
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/b9f42e91/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/graph/StreamGraphGeneratorTest.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/graph/StreamGraphGeneratorTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/graph/StreamGraphGeneratorTest.java
index 46a985c..a4ee18e 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/graph/StreamGraphGeneratorTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/graph/StreamGraphGeneratorTest.java
@@ -25,6 +25,7 @@ import org.apache.flink.streaming.api.datastream.ConnectedStreams;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.functions.sink.DiscardingSink;
import org.apache.flink.streaming.api.operators.AbstractStreamOperator;
import org.apache.flink.streaming.api.operators.OneInputStreamOperator;
import org.apache.flink.streaming.api.operators.Output;
@@ -39,7 +40,6 @@ import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
import org.apache.flink.streaming.runtime.tasks.StreamTask;
import org.apache.flink.streaming.util.EvenOddOutputSelector;
import org.apache.flink.streaming.util.NoOpIntMap;
-import org.apache.flink.streaming.util.NoOpSink;
import org.junit.Test;
@@ -49,8 +49,7 @@ import static org.junit.Assert.assertTrue;
/**
* Tests for {@link StreamGraphGenerator}. This only tests correct translation of split/select,
* union, partitioning since the other translation routines are tested already in operation
- * specific tests, for example in {@link org.apache.flink.streaming.api.IterateTest} for
- * iterations.
+ * specific tests.
*/
public class StreamGraphGeneratorTest {
@@ -77,7 +76,7 @@ public class StreamGraphGeneratorTest {
.broadcast()
.map(new NoOpIntMap());
- broadcastMap.addSink(new NoOpSink<Integer>());
+ broadcastMap.addSink(new DiscardingSink<Integer>());
// verify that partitioning is preserved across union and split/select
EvenOddOutputSelector selector1 = new EvenOddOutputSelector();
@@ -113,7 +112,7 @@ public class StreamGraphGeneratorTest {
SingleOutputStreamOperator<Integer> unionedMap = map1.union(map2).union(map3)
.map(new NoOpIntMap());
- unionedMap.addSink(new NoOpSink<Integer>());
+ unionedMap.addSink(new DiscardingSink<Integer>());
StreamGraph graph = env.getStreamGraph();
@@ -169,7 +168,7 @@ public class StreamGraphGeneratorTest {
.select("foo")
.map(new NoOpIntMap());
- unionedMap.addSink(new NoOpSink<Integer>());
+ unionedMap.addSink(new DiscardingSink<Integer>());
StreamGraph graph = env.getStreamGraph();
@@ -207,9 +206,9 @@ public class StreamGraphGeneratorTest {
BasicTypeInfo.INT_TYPE_INFO,
outputTypeConfigurableOperation);
- result.addSink(new NoOpSink<Integer>());
+ result.addSink(new DiscardingSink<Integer>());
- StreamGraph graph = env.getStreamGraph();
+ env.getStreamGraph();
assertEquals(BasicTypeInfo.INT_TYPE_INFO, outputTypeConfigurableOperation.getTypeInformation());
}
@@ -230,9 +229,9 @@ public class StreamGraphGeneratorTest {
BasicTypeInfo.INT_TYPE_INFO,
outputTypeConfigurableOperation);
- result.addSink(new NoOpSink<Integer>());
+ result.addSink(new DiscardingSink<Integer>());
- StreamGraph graph = env.getStreamGraph();
+ env.getStreamGraph();
assertEquals(BasicTypeInfo.INT_TYPE_INFO, outputTypeConfigurableOperation.getTypeInformation());
}
http://git-wip-us.apache.org/repos/asf/flink/blob/b9f42e91/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/StreamProjectTest.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/StreamProjectTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/StreamProjectTest.java
index 14abd18..51e995b 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/StreamProjectTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/StreamProjectTest.java
@@ -17,29 +17,19 @@
package org.apache.flink.streaming.api.operators;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
-import java.util.HashSet;
import java.util.concurrent.ConcurrentLinkedQueue;
import org.apache.flink.api.common.ExecutionConfig;
-import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.api.java.tuple.Tuple;
-import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.api.java.tuple.Tuple5;
import org.apache.flink.api.java.typeutils.TupleTypeInfo;
import org.apache.flink.api.java.typeutils.TypeExtractor;
import org.apache.flink.api.java.typeutils.runtime.TupleSerializer;
import org.apache.flink.streaming.api.datastream.StreamProjection;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.api.functions.sink.SinkFunction;
import org.apache.flink.streaming.api.watermark.Watermark;
import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
import org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness;
-import org.apache.flink.streaming.util.StreamingMultipleProgramsTestBase;
import org.apache.flink.streaming.util.TestHarnessUtil;
import org.junit.Test;
@@ -52,7 +42,7 @@ import org.junit.Test;
* <li>Watermarks are correctly forwarded</li>
* </ul>
*/
-public class StreamProjectTest extends StreamingMultipleProgramsTestBase {
+public class StreamProjectTest {
@Test
public void testProject() throws Exception {
@@ -91,47 +81,4 @@ public class StreamProjectTest extends StreamingMultipleProgramsTestBase {
TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
}
-
-
- // tests using projection from the API without explicitly specifying the types
- private static HashSet<Tuple2<Long, Double>> expected = new HashSet<Tuple2<Long, Double>>();
- private static HashSet<Tuple2<Long, Double>> actual = new HashSet<Tuple2<Long, Double>>();
-
- @Test
- public void APIWithoutTypesTest() {
-
- for (Long i = 1L; i < 11L; i++) {
- expected.add(new Tuple2<Long, Double>(i, i.doubleValue()));
- }
-
- StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
- env.setParallelism(1);
-
- env.generateSequence(1, 10).map(new MapFunction<Long, Tuple3<Long, Character, Double>>() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public Tuple3<Long, Character, Double> map(Long value) throws Exception {
- return new Tuple3<Long, Character, Double>(value, 'c', value.doubleValue());
- }
- })
- .project(0, 2)
- .addSink(new SinkFunction<Tuple>() {
- private static final long serialVersionUID = 1L;
-
- @Override
- @SuppressWarnings("unchecked")
- public void invoke(Tuple value) throws Exception {
- actual.add( (Tuple2<Long,Double>) value);
- }
- });
-
- try {
- env.execute();
- } catch (Exception e) {
- fail(e.getMessage());
- }
-
- assertEquals(expected, actual);
- }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/b9f42e91/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/co/CoGroupedReduceTest.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/co/CoGroupedReduceTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/co/CoGroupedReduceTest.java
deleted file mode 100644
index 39e85e9..0000000
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/co/CoGroupedReduceTest.java
+++ /dev/null
@@ -1,125 +0,0 @@
-///*
-// * Licensed to the Apache Software Foundation (ASF) under one or more
-// * contributor license agreements. See the NOTICE file distributed with
-// * this work for additional information regarding copyright ownership.
-// * The ASF licenses this file to You under the Apache License, Version 2.0
-// * (the "License"); you may not use this file except in compliance with
-// * the License. You may obtain a copy of the License at
-// *
-// * http://www.apache.org/licenses/LICENSE-2.0
-// *
-// * Unless required by applicable law or agreed to in writing, software
-// * distributed under the License is distributed on an "AS IS" BASIS,
-// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// * See the License for the specific language governing permissions and
-// * limitations under the License.
-// */
-//
-//package org.apache.flink.streaming.api.operators.co;
-//
-//import static org.junit.Assert.assertEquals;
-//
-//import java.util.Arrays;
-//import java.util.List;
-//
-//import org.apache.flink.api.java.functions.KeySelector;
-//import org.apache.flink.api.java.tuple.Tuple2;
-//import org.apache.flink.api.java.tuple.Tuple3;
-//import org.apache.flink.streaming.api.functions.co.CoReduceFunction;
-//import org.apache.flink.streaming.api.operators.co.CoStreamGroupedReduce;
-//import org.apache.flink.streaming.util.MockCoContext;
-//import org.junit.Test;
-//
-//public class CoGroupedReduceTest {
-//
-// private final static class MyCoReduceFunction implements
-// CoReduceFunction<Tuple3<String, String, String>, Tuple2<Integer, Integer>, String> {
-// private static final long serialVersionUID = 1L;
-//
-// @Override
-// public Tuple3<String, String, String> reduce1(Tuple3<String, String, String> value1,
-// Tuple3<String, String, String> value2) {
-// return new Tuple3<String, String, String>(value1.f0, value1.f1 + value2.f1, value1.f2);
-// }
-//
-// @Override
-// public Tuple2<Integer, Integer> reduce2(Tuple2<Integer, Integer> value1,
-// Tuple2<Integer, Integer> value2) {
-// return new Tuple2<Integer, Integer>(value1.f0, value1.f1 + value2.f1);
-// }
-//
-// @Override
-// public String map1(Tuple3<String, String, String> value) {
-// return value.f1;
-// }
-//
-// @Override
-// public String map2(Tuple2<Integer, Integer> value) {
-// return value.f1.toString();
-// }
-// }
-//
-// @SuppressWarnings("unchecked")
-// @Test
-// public void coGroupedReduceTest() {
-// Tuple3<String, String, String> word1 = new Tuple3<String, String, String>("a", "word1", "b");
-// Tuple3<String, String, String> word2 = new Tuple3<String, String, String>("b", "word2", "a");
-// Tuple3<String, String, String> word3 = new Tuple3<String, String, String>("a", "word3", "a");
-// Tuple2<Integer, Integer> int1 = new Tuple2<Integer, Integer>(2, 1);
-// Tuple2<Integer, Integer> int2 = new Tuple2<Integer, Integer>(1, 2);
-// Tuple2<Integer, Integer> int3 = new Tuple2<Integer, Integer>(0, 3);
-// Tuple2<Integer, Integer> int4 = new Tuple2<Integer, Integer>(2, 4);
-// Tuple2<Integer, Integer> int5 = new Tuple2<Integer, Integer>(1, 5);
-//
-// KeySelector<Tuple3<String, String, String>, ?> keySelector0 = new KeySelector<Tuple3<String, String, String>, String>() {
-//
-// private static final long serialVersionUID = 1L;
-//
-// @Override
-// public String getKey(Tuple3<String, String, String> value) throws Exception {
-// return value.f0;
-// }
-// };
-//
-// KeySelector<Tuple2<Integer, Integer>, ?> keySelector1 = new KeySelector<Tuple2<Integer, Integer>, Integer>() {
-//
-// private static final long serialVersionUID = 1L;
-//
-// @Override
-// public Integer getKey(Tuple2<Integer, Integer> value) throws Exception {
-// return value.f0;
-// }
-// };
-//
-// KeySelector<Tuple3<String, String, String>, ?> keySelector2 = new KeySelector<Tuple3<String, String, String>, String>() {
-//
-// private static final long serialVersionUID = 1L;
-//
-// @Override
-// public String getKey(Tuple3<String, String, String> value) throws Exception {
-// return value.f2;
-// }
-// };
-//
-// CoStreamGroupedReduce<Tuple3<String, String, String>, Tuple2<Integer, Integer>, String> invokable = new CoStreamGroupedReduce<Tuple3<String, String, String>, Tuple2<Integer, Integer>, String>(
-// new MyCoReduceFunction(), keySelector0, keySelector1);
-//
-// List<String> expected = Arrays.asList("word1", "1", "word2", "2", "word1word3", "3", "5",
-// "7");
-//
-// List<String> actualList = MockCoContext.createAndExecute(invokable,
-// Arrays.asList(word1, word2, word3), Arrays.asList(int1, int2, int3, int4, int5));
-//
-// assertEquals(expected, actualList);
-//
-// invokable = new CoStreamGroupedReduce<Tuple3<String, String, String>, Tuple2<Integer, Integer>, String>(
-// new MyCoReduceFunction(), keySelector2, keySelector1);
-//
-// expected = Arrays.asList("word1", "1", "word2", "2", "word2word3", "3", "5", "7");
-//
-// actualList = MockCoContext.createAndExecute(invokable, Arrays.asList(word1, word2, word3),
-// Arrays.asList(int1, int2, int3, int4, int5));
-//
-// assertEquals(expected, actualList);
-// }
-//}
http://git-wip-us.apache.org/repos/asf/flink/blob/b9f42e91/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/co/CoWindowTest.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/co/CoWindowTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/co/CoWindowTest.java
deleted file mode 100644
index 130842e..0000000
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/co/CoWindowTest.java
+++ /dev/null
@@ -1,182 +0,0 @@
-///*
-// * Licensed to the Apache Software Foundation (ASF) under one or more
-// * contributor license agreements. See the NOTICE file distributed with
-// * this work for additional information regarding copyright ownership.
-// * The ASF licenses this file to You under the Apache License, Version 2.0
-// * (the "License"); you may not use this file except in compliance with
-// * the License. You may obtain a copy of the License at
-// *
-// * http://www.apache.org/licenses/LICENSE-2.0
-// *
-// * Unless required by applicable law or agreed to in writing, software
-// * distributed under the License is distributed on an "AS IS" BASIS,
-// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// * See the License for the specific language governing permissions and
-// * limitations under the License.
-// */
-//
-//package org.apache.flink.streaming.api.operators.co;
-//
-//import static org.junit.Assert.assertEquals;
-//
-//import java.util.ArrayList;
-//import java.util.HashSet;
-//import java.util.List;
-//import java.util.Set;
-//
-//import org.apache.flink.api.java.tuple.Tuple2;
-//import org.apache.flink.streaming.api.functions.co.CoWindowFunction;
-//import org.apache.flink.streaming.api.operators.co.CoStreamWindow;
-//import org.apache.flink.streaming.api.windowing.helper.Timestamp;
-//import org.apache.flink.streaming.api.windowing.helper.TimestampWrapper;
-//import org.apache.flink.streaming.util.MockCoContext;
-//import org.apache.flink.util.Collector;
-//import org.junit.Test;
-//
-//public class CoWindowTest {
-//
-// public static final class MyCoGroup1 implements CoWindowFunction<Integer, Integer, Integer> {
-//
-// private static final long serialVersionUID = 1L;
-//
-// @SuppressWarnings("unused")
-// @Override
-// public void coWindow(List<Integer> first, List<Integer> second, Collector<Integer> out)
-// throws Exception {
-// Integer count1 = 0;
-// for (Integer i : first) {
-// count1++;
-// }
-// Integer count2 = 0;
-// for (Integer i : second) {
-// count2++;
-// }
-// out.collect(count1);
-// out.collect(count2);
-//
-// }
-//
-// }
-//
-// public static final class MyCoGroup2 implements
-// CoWindowFunction<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Integer> {
-//
-// private static final long serialVersionUID = 1L;
-//
-// @Override
-// public void coWindow(List<Tuple2<Integer, Integer>> first,
-// List<Tuple2<Integer, Integer>> second, Collector<Integer> out) throws Exception {
-//
-// Set<Integer> firstElements = new HashSet<Integer>();
-// for (Tuple2<Integer, Integer> value : first) {
-// firstElements.add(value.f1);
-// }
-// for (Tuple2<Integer, Integer> value : second) {
-// if (firstElements.contains(value.f1)) {
-// out.collect(value.f1);
-// }
-// }
-//
-// }
-//
-// }
-//
-// private static final class MyTS1 implements Timestamp<Integer> {
-//
-// private static final long serialVersionUID = 1L;
-//
-// @Override
-// public long getTimestamp(Integer value) {
-// return value;
-// }
-//
-// }
-//
-// private static final class MyTS2 implements Timestamp<Tuple2<Integer, Integer>> {
-//
-// private static final long serialVersionUID = 1L;
-//
-// @Override
-// public long getTimestamp(Tuple2<Integer, Integer> value) {
-// return value.f0;
-// }
-//
-// }
-//
-// @Test
-// public void coWindowGroupReduceTest2() throws Exception {
-//
-// CoStreamWindow<Integer, Integer, Integer> invokable1 = new CoStreamWindow<Integer, Integer, Integer>(
-// new MyCoGroup1(), 2, 1, new TimestampWrapper<Integer>(new MyTS1(), 1),
-// new TimestampWrapper<Integer>(new MyTS1(), 1));
-//
-// // Windowsize 2, slide 1
-// // 1,2|2,3|3,4|4,5
-//
-// List<Integer> input11 = new ArrayList<Integer>();
-// input11.add(1);
-// input11.add(1);
-// input11.add(2);
-// input11.add(3);
-// input11.add(3);
-//
-// List<Integer> input12 = new ArrayList<Integer>();
-// input12.add(1);
-// input12.add(2);
-// input12.add(3);
-// input12.add(3);
-// input12.add(5);
-//
-// // Windows: (1,1,2)(1,1,2)|(2,3,3)(2,3,3)|(3,3)(3,3)|(5)(5)
-// // expected output: 3,2|3,3|2,2|0,1
-//
-// List<Integer> expected1 = new ArrayList<Integer>();
-// expected1.add(3);
-// expected1.add(2);
-// expected1.add(3);
-// expected1.add(3);
-// expected1.add(2);
-// expected1.add(2);
-// expected1.add(0);
-// expected1.add(1);
-//
-// List<Integer> actual1 = MockCoContext.createAndExecute(invokable1, input11, input12);
-// assertEquals(expected1, actual1);
-//
-// CoStreamWindow<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Integer> invokable2 = new CoStreamWindow<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Integer>(
-// new MyCoGroup2(), 2, 3, new TimestampWrapper<Tuple2<Integer, Integer>>(new MyTS2(),
-// 1), new TimestampWrapper<Tuple2<Integer, Integer>>(new MyTS2(), 1));
-//
-// // WindowSize 2, slide 3
-// // 1,2|4,5|7,8|
-//
-// List<Tuple2<Integer, Integer>> input21 = new ArrayList<Tuple2<Integer, Integer>>();
-// input21.add(new Tuple2<Integer, Integer>(1, 1));
-// input21.add(new Tuple2<Integer, Integer>(1, 2));
-// input21.add(new Tuple2<Integer, Integer>(2, 3));
-// input21.add(new Tuple2<Integer, Integer>(3, 4));
-// input21.add(new Tuple2<Integer, Integer>(3, 5));
-// input21.add(new Tuple2<Integer, Integer>(4, 6));
-// input21.add(new Tuple2<Integer, Integer>(4, 7));
-// input21.add(new Tuple2<Integer, Integer>(5, 8));
-//
-// List<Tuple2<Integer, Integer>> input22 = new ArrayList<Tuple2<Integer, Integer>>();
-// input22.add(new Tuple2<Integer, Integer>(1, 1));
-// input22.add(new Tuple2<Integer, Integer>(2, 0));
-// input22.add(new Tuple2<Integer, Integer>(2, 2));
-// input22.add(new Tuple2<Integer, Integer>(3, 9));
-// input22.add(new Tuple2<Integer, Integer>(3, 4));
-// input22.add(new Tuple2<Integer, Integer>(4, 10));
-// input22.add(new Tuple2<Integer, Integer>(5, 8));
-// input22.add(new Tuple2<Integer, Integer>(5, 7));
-//
-// List<Integer> expected2 = new ArrayList<Integer>();
-// expected2.add(1);
-// expected2.add(2);
-// expected2.add(8);
-// expected2.add(7);
-//
-// List<Integer> actual2 = MockCoContext.createAndExecute(invokable2, input21, input22);
-// assertEquals(expected2, actual2);
-// }
-//}
http://git-wip-us.apache.org/repos/asf/flink/blob/b9f42e91/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/co/SelfConnectionTest.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/co/SelfConnectionTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/co/SelfConnectionTest.java
deleted file mode 100644
index 8f04d41..0000000
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/co/SelfConnectionTest.java
+++ /dev/null
@@ -1,180 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.operators.co;
-
-import static org.junit.Assert.assertEquals;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.flink.api.common.functions.FlatMapFunction;
-import org.apache.flink.api.common.functions.MapFunction;
-import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.api.functions.co.CoMapFunction;
-import org.apache.flink.streaming.util.StreamingMultipleProgramsTestBase;
-import org.apache.flink.streaming.util.TestListResultSink;
-import org.apache.flink.util.Collector;
-import org.junit.Test;
-
-public class SelfConnectionTest extends StreamingMultipleProgramsTestBase {
-
- private static List<String> expected;
-
- /**
- * We connect two different data streams in a chain to a CoMap.
- */
- @Test
- public void differentDataStreamSameChain() {
-
- TestListResultSink<String> resultSink = new TestListResultSink<String>();
-
- StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
- env.setParallelism(1);
-
- DataStream<Integer> src = env.fromElements(1, 3, 5);
-
- DataStream<String> stringMap = src.map(new MapFunction<Integer, String>() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public String map(Integer value) throws Exception {
- return "x " + value;
- }
- });
-
- stringMap.connect(src).map(new CoMapFunction<String, Integer, String>() {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public String map1(String value) {
- return value;
- }
-
- @Override
- public String map2(Integer value) {
- return String.valueOf(value + 1);
- }
- }).addSink(resultSink);
-
- try {
- env.execute();
- } catch (Exception e) {
- e.printStackTrace();
- }
-
- expected = new ArrayList<String>();
-
- expected.addAll(Arrays.asList("x 1", "x 3", "x 5", "2", "4", "6"));
-
- List<String> result = resultSink.getResult();
-
- Collections.sort(expected);
- Collections.sort(result);
-
- assertEquals(expected, result);
- }
-
- /**
- * We connect two different data streams in different chains to a CoMap.
- * (This is not actually self-connect.)
- */
- @Test
- public void differentDataStreamDifferentChain() {
-
- TestListResultSink<String> resultSink = new TestListResultSink<String>();
-
- StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
- env.setParallelism(3);
-
- DataStream<Integer> src = env.fromElements(1, 3, 5).disableChaining();
-
- DataStream<String> stringMap = src.flatMap(new FlatMapFunction<Integer, String>() {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public void flatMap(Integer value, Collector<String> out) throws Exception {
- out.collect("x " + value);
- }
- }).keyBy(new KeySelector<String, Integer>() {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public Integer getKey(String value) throws Exception {
- return value.length();
- }
- });
-
- DataStream<Long> longMap = src.map(new MapFunction<Integer, Long>() {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public Long map(Integer value) throws Exception {
- return Long.valueOf(value + 1);
- }
- }).keyBy(new KeySelector<Long, Integer>() {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public Integer getKey(Long value) throws Exception {
- return value.intValue();
- }
- });
-
-
- stringMap.connect(longMap).map(new CoMapFunction<String, Long, String>() {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public String map1(String value) {
- return value;
- }
-
- @Override
- public String map2(Long value) {
- return value.toString();
- }
- }).addSink(resultSink);
-
- try {
- env.execute();
- } catch (Exception e) {
- e.printStackTrace();
- }
-
- expected = new ArrayList<String>();
-
- expected.addAll(Arrays.asList("x 1", "x 3", "x 5", "2", "4", "6"));
-
- List<String> result = resultSink.getResult();
-
- Collections.sort(expected);
- Collections.sort(result);
-
- assertEquals(expected, result);
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/b9f42e91/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/outputformat/CsvOutputFormatITCase.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/outputformat/CsvOutputFormatITCase.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/outputformat/CsvOutputFormatITCase.java
deleted file mode 100644
index 5377e09..0000000
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/outputformat/CsvOutputFormatITCase.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.outputformat;
-
-import org.apache.flink.api.common.functions.FlatMapFunction;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.util.StreamingProgramTestBase;
-import org.apache.flink.test.testdata.WordCountData;
-import org.apache.flink.util.Collector;
-
-public class CsvOutputFormatITCase extends StreamingProgramTestBase {
-
- protected String resultPath;
-
- @Override
- protected void preSubmit() throws Exception {
- resultPath = getTempDirPath("result");
- }
-
- @Override
- protected void testProgram() throws Exception {
- StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
-
- DataStream<String> text = env.fromElements(WordCountData.TEXT);
-
- DataStream<Tuple2<String, Integer>> counts = text
- .flatMap(new Tokenizer())
- .keyBy(0).sum(1);
-
- counts.writeAsCsv(resultPath);
-
- env.execute("WriteAsCsvTest");
- }
-
- @Override
- protected void postSubmit() throws Exception {
- //Strip the parentheses from the expected text like output
- compareResultsByLinesInMemory(WordCountData.STREAMING_COUNTS_AS_TUPLES
- .replaceAll("[\\\\(\\\\)]", ""), resultPath);
- }
-
- public static final class Tokenizer implements FlatMapFunction<String, Tuple2<String, Integer>> {
- private static final long serialVersionUID = 1L;
-
- @Override
- public void flatMap(String value, Collector<Tuple2<String, Integer>> out)
- throws Exception {
- // normalize and split the line
- String[] tokens = value.toLowerCase().split("\\W+");
-
- // emit the pairs
- for (String token : tokens) {
- if (token.length() > 0) {
- out.collect(new Tuple2<String, Integer>(token, 1));
- }
- }
- }
- }
-
-}
-
http://git-wip-us.apache.org/repos/asf/flink/blob/b9f42e91/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/outputformat/SocketOutputFormatITCase.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/outputformat/SocketOutputFormatITCase.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/outputformat/SocketOutputFormatITCase.java
deleted file mode 100644
index 07c7b74..0000000
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/outputformat/SocketOutputFormatITCase.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.outputformat;
-
-import org.apache.flink.api.common.functions.MapFunction;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.util.SocketOutputTestBase;
-import org.apache.flink.streaming.util.serialization.SimpleStringSchema;
-import org.apache.flink.test.testdata.WordCountData;
-import org.junit.Ignore;
-
-@Ignore
-//This test sometimes failes most likely due to the behaviour
-//of the socket. Disabled for now.
-public class SocketOutputFormatITCase extends SocketOutputTestBase {
-
- @Override
- protected void testProgram() throws Exception {
- StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
-
- DataStream<String> text = env.fromElements(WordCountData.TEXT);
-
- DataStream<String> counts =
- text.flatMap(new CsvOutputFormatITCase.Tokenizer())
- .keyBy(0).sum(1).map(new MapFunction<Tuple2<String, Integer>, String>() {
- @Override
- public String map(Tuple2<String, Integer> value) throws Exception {
- return value.toString() + "\n";
- }
- });
- counts.writeToSocket(HOST, port, new SimpleStringSchema());
-
- env.execute("WriteToSocketTest");
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/b9f42e91/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/outputformat/TextOutputFormatITCase.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/outputformat/TextOutputFormatITCase.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/outputformat/TextOutputFormatITCase.java
deleted file mode 100644
index 380f00d..0000000
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/outputformat/TextOutputFormatITCase.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.outputformat;
-
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.util.StreamingProgramTestBase;
-import org.apache.flink.test.testdata.WordCountData;
-
-public class TextOutputFormatITCase extends StreamingProgramTestBase {
-
- protected String resultPath;
-
- @Override
- protected void preSubmit() throws Exception {
- resultPath = getTempDirPath("result");
- }
-
- @Override
- protected void testProgram() throws Exception {
- StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
-
- DataStream<String> text = env.fromElements(WordCountData.TEXT);
-
- DataStream<Tuple2<String, Integer>> counts = text
- .flatMap(new CsvOutputFormatITCase.Tokenizer())
- .keyBy(0).sum(1);
-
- counts.writeAsText(resultPath);
-
- env.execute("WriteAsTextTest");
- }
-
- @Override
- protected void postSubmit() throws Exception {
- compareResultsByLinesInMemory(WordCountData.STREAMING_COUNTS_AS_TUPLES, resultPath);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/b9f42e91/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/streamtask/StreamVertexTest.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/streamtask/StreamVertexTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/streamtask/StreamVertexTest.java
deleted file mode 100644
index 122aa8a..0000000
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/streamtask/StreamVertexTest.java
+++ /dev/null
@@ -1,187 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.streamtask;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.flink.api.common.functions.RichMapFunction;
-import org.apache.flink.api.java.tuple.Tuple1;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.environment.LocalStreamEnvironment;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.api.functions.co.CoMapFunction;
-import org.apache.flink.streaming.api.functions.sink.SinkFunction;
-import org.apache.flink.streaming.api.functions.source.SourceFunction;
-import org.apache.flink.streaming.util.StreamingMultipleProgramsTestBase;
-import org.apache.flink.streaming.util.TestStreamEnvironment;
-import org.junit.Test;
-
-public class StreamVertexTest extends StreamingMultipleProgramsTestBase {
-
- private static Map<Integer, Integer> data = new HashMap<Integer, Integer>();
-
- public static class MySource implements SourceFunction<Tuple1<Integer>> {
- private static final long serialVersionUID = 1L;
-
- private Tuple1<Integer> tuple = new Tuple1<Integer>(0);
-
- private int i = 0;
-
- @Override
- public void run(SourceContext<Tuple1<Integer>> ctx) throws Exception {
- for (int i = 0; i < 10; i++) {
- tuple.f0 = i;
- ctx.collect(tuple);
- }
- }
-
- @Override
- public void cancel() {
- }
- }
-
- public static class MyTask extends RichMapFunction<Tuple1<Integer>, Tuple2<Integer, Integer>> {
- private static final long serialVersionUID = 1L;
-
- @Override
- public Tuple2<Integer, Integer> map(Tuple1<Integer> value) throws Exception {
- Integer i = value.f0;
- return new Tuple2<Integer, Integer>(i, i + 1);
- }
- }
-
- public static class MySink implements SinkFunction<Tuple2<Integer, Integer>> {
- private static final long serialVersionUID = 1L;
-
- @Override
- public void invoke(Tuple2<Integer, Integer> tuple) {
- Integer k = tuple.getField(0);
- Integer v = tuple.getField(1);
- data.put(k, v);
- }
-
- }
-
- @SuppressWarnings("unused")
- private static final int SOURCE_PARALELISM = 1;
-
- @Test
- public void wrongJobGraph() {
- StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
- env.setParallelism(SOURCE_PARALELISM);
-
- try {
- env.fromCollection(null);
- fail();
- } catch (NullPointerException e) {
- }
-
- try {
- env.fromElements();
- fail();
- } catch (IllegalArgumentException e) {
- }
-
- try {
- env.generateSequence(-10, -30);
- fail();
- } catch (IllegalArgumentException e) {
- }
-
- try {
- env.setBufferTimeout(-10);
- fail();
- } catch (IllegalArgumentException e) {
- }
-
- try {
- env.generateSequence(1, 10).project(2);
- fail();
- } catch (RuntimeException e) {
- }
- }
-
- private static class CoMap implements CoMapFunction<String, Long, String> {
- private static final long serialVersionUID = 1L;
-
- @Override
- public String map1(String value) {
- // System.out.println(value);
- return value;
- }
-
- @Override
- public String map2(Long value) {
- // System.out.println(value);
- return value.toString();
- }
- }
-
- private static class SetSink implements SinkFunction<String> {
- private static final long serialVersionUID = 1L;
- public static Set<String> result = Collections.synchronizedSet(new HashSet<String>());
-
- @Override
- public void invoke(String value) {
- result.add(value);
- }
-
- }
-
- @Test
- public void coTest() throws Exception {
- StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
- env.setParallelism(SOURCE_PARALELISM);
-
- DataStream<String> fromStringElements = env.fromElements("aa", "bb", "cc");
- DataStream<Long> generatedSequence = env.generateSequence(0, 3);
-
- fromStringElements.connect(generatedSequence).map(new CoMap()).addSink(new SetSink());
-
- env.execute();
-
- HashSet<String> expectedSet = new HashSet<String>(Arrays.asList("aa", "bb", "cc", "0", "1",
- "2", "3"));
- assertEquals(expectedSet, SetSink.result);
- }
-
- @Test
- public void runStream() throws Exception {
- StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
- env.setParallelism(SOURCE_PARALELISM);
-
- env.addSource(new MySource()).setParallelism(SOURCE_PARALELISM).map(new MyTask())
- .addSink(new MySink());
-
- env.execute();
- assertEquals(10, data.keySet().size());
-
- for (Integer k : data.keySet()) {
- assertEquals((Integer) (k + 1), data.get(k));
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/b9f42e91/flink-streaming-java/src/test/java/org/apache/flink/streaming/graph/StreamingJobGraphGeneratorNodeHashTest.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/graph/StreamingJobGraphGeneratorNodeHashTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/graph/StreamingJobGraphGeneratorNodeHashTest.java
index 9e1e9b4..bcf621a 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/graph/StreamingJobGraphGeneratorNodeHashTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/graph/StreamingJobGraphGeneratorNodeHashTest.java
@@ -27,17 +27,14 @@ import org.apache.flink.runtime.jobgraph.JobVertex;
import org.apache.flink.runtime.jobgraph.JobVertexID;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.functions.sink.DiscardingSink;
import org.apache.flink.streaming.api.functions.sink.SinkFunction;
import org.apache.flink.streaming.api.functions.source.ParallelSourceFunction;
import org.apache.flink.streaming.api.graph.StreamGraph;
import org.apache.flink.streaming.api.graph.StreamNode;
-import org.apache.flink.streaming.api.operators.ChainingStrategy;
-import org.apache.flink.streaming.api.transformations.StreamTransformation;
-import org.apache.flink.streaming.util.NoOpSink;
+
import org.junit.Test;
-import java.lang.reflect.Field;
-import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -54,6 +51,7 @@ import static org.junit.Assert.assertTrue;
* Tests the {@link StreamNode} hash assignment during translation from {@link StreamGraph} to
* {@link JobGraph} instances.
*/
+@SuppressWarnings("serial")
public class StreamingJobGraphGeneratorNodeHashTest {
// ------------------------------------------------------------------------
@@ -136,7 +134,7 @@ public class StreamingJobGraphGeneratorNodeHashTest {
env.disableOperatorChaining();
env.addSource(new NoOpSourceFunction(), "src").setParallelism(4)
- .addSink(new NoOpSink<String>()).name("sink").setParallelism(4);
+ .addSink(new DiscardingSink<String>()).name("sink").setParallelism(4);
JobGraph jobGraph = env.getStreamGraph().getJobGraph();
@@ -147,7 +145,7 @@ public class StreamingJobGraphGeneratorNodeHashTest {
env.disableOperatorChaining();
env.addSource(new NoOpSourceFunction(), "src").setParallelism(8)
- .addSink(new NoOpSink<String>()).name("sink").setParallelism(4);
+ .addSink(new DiscardingSink<String>()).name("sink").setParallelism(4);
jobGraph = env.getStreamGraph().getJobGraph();
@@ -158,7 +156,7 @@ public class StreamingJobGraphGeneratorNodeHashTest {
env.disableOperatorChaining();
env.addSource(new NoOpSourceFunction(), "src").setParallelism(4)
- .addSink(new NoOpSink<String>()).name("sink").setParallelism(8);
+ .addSink(new DiscardingSink<String>()).name("sink").setParallelism(8);
jobGraph = env.getStreamGraph().getJobGraph();
http://git-wip-us.apache.org/repos/asf/flink/blob/b9f42e91/flink-streaming-java/src/test/java/org/apache/flink/streaming/graph/TranslationTest.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/graph/TranslationTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/graph/TranslationTest.java
index 9d9d47b..66c6a40 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/graph/TranslationTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/graph/TranslationTest.java
@@ -24,13 +24,12 @@ import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.sink.SinkFunction;
import org.apache.flink.streaming.api.graph.StreamConfig;
-import org.apache.flink.streaming.util.StreamingMultipleProgramsTestBase;
import org.junit.Test;
import static org.junit.Assert.*;
@SuppressWarnings("serial")
-public class TranslationTest extends StreamingMultipleProgramsTestBase {
+public class TranslationTest {
@Test
public void testCheckpointModeTranslation() {
http://git-wip-us.apache.org/repos/asf/flink/blob/b9f42e91/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/AllWindowTranslationTest.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/AllWindowTranslationTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/AllWindowTranslationTest.java
index 4c4ed8a..ebe6bea 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/AllWindowTranslationTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/AllWindowTranslationTest.java
@@ -42,8 +42,8 @@ import org.apache.flink.streaming.api.windowing.triggers.EventTimeTrigger;
import org.apache.flink.streaming.api.windowing.triggers.Trigger;
import org.apache.flink.streaming.api.windowing.triggers.TriggerResult;
import org.apache.flink.streaming.api.windowing.windows.TimeWindow;
-import org.apache.flink.streaming.util.StreamingMultipleProgramsTestBase;
import org.apache.flink.util.Collector;
+
import org.junit.Assert;
import org.junit.Test;
@@ -56,7 +56,7 @@ import static org.junit.Assert.fail;
* {@link org.apache.flink.streaming.api.datastream.AllWindowedStream} instantiate
* the correct window operator.
*/
-public class AllWindowTranslationTest extends StreamingMultipleProgramsTestBase {
+public class AllWindowTranslationTest {
/**
* These tests ensure that the correct trigger is set when using event-time windows.
@@ -265,8 +265,6 @@ public class AllWindowTranslationTest extends StreamingMultipleProgramsTestBase
}
fail("The fold call should fail.");
-
- env.execute();
}
@Test
@@ -314,9 +312,6 @@ public class AllWindowTranslationTest extends StreamingMultipleProgramsTestBase
}
fail("The trigger call should fail.");
-
- env.execute();
-
}
// ------------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/b9f42e91/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/CoGroupJoinITCase.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/CoGroupJoinITCase.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/CoGroupJoinITCase.java
deleted file mode 100644
index 5e67c72..0000000
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/CoGroupJoinITCase.java
+++ /dev/null
@@ -1,370 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.runtime.operators.windowing;
-
-import org.apache.flink.api.common.functions.CoGroupFunction;
-import org.apache.flink.api.common.functions.JoinFunction;
-import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.api.java.tuple.Tuple3;
-import org.apache.flink.streaming.api.TimeCharacteristic;
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.api.functions.AssignerWithPunctuatedWatermarks;
-import org.apache.flink.streaming.api.functions.sink.SinkFunction;
-import org.apache.flink.streaming.api.functions.source.SourceFunction;
-import org.apache.flink.streaming.api.watermark.Watermark;
-import org.apache.flink.streaming.api.windowing.assigners.TumblingEventTimeWindows;
-import org.apache.flink.streaming.api.windowing.time.Time;
-import org.apache.flink.streaming.util.StreamingMultipleProgramsTestBase;
-import org.apache.flink.util.Collector;
-
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-
-@SuppressWarnings("serial")
-public class CoGroupJoinITCase extends StreamingMultipleProgramsTestBase {
-
- private static List<String> testResults;
-
- @Test
- public void testCoGroup() throws Exception {
-
- testResults = new ArrayList<>();
-
- StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
- env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
- env.setParallelism(1);
-
- DataStream<Tuple2<String, Integer>> source1 = env.addSource(new SourceFunction<Tuple2<String, Integer>>() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public void run(SourceContext<Tuple2<String, Integer>> ctx) throws Exception {
- ctx.collect(Tuple2.of("a", 0));
- ctx.collect(Tuple2.of("a", 1));
- ctx.collect(Tuple2.of("a", 2));
-
- ctx.collect(Tuple2.of("b", 3));
- ctx.collect(Tuple2.of("b", 4));
- ctx.collect(Tuple2.of("b", 5));
-
- ctx.collect(Tuple2.of("a", 6));
- ctx.collect(Tuple2.of("a", 7));
- ctx.collect(Tuple2.of("a", 8));
-
- // source is finite, so it will have an implicit MAX watermark when it finishes
- }
-
- @Override
- public void cancel() {
- }
- }).assignTimestampsAndWatermarks(new Tuple2TimestampExtractor());
-
- DataStream<Tuple2<String, Integer>> source2 = env.addSource(new SourceFunction<Tuple2<String, Integer>>() {
-
- @Override
- public void run(SourceContext<Tuple2<String, Integer>> ctx) throws Exception {
- ctx.collect(Tuple2.of("a", 0));
- ctx.collect(Tuple2.of("a", 1));
-
- ctx.collect(Tuple2.of("b", 3));
-
- ctx.collect(Tuple2.of("c", 6));
- ctx.collect(Tuple2.of("c", 7));
- ctx.collect(Tuple2.of("c", 8));
-
- // source is finite, so it will have an implicit MAX watermark when it finishes
- }
-
- @Override
- public void cancel() {
- }
- }).assignTimestampsAndWatermarks(new Tuple2TimestampExtractor());
-
-
- source1.coGroup(source2)
- .where(new Tuple2KeyExtractor())
- .equalTo(new Tuple2KeyExtractor())
- .window(TumblingEventTimeWindows.of(Time.of(3, TimeUnit.MILLISECONDS)))
- .apply(new CoGroupFunction<Tuple2<String,Integer>, Tuple2<String,Integer>, String>() {
- @Override
- public void coGroup(Iterable<Tuple2<String, Integer>> first,
- Iterable<Tuple2<String, Integer>> second,
- Collector<String> out) throws Exception {
- StringBuilder result = new StringBuilder();
- result.append("F:");
- for (Tuple2<String, Integer> t: first) {
- result.append(t.toString());
- }
- result.append(" S:");
- for (Tuple2<String, Integer> t: second) {
- result.append(t.toString());
- }
- out.collect(result.toString());
- }
- })
- .addSink(new SinkFunction<String>() {
- @Override
- public void invoke(String value) throws Exception {
- testResults.add(value);
- }
- });
-
- env.execute("CoGroup Test");
-
- List<String> expectedResult = Arrays.asList(
- "F:(a,0)(a,1)(a,2) S:(a,0)(a,1)",
- "F:(b,3)(b,4)(b,5) S:(b,3)",
- "F:(a,6)(a,7)(a,8) S:",
- "F: S:(c,6)(c,7)(c,8)");
-
- Collections.sort(expectedResult);
- Collections.sort(testResults);
-
- Assert.assertEquals(expectedResult, testResults);
- }
-
- @Test
- public void testJoin() throws Exception {
-
- testResults = new ArrayList<>();
-
- StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
- env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
- env.setParallelism(1);
-
- DataStream<Tuple3<String, String, Integer>> source1 = env.addSource(new SourceFunction<Tuple3<String, String, Integer>>() {
-
- @Override
- public void run(SourceContext<Tuple3<String, String, Integer>> ctx) throws Exception {
- ctx.collect(Tuple3.of("a", "x", 0));
- ctx.collect(Tuple3.of("a", "y", 1));
- ctx.collect(Tuple3.of("a", "z", 2));
-
- ctx.collect(Tuple3.of("b", "u", 3));
- ctx.collect(Tuple3.of("b", "w", 5));
-
- ctx.collect(Tuple3.of("a", "i", 6));
- ctx.collect(Tuple3.of("a", "j", 7));
- ctx.collect(Tuple3.of("a", "k", 8));
-
- // source is finite, so it will have an implicit MAX watermark when it finishes
- }
-
- @Override
- public void cancel() {}
-
- }).assignTimestampsAndWatermarks(new Tuple3TimestampExtractor());
-
- DataStream<Tuple3<String, String, Integer>> source2 = env.addSource(new SourceFunction<Tuple3<String, String, Integer>>() {
-
- @Override
- public void run(SourceContext<Tuple3<String, String, Integer>> ctx) throws Exception {
- ctx.collect(Tuple3.of("a", "u", 0));
- ctx.collect(Tuple3.of("a", "w", 1));
-
- ctx.collect(Tuple3.of("b", "i", 3));
- ctx.collect(Tuple3.of("b", "k", 5));
-
- ctx.collect(Tuple3.of("a", "x", 6));
- ctx.collect(Tuple3.of("a", "z", 8));
-
- // source is finite, so it will have an implicit MAX watermark when it finishes
- }
-
- @Override
- public void cancel() {}
-
- }).assignTimestampsAndWatermarks(new Tuple3TimestampExtractor());
-
-
- source1.join(source2)
- .where(new Tuple3KeyExtractor())
- .equalTo(new Tuple3KeyExtractor())
- .window(TumblingEventTimeWindows.of(Time.of(3, TimeUnit.MILLISECONDS)))
- .apply(new JoinFunction<Tuple3<String, String, Integer>, Tuple3<String, String, Integer>, String>() {
- @Override
- public String join(Tuple3<String, String, Integer> first, Tuple3<String, String, Integer> second) throws Exception {
- return first + ":" + second;
- }
- })
- .addSink(new SinkFunction<String>() {
- @Override
- public void invoke(String value) throws Exception {
- testResults.add(value);
- }
- });
-
- env.execute("Join Test");
-
- List<String> expectedResult = Arrays.asList(
- "(a,x,0):(a,u,0)",
- "(a,x,0):(a,w,1)",
- "(a,y,1):(a,u,0)",
- "(a,y,1):(a,w,1)",
- "(a,z,2):(a,u,0)",
- "(a,z,2):(a,w,1)",
- "(b,u,3):(b,i,3)",
- "(b,u,3):(b,k,5)",
- "(b,w,5):(b,i,3)",
- "(b,w,5):(b,k,5)",
- "(a,i,6):(a,x,6)",
- "(a,i,6):(a,z,8)",
- "(a,j,7):(a,x,6)",
- "(a,j,7):(a,z,8)",
- "(a,k,8):(a,x,6)",
- "(a,k,8):(a,z,8)");
-
- Collections.sort(expectedResult);
- Collections.sort(testResults);
-
- Assert.assertEquals(expectedResult, testResults);
- }
-
- @Test
- public void testSelfJoin() throws Exception {
-
- testResults = new ArrayList<>();
-
- StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
- env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
- env.setParallelism(1);
-
- DataStream<Tuple3<String, String, Integer>> source1 = env.addSource(new SourceFunction<Tuple3<String, String, Integer>>() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public void run(SourceContext<Tuple3<String, String, Integer>> ctx) throws Exception {
- ctx.collect(Tuple3.of("a", "x", 0));
- ctx.collect(Tuple3.of("a", "y", 1));
- ctx.collect(Tuple3.of("a", "z", 2));
-
- ctx.collect(Tuple3.of("b", "u", 3));
- ctx.collect(Tuple3.of("b", "w", 5));
-
- ctx.collect(Tuple3.of("a", "i", 6));
- ctx.collect(Tuple3.of("a", "j", 7));
- ctx.collect(Tuple3.of("a", "k", 8));
-
- // source is finite, so it will have an implicit MAX watermark when it finishes
- }
-
- @Override
- public void cancel() {
- }
- }).assignTimestampsAndWatermarks(new Tuple3TimestampExtractor());
-
- source1.join(source1)
- .where(new Tuple3KeyExtractor())
- .equalTo(new Tuple3KeyExtractor())
- .window(TumblingEventTimeWindows.of(Time.of(3, TimeUnit.MILLISECONDS)))
- .apply(new JoinFunction<Tuple3<String, String, Integer>, Tuple3<String, String, Integer>, String>() {
- @Override
- public String join(Tuple3<String, String, Integer> first, Tuple3<String, String, Integer> second) throws Exception {
- return first + ":" + second;
- }
- })
- .addSink(new SinkFunction<String>() {
- @Override
- public void invoke(String value) throws Exception {
- testResults.add(value);
- }
- });
-
- env.execute("Self-Join Test");
-
- List<String> expectedResult = Arrays.asList(
- "(a,x,0):(a,x,0)",
- "(a,x,0):(a,y,1)",
- "(a,x,0):(a,z,2)",
- "(a,y,1):(a,x,0)",
- "(a,y,1):(a,y,1)",
- "(a,y,1):(a,z,2)",
- "(a,z,2):(a,x,0)",
- "(a,z,2):(a,y,1)",
- "(a,z,2):(a,z,2)",
- "(b,u,3):(b,u,3)",
- "(b,u,3):(b,w,5)",
- "(b,w,5):(b,u,3)",
- "(b,w,5):(b,w,5)",
- "(a,i,6):(a,i,6)",
- "(a,i,6):(a,j,7)",
- "(a,i,6):(a,k,8)",
- "(a,j,7):(a,i,6)",
- "(a,j,7):(a,j,7)",
- "(a,j,7):(a,k,8)",
- "(a,k,8):(a,i,6)",
- "(a,k,8):(a,j,7)",
- "(a,k,8):(a,k,8)");
-
- Collections.sort(expectedResult);
- Collections.sort(testResults);
-
- Assert.assertEquals(expectedResult, testResults);
- }
-
- private static class Tuple2TimestampExtractor implements AssignerWithPunctuatedWatermarks<Tuple2<String, Integer>> {
-
- @Override
- public long extractTimestamp(Tuple2<String, Integer> element, long previousTimestamp) {
- return element.f1;
- }
-
- @Override
- public Watermark checkAndGetNextWatermark(Tuple2<String, Integer> element, long extractedTimestamp) {
- return new Watermark(extractedTimestamp - 1);
- }
- }
-
- private static class Tuple3TimestampExtractor implements AssignerWithPunctuatedWatermarks<Tuple3<String, String, Integer>> {
-
- @Override
- public long extractTimestamp(Tuple3<String, String, Integer> element, long previousTimestamp) {
- return element.f2;
- }
-
- @Override
- public Watermark checkAndGetNextWatermark(Tuple3<String, String, Integer> lastElement, long extractedTimestamp) {
- return new Watermark(lastElement.f2 - 1);
- }
- }
-
- private static class Tuple2KeyExtractor implements KeySelector<Tuple2<String,Integer>, String> {
-
- @Override
- public String getKey(Tuple2<String, Integer> value) throws Exception {
- return value.f0;
- }
- }
-
- private static class Tuple3KeyExtractor implements KeySelector<Tuple3<String, String, Integer>, String> {
-
- @Override
- public String getKey(Tuple3<String, String, Integer> value) throws Exception {
- return value.f0;
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/b9f42e91/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/TimeWindowTranslationTest.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/TimeWindowTranslationTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/TimeWindowTranslationTest.java
index 87da045..c1ad0fc 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/TimeWindowTranslationTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/TimeWindowTranslationTest.java
@@ -35,8 +35,8 @@ import org.apache.flink.streaming.api.windowing.assigners.TumblingEventTimeWindo
import org.apache.flink.streaming.api.windowing.time.Time;
import org.apache.flink.streaming.api.windowing.triggers.EventTimeTrigger;
import org.apache.flink.streaming.api.windowing.windows.TimeWindow;
-import org.apache.flink.streaming.util.StreamingMultipleProgramsTestBase;
import org.apache.flink.util.Collector;
+
import org.junit.Assert;
import org.junit.Ignore;
import org.junit.Test;
@@ -48,7 +48,7 @@ import java.util.concurrent.TimeUnit;
* {@link WindowedStream} instantiate
* the correct window operator.
*/
-public class TimeWindowTranslationTest extends StreamingMultipleProgramsTestBase {
+public class TimeWindowTranslationTest {
/**
* These tests ensure that the fast aligned time windows operator is used if the
http://git-wip-us.apache.org/repos/asf/flink/blob/b9f42e91/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowFoldITCase.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowFoldITCase.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowFoldITCase.java
deleted file mode 100644
index fbe03c5..0000000
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowFoldITCase.java
+++ /dev/null
@@ -1,191 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.runtime.operators.windowing;
-
-import org.apache.flink.api.common.functions.FoldFunction;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.streaming.api.TimeCharacteristic;
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.api.functions.AssignerWithPunctuatedWatermarks;
-import org.apache.flink.streaming.api.functions.sink.SinkFunction;
-import org.apache.flink.streaming.api.functions.source.SourceFunction;
-import org.apache.flink.streaming.api.watermark.Watermark;
-import org.apache.flink.streaming.api.windowing.assigners.TumblingEventTimeWindows;
-import org.apache.flink.streaming.api.windowing.time.Time;
-import org.apache.flink.streaming.util.StreamingMultipleProgramsTestBase;
-
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-
-/**
- * Tests for Folds over windows. These also test whether OutputTypeConfigurable functions
- * work for windows, because FoldWindowFunction is OutputTypeConfigurable.
- */
-@SuppressWarnings("serial")
-public class WindowFoldITCase extends StreamingMultipleProgramsTestBase {
-
- private static List<String> testResults;
-
- @Test
- public void testFoldWindow() throws Exception {
-
- testResults = new ArrayList<>();
-
- StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
- env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
- env.setParallelism(1);
-
- DataStream<Tuple2<String, Integer>> source1 = env.addSource(new SourceFunction<Tuple2<String, Integer>>() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public void run(SourceContext<Tuple2<String, Integer>> ctx) throws Exception {
- ctx.collect(Tuple2.of("a", 0));
- ctx.collect(Tuple2.of("a", 1));
- ctx.collect(Tuple2.of("a", 2));
-
- ctx.collect(Tuple2.of("b", 3));
- ctx.collect(Tuple2.of("b", 4));
- ctx.collect(Tuple2.of("b", 5));
-
- ctx.collect(Tuple2.of("a", 6));
- ctx.collect(Tuple2.of("a", 7));
- ctx.collect(Tuple2.of("a", 8));
-
- // source is finite, so it will have an implicit MAX watermark when it finishes
- }
-
- @Override
- public void cancel() {}
-
- }).assignTimestampsAndWatermarks(new Tuple2TimestampExtractor());
-
- source1
- .keyBy(0)
- .window(TumblingEventTimeWindows.of(Time.of(3, TimeUnit.MILLISECONDS)))
- .fold(Tuple2.of("R:", 0), new FoldFunction<Tuple2<String, Integer>, Tuple2<String, Integer>>() {
- @Override
- public Tuple2<String, Integer> fold(Tuple2<String, Integer> accumulator,
- Tuple2<String, Integer> value) throws Exception {
- accumulator.f0 += value.f0;
- accumulator.f1 += value.f1;
- return accumulator;
- }
- })
- .addSink(new SinkFunction<Tuple2<String, Integer>>() {
- @Override
- public void invoke(Tuple2<String, Integer> value) throws Exception {
- testResults.add(value.toString());
- }
- });
-
- env.execute("Fold Window Test");
-
- List<String> expectedResult = Arrays.asList(
- "(R:aaa,3)",
- "(R:aaa,21)",
- "(R:bbb,12)");
-
- Collections.sort(expectedResult);
- Collections.sort(testResults);
-
- Assert.assertEquals(expectedResult, testResults);
- }
-
- @Test
- public void testFoldAllWindow() throws Exception {
-
- testResults = new ArrayList<>();
-
- StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
- env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
- env.setParallelism(1);
-
- DataStream<Tuple2<String, Integer>> source1 = env.addSource(new SourceFunction<Tuple2<String, Integer>>() {
-
- @Override
- public void run(SourceContext<Tuple2<String, Integer>> ctx) throws Exception {
- ctx.collect(Tuple2.of("a", 0));
- ctx.collect(Tuple2.of("a", 1));
- ctx.collect(Tuple2.of("a", 2));
-
- ctx.collect(Tuple2.of("b", 3));
- ctx.collect(Tuple2.of("a", 3));
- ctx.collect(Tuple2.of("b", 4));
- ctx.collect(Tuple2.of("a", 4));
- ctx.collect(Tuple2.of("b", 5));
- ctx.collect(Tuple2.of("a", 5));
-
- // source is finite, so it will have an implicit MAX watermark when it finishes
- }
-
- @Override
- public void cancel() {
- }
- }).assignTimestampsAndWatermarks(new Tuple2TimestampExtractor());
-
- source1
- .windowAll(TumblingEventTimeWindows.of(Time.of(3, TimeUnit.MILLISECONDS)))
- .fold(Tuple2.of("R:", 0), new FoldFunction<Tuple2<String, Integer>, Tuple2<String, Integer>>() {
- @Override
- public Tuple2<String, Integer> fold(Tuple2<String, Integer> accumulator,
- Tuple2<String, Integer> value) throws Exception {
- accumulator.f0 += value.f0;
- accumulator.f1 += value.f1;
- return accumulator;
- }
- })
- .addSink(new SinkFunction<Tuple2<String, Integer>>() {
- @Override
- public void invoke(Tuple2<String, Integer> value) throws Exception {
- testResults.add(value.toString());
- }
- });
-
- env.execute("Fold All-Window Test");
-
- List<String> expectedResult = Arrays.asList(
- "(R:aaa,3)",
- "(R:bababa,24)");
-
- Collections.sort(expectedResult);
- Collections.sort(testResults);
-
- Assert.assertEquals(expectedResult, testResults);
- }
-
- private static class Tuple2TimestampExtractor implements AssignerWithPunctuatedWatermarks<Tuple2<String, Integer>> {
-
- @Override
- public long extractTimestamp(Tuple2<String, Integer> element, long previousTimestamp) {
- return element.f1;
- }
-
- @Override
- public Watermark checkAndGetNextWatermark(Tuple2<String, Integer> lastElement, long extractedTimestamp) {
- return new Watermark(lastElement.f1 - 1);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/b9f42e91/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowTranslationTest.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowTranslationTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowTranslationTest.java
index cb3801e..39d89cf 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowTranslationTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowTranslationTest.java
@@ -44,8 +44,8 @@ import org.apache.flink.streaming.api.windowing.triggers.EventTimeTrigger;
import org.apache.flink.streaming.api.windowing.triggers.Trigger;
import org.apache.flink.streaming.api.windowing.triggers.TriggerResult;
import org.apache.flink.streaming.api.windowing.windows.TimeWindow;
-import org.apache.flink.streaming.util.StreamingMultipleProgramsTestBase;
import org.apache.flink.util.Collector;
+
import org.junit.Assert;
import org.junit.Test;
@@ -58,7 +58,8 @@ import static org.junit.Assert.fail;
* {@link WindowedStream} instantiate
* the correct window operator.
*/
-public class WindowTranslationTest extends StreamingMultipleProgramsTestBase {
+@SuppressWarnings("serial")
+public class WindowTranslationTest {
/**
* .reduce() does not support RichReduceFunction, since the reduce function is used internally
@@ -262,8 +263,6 @@ public class WindowTranslationTest extends StreamingMultipleProgramsTestBase {
}
fail("The fold call should fail.");
-
- env.execute();
}
@Test
@@ -317,8 +316,6 @@ public class WindowTranslationTest extends StreamingMultipleProgramsTestBase {
}
fail("The trigger call should fail.");
-
- env.execute();
}
http://git-wip-us.apache.org/repos/asf/flink/blob/b9f42e91/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/state/StateBackendITCase.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/state/StateBackendITCase.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/state/StateBackendITCase.java
deleted file mode 100644
index 222924c..0000000
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/state/StateBackendITCase.java
+++ /dev/null
@@ -1,149 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.runtime.state;
-
-import org.apache.flink.api.common.functions.RichMapFunction;
-import org.apache.flink.api.common.state.FoldingState;
-import org.apache.flink.api.common.restartstrategy.RestartStrategies;
-import org.apache.flink.api.common.state.FoldingStateDescriptor;
-import org.apache.flink.api.common.state.ListState;
-import org.apache.flink.api.common.state.ListStateDescriptor;
-import org.apache.flink.api.common.state.ReducingState;
-import org.apache.flink.api.common.state.ReducingStateDescriptor;
-import org.apache.flink.api.common.state.ValueState;
-import org.apache.flink.api.common.state.ValueStateDescriptor;
-import org.apache.flink.api.common.typeutils.TypeSerializer;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.runtime.client.JobExecutionException;
-import org.apache.flink.runtime.execution.Environment;
-import org.apache.flink.runtime.state.AbstractStateBackend;
-import org.apache.flink.runtime.state.StateHandle;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.util.StreamingMultipleProgramsTestBase;
-
-import org.junit.Test;
-
-import java.io.Serializable;
-
-import static org.junit.Assert.fail;
-
-public class StateBackendITCase extends StreamingMultipleProgramsTestBase {
-
- /**
- * Verify that the user-specified state backend is used even if checkpointing is disabled.
- *
- * @throws Exception
- */
- @Test
- public void testStateBackendWithoutCheckpointing() throws Exception {
-
- StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment();
- see.setParallelism(1);
-
- see.getConfig().setRestartStrategy(RestartStrategies.noRestart());
- see.setStateBackend(new FailingStateBackend());
-
-
- see.fromElements(new Tuple2<>("Hello", 1))
- .keyBy(0)
- .map(new RichMapFunction<Tuple2<String,Integer>, String>() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public void open(Configuration parameters) throws Exception {
- super.open(parameters);
- getRuntimeContext().getKeyValueState("test", String.class, "");
- }
-
- @Override
- public String map(Tuple2<String, Integer> value) throws Exception {
- return value.f0;
- }
- })
- .print();
-
- try {
- see.execute();
- fail();
- }
- catch (JobExecutionException e) {
- Throwable t = e.getCause();
- if (!(t != null && t.getCause() instanceof SuccessException)) {
- throw e;
- }
- }
- }
-
-
- public static class FailingStateBackend extends AbstractStateBackend {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public void initializeForJob(Environment env, String operatorIdentifier, TypeSerializer<?> keySerializer) throws Exception {
- throw new SuccessException();
- }
-
- @Override
- public void disposeAllStateForCurrentJob() throws Exception {}
-
- @Override
- public void close() throws Exception {}
-
- @Override
- protected <N, T> ValueState<T> createValueState(TypeSerializer<N> namespaceSerializer, ValueStateDescriptor<T> stateDesc) throws Exception {
- return null;
- }
-
- @Override
- protected <N, T> ListState<T> createListState(TypeSerializer<N> namespaceSerializer, ListStateDescriptor<T> stateDesc) throws Exception {
- return null;
- }
-
- @Override
- protected <N, T> ReducingState<T> createReducingState(TypeSerializer<N> namespaceSerializer, ReducingStateDescriptor<T> stateDesc) throws Exception {
- return null;
- }
-
- @Override
- protected <N, T, ACC> FoldingState<T, ACC> createFoldingState(TypeSerializer<N> namespaceSerializer,
- FoldingStateDescriptor<T, ACC> stateDesc) throws Exception {
- return null;
- }
-
- @Override
- public CheckpointStateOutputStream createCheckpointStateOutputStream(long checkpointID,
- long timestamp) throws Exception {
- return null;
- }
-
- @Override
- public <S extends Serializable> StateHandle<S> checkpointStateSerializable(S state,
- long checkpointID,
- long timestamp) throws Exception {
- return null;
- }
- }
-
- static final class SuccessException extends Exception {
- private static final long serialVersionUID = -9218191172606739598L;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/b9f42e91/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskTimerITCase.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskTimerITCase.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskTimerITCase.java
deleted file mode 100644
index 32d57ca..0000000
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskTimerITCase.java
+++ /dev/null
@@ -1,338 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.runtime.tasks;
-
-import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
-import org.apache.flink.runtime.client.JobExecutionException;
-import org.apache.flink.streaming.api.TimeCharacteristic;
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.api.functions.source.SourceFunction;
-import org.apache.flink.streaming.api.operators.AbstractStreamOperator;
-import org.apache.flink.streaming.api.operators.ChainingStrategy;
-import org.apache.flink.streaming.api.operators.OneInputStreamOperator;
-import org.apache.flink.streaming.api.operators.TwoInputStreamOperator;
-import org.apache.flink.streaming.api.watermark.Watermark;
-import org.apache.flink.streaming.runtime.operators.Triggerable;
-import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
-import org.apache.flink.streaming.util.StreamingMultipleProgramsTestBase;
-
-import org.junit.Assert;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.concurrent.Semaphore;
-
-/**
- * Tests for the timer service of {@code StreamTask}.
- *
- * <p>
- * These tests ensure that exceptions are properly forwarded from the timer thread to
- * the task thread and that operator methods are not invoked concurrently.
- */
-@RunWith(Parameterized.class)
-public class StreamTaskTimerITCase extends StreamingMultipleProgramsTestBase {
-
- private final TimeCharacteristic timeCharacteristic;
-
- public StreamTaskTimerITCase(TimeCharacteristic characteristic) {
- timeCharacteristic = characteristic;
- }
-
-
- /**
- * Note: this test fails if we don't check for exceptions in the source contexts and do not
- * synchronize in the source contexts.
- */
- @Test
- public void testOperatorChainedToSource() throws Exception {
-
- StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
- env.setStreamTimeCharacteristic(timeCharacteristic);
- env.setParallelism(1);
-
- DataStream<String> source = env.addSource(new InfiniteTestSource());
-
- source.transform("Custom Operator", BasicTypeInfo.STRING_TYPE_INFO, new TimerOperator(ChainingStrategy.ALWAYS));
-
- boolean testSuccess = false;
- try {
- env.execute("Timer test");
- } catch (JobExecutionException e) {
- if (e.getCause() instanceof TimerException) {
- TimerException te = (TimerException) e.getCause();
- if (te.getCause() instanceof RuntimeException) {
- RuntimeException re = (RuntimeException) te.getCause();
- if (re.getMessage().equals("TEST SUCCESS")) {
- testSuccess = true;
- } else {
- throw e;
- }
- } else {
- throw e;
- }
- } else {
- throw e;
- }
- }
- Assert.assertTrue(testSuccess);
- }
-
- /**
- * Note: this test fails if we don't check for exceptions in the source contexts and do not
- * synchronize in the source contexts.
- */
- @Test
- public void testOneInputOperatorWithoutChaining() throws Exception {
- StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
- env.setStreamTimeCharacteristic(timeCharacteristic);
- env.setParallelism(1);
-
- DataStream<String> source = env.addSource(new InfiniteTestSource());
-
- source.transform("Custom Operator", BasicTypeInfo.STRING_TYPE_INFO, new TimerOperator(ChainingStrategy.NEVER));
-
- boolean testSuccess = false;
- try {
- env.execute("Timer test");
- } catch (JobExecutionException e) {
- if (e.getCause() instanceof TimerException) {
- TimerException te = (TimerException) e.getCause();
- if (te.getCause() instanceof RuntimeException) {
- RuntimeException re = (RuntimeException) te.getCause();
- if (re.getMessage().equals("TEST SUCCESS")) {
- testSuccess = true;
- } else {
- throw e;
- }
- } else {
- throw e;
- }
- } else {
- throw e;
- }
- }
- Assert.assertTrue(testSuccess);
- }
-
- @Test
- public void testTwoInputOperatorWithoutChaining() throws Exception {
- StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
- env.setStreamTimeCharacteristic(timeCharacteristic);
- env.setParallelism(1);
-
- DataStream<String> source = env.addSource(new InfiniteTestSource());
-
- source.connect(source).transform(
- "Custom Operator",
- BasicTypeInfo.STRING_TYPE_INFO,
- new TwoInputTimerOperator(ChainingStrategy.NEVER));
-
- boolean testSuccess = false;
- try {
- env.execute("Timer test");
- } catch (JobExecutionException e) {
- if (e.getCause() instanceof TimerException) {
- TimerException te = (TimerException) e.getCause();
- if (te.getCause() instanceof RuntimeException) {
- RuntimeException re = (RuntimeException) te.getCause();
- if (re.getMessage().equals("TEST SUCCESS")) {
- testSuccess = true;
- } else {
- throw e;
- }
- } else {
- throw e;
- }
- } else {
- throw e;
- }
- }
- Assert.assertTrue(testSuccess);
- }
-
- public static class TimerOperator extends AbstractStreamOperator<String> implements OneInputStreamOperator<String, String>, Triggerable {
- private static final long serialVersionUID = 1L;
-
- int numTimers = 0;
- int numElements = 0;
-
- private boolean first = true;
-
- private Semaphore semaphore = new Semaphore(1);
-
- public TimerOperator(ChainingStrategy chainingStrategy) {
- setChainingStrategy(chainingStrategy);
- }
-
- @Override
- public void processElement(StreamRecord<String> element) throws Exception {
- if (!semaphore.tryAcquire()) {
- Assert.fail("Concurrent invocation of operator functions.");
- }
-
- if (first) {
- registerTimer(System.currentTimeMillis() + 100, this);
- first = false;
- }
- numElements++;
-
- semaphore.release();
- }
-
- @Override
- public void trigger(long time) throws Exception {
- if (!semaphore.tryAcquire()) {
- Assert.fail("Concurrent invocation of operator functions.");
- }
-
- try {
- numTimers++;
- throwIfDone();
- registerTimer(System.currentTimeMillis() + 1, this);
- } finally {
- semaphore.release();
- }
- }
-
- private void throwIfDone() {
- if (numTimers > 1000 && numElements > 10_000) {
- throw new RuntimeException("TEST SUCCESS");
- }
- }
-
- @Override
- public void processWatermark(Watermark mark) throws Exception {
- if (!semaphore.tryAcquire()) {
- Assert.fail("Concurrent invocation of operator functions.");
- }
- semaphore.release();
- }
- }
-
- public static class TwoInputTimerOperator extends AbstractStreamOperator<String> implements TwoInputStreamOperator<String, String, String>, Triggerable {
- private static final long serialVersionUID = 1L;
-
- int numTimers = 0;
- int numElements = 0;
-
- private boolean first = true;
-
- private Semaphore semaphore = new Semaphore(1);
-
- public TwoInputTimerOperator(ChainingStrategy chainingStrategy) {
- setChainingStrategy(chainingStrategy);
- }
-
- @Override
- public void processElement1(StreamRecord<String> element) throws Exception {
- if (!semaphore.tryAcquire()) {
- Assert.fail("Concurrent invocation of operator functions.");
- }
-
- if (first) {
- registerTimer(System.currentTimeMillis() + 100, this);
- first = false;
- }
- numElements++;
-
- semaphore.release();
- }
-
- @Override
- public void processElement2(StreamRecord<String> element) throws Exception {
- if (!semaphore.tryAcquire()) {
- Assert.fail("Concurrent invocation of operator functions.");
- }
-
- if (first) {
- registerTimer(System.currentTimeMillis() + 100, this);
- first = false;
- }
- numElements++;
-
- semaphore.release();
- }
-
-
- @Override
- public void trigger(long time) throws Exception {
- if (!semaphore.tryAcquire()) {
- Assert.fail("Concurrent invocation of operator functions.");
- }
-
- try {
- numTimers++;
- throwIfDone();
- registerTimer(System.currentTimeMillis() + 1, this);
- } finally {
- semaphore.release();
- }
- }
-
- private void throwIfDone() {
- if (numTimers > 1000 && numElements > 10_000) {
- throw new RuntimeException("TEST SUCCESS");
- }
- }
-
- @Override
- public void processWatermark1(Watermark mark) throws Exception {
- //ignore
- }
-
- @Override
- public void processWatermark2(Watermark mark) throws Exception {
- //ignore
- }
- }
-
-
- private static class InfiniteTestSource implements SourceFunction<String> {
- private static final long serialVersionUID = 1L;
- private volatile boolean running = true;
-
- @Override
- public void run(SourceContext<String> ctx) throws Exception {
- while (running) {
- ctx.collect("hello");
- }
- }
-
- @Override
- public void cancel() {
- running = false;
- }
- }
-
- // ------------------------------------------------------------------------
- // parametrization
- // ------------------------------------------------------------------------
-
- @Parameterized.Parameters(name = "Time Characteristic = {0}")
- public static Collection<Object[]> executionModes() {
- return Arrays.asList(
- new Object[] { TimeCharacteristic.ProcessingTime },
- new Object[] { TimeCharacteristic.IngestionTime },
- new Object[] { TimeCharacteristic.EventTime });
- }
-}
[05/15] flink git commit: [FLINK-3995] [build] Properly structure
test scopes and dependencies
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskAsyncCheckpointTest.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskAsyncCheckpointTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskAsyncCheckpointTest.java
index 0e1da88..8e1cadf 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskAsyncCheckpointTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskAsyncCheckpointTest.java
@@ -21,11 +21,11 @@ package org.apache.flink.streaming.runtime.tasks;
import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
+import org.apache.flink.core.testutils.OneShotLatch;
import org.apache.flink.runtime.io.network.api.writer.ResultPartitionWriter;
import org.apache.flink.runtime.operators.testutils.MockInputSplitProvider;
import org.apache.flink.runtime.state.AsynchronousStateHandle;
import org.apache.flink.runtime.state.StateHandle;
-import org.apache.flink.runtime.taskmanager.OneShotLatch;
import org.apache.flink.streaming.api.graph.StreamConfig;
import org.apache.flink.streaming.api.operators.AbstractStreamOperator;
import org.apache.flink.streaming.api.operators.OneInputStreamOperator;
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskTest.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskTest.java
index f054e18..76f22be 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskTest.java
@@ -20,7 +20,7 @@ package org.apache.flink.streaming.runtime.tasks;
import akka.actor.ActorRef;
-import org.apache.flink.api.common.ExecutionConfigTest;
+import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.api.common.JobID;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.metrics.groups.TaskMetricGroup;
@@ -50,6 +50,7 @@ import org.apache.flink.streaming.api.operators.StreamSource;
import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
import org.apache.flink.util.ExceptionUtils;
+import org.apache.flink.util.SerializedValue;
import org.junit.Test;
import scala.concurrent.ExecutionContext;
@@ -123,7 +124,7 @@ public class StreamTaskTest {
// Test Utilities
// ------------------------------------------------------------------------
- private Task createTask(Class<? extends AbstractInvokable> invokable, StreamConfig taskConfig) {
+ private Task createTask(Class<? extends AbstractInvokable> invokable, StreamConfig taskConfig) throws Exception {
LibraryCacheManager libCache = mock(LibraryCacheManager.class);
when(libCache.getClassLoader(any(JobID.class))).thenReturn(getClass().getClassLoader());
@@ -136,7 +137,7 @@ public class StreamTaskTest {
TaskDeploymentDescriptor tdd = new TaskDeploymentDescriptor(
new JobID(), "Job Name", new JobVertexID(), new ExecutionAttemptID(),
- ExecutionConfigTest.getSerializedConfig(),
+ new SerializedValue<>(new ExecutionConfig()),
"Test Task", 0, 1, 0,
new Configuration(),
taskConfig.getConfiguration(),
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-streaming-java/src/test/java/org/apache/flink/streaming/timestamp/TimestampITCase.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/timestamp/TimestampITCase.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/timestamp/TimestampITCase.java
index d3cdcab..6c1d0e6 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/timestamp/TimestampITCase.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/timestamp/TimestampITCase.java
@@ -26,7 +26,7 @@ import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.configuration.ConfigConstants;
import org.apache.flink.configuration.Configuration;
-import org.apache.flink.runtime.taskmanager.MultiShotLatch;
+import org.apache.flink.core.testutils.MultiShotLatch;
import org.apache.flink.streaming.api.TimeCharacteristic;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
@@ -45,6 +45,7 @@ import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
import org.apache.flink.streaming.util.NoOpSink;
import org.apache.flink.test.util.ForkableFlinkMiniCluster;
+import org.apache.flink.util.TestLogger;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.Before;
@@ -63,7 +64,7 @@ import static org.junit.Assert.fail;
* Tests for timestamps, watermarks, and event-time sources.
*/
@SuppressWarnings("serial")
-public class TimestampITCase {
+public class TimestampITCase extends TestLogger {
private static final int NUM_TASK_MANAGERS = 2;
private static final int NUM_TASK_SLOTS = 3;
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-streaming-scala/pom.xml
----------------------------------------------------------------------
diff --git a/flink-streaming-scala/pom.xml b/flink-streaming-scala/pom.xml
index 78d5fd3..ffbcb87 100644
--- a/flink-streaming-scala/pom.xml
+++ b/flink-streaming-scala/pom.xml
@@ -33,6 +33,9 @@ under the License.
<packaging>jar</packaging>
<dependencies>
+
+ <!-- core dependencies -->
+
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-streaming-java_2.10</artifactId>
@@ -72,22 +75,27 @@ under the License.
<version>${guava.version}</version>
</dependency>
- <!-- To access general test utils -->
+ <!-- test dependencies -->
+
+ <dependency>
+ <groupId>org.scalatest</groupId>
+ <artifactId>scalatest_${scala.binary.version}</artifactId>
+ <scope>test</scope>
+ </dependency>
+
<dependency>
<groupId>org.apache.flink</groupId>
- <artifactId>flink-tests_2.10</artifactId>
+ <artifactId>flink-test-utils_2.10</artifactId>
<version>${project.version}</version>
<scope>test</scope>
- <type>test-jar</type>
</dependency>
- <!-- To access test data -->
<dependency>
<groupId>org.apache.flink</groupId>
- <artifactId>flink-test-utils_2.10</artifactId>
+ <artifactId>flink-tests_2.10</artifactId>
<version>${project.version}</version>
- <type>test-jar</type>
<scope>test</scope>
+ <type>test-jar</type>
</dependency>
<!-- To access streaming test utils -->
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-test-utils-parent/flink-test-utils-junit/pom.xml
----------------------------------------------------------------------
diff --git a/flink-test-utils-parent/flink-test-utils-junit/pom.xml b/flink-test-utils-parent/flink-test-utils-junit/pom.xml
new file mode 100644
index 0000000..78e33ab
--- /dev/null
+++ b/flink-test-utils-parent/flink-test-utils-junit/pom.xml
@@ -0,0 +1,46 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-test-utils-parent</artifactId>
+ <version>1.1-SNAPSHOT</version>
+ <relativePath>..</relativePath>
+ </parent>
+
+ <artifactId>flink-test-utils-junit</artifactId>
+ <name>flink-test-utils-junit</name>
+
+ <packaging>jar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>${junit.version}</version>
+ <scope>compile</scope>
+ </dependency>
+ </dependencies>
+
+</project>
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-test-utils-parent/flink-test-utils-junit/src/main/java/org/apache/flink/core/testutils/CommonTestUtils.java
----------------------------------------------------------------------
diff --git a/flink-test-utils-parent/flink-test-utils-junit/src/main/java/org/apache/flink/core/testutils/CommonTestUtils.java b/flink-test-utils-parent/flink-test-utils-junit/src/main/java/org/apache/flink/core/testutils/CommonTestUtils.java
new file mode 100644
index 0000000..a50a106
--- /dev/null
+++ b/flink-test-utils-parent/flink-test-utils-junit/src/main/java/org/apache/flink/core/testutils/CommonTestUtils.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.core.testutils;
+
+import java.io.BufferedWriter;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+
+/**
+ * This class contains reusable utility methods for unit tests.
+ */
+public class CommonTestUtils {
+
+ /**
+ * Reads the path to the directory for temporary files from the configuration and returns it.
+ *
+ * @return the path to the directory for temporary files
+ */
+ public static String getTempDir() {
+ return System.getProperty("java.io.tmpdir");
+ }
+
+ /**
+ * Creates a copy of an object via Java Serialization.
+ *
+ * @param original The original object.
+ * @return The copied object.
+ */
+ public static <T extends java.io.Serializable> T createCopySerializable(T original) throws IOException {
+ if (original == null) {
+ throw new IllegalArgumentException();
+ }
+
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ ObjectOutputStream oos = new ObjectOutputStream(baos);
+ oos.writeObject(original);
+ oos.close();
+ baos.close();
+
+ ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
+
+ try (ObjectInputStream ois = new ObjectInputStream(bais)) {
+ @SuppressWarnings("unchecked")
+ T copy = (T) ois.readObject();
+ return copy;
+ }
+ catch (ClassNotFoundException e) {
+ throw new IOException(e);
+ }
+ }
+
+ /**
+ * Creates a temporary file that contains the given string.
+ * The file is written with the platform's default encoding.
+ *
+ * <p>The temp file is automatically deleted on JVM exit.
+ *
+ * @param contents The contents to be written to the file.
+ * @return The temp file URI.
+ */
+ public static String createTempFile(String contents) throws IOException {
+ File f = File.createTempFile("flink_test_", ".tmp");
+ f.deleteOnExit();
+
+ try (BufferedWriter out = new BufferedWriter(new FileWriter(f))) {
+ out.write(contents);
+ }
+ return f.toURI().toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-test-utils-parent/flink-test-utils-junit/src/main/java/org/apache/flink/core/testutils/MultiShotLatch.java
----------------------------------------------------------------------
diff --git a/flink-test-utils-parent/flink-test-utils-junit/src/main/java/org/apache/flink/core/testutils/MultiShotLatch.java b/flink-test-utils-parent/flink-test-utils-junit/src/main/java/org/apache/flink/core/testutils/MultiShotLatch.java
new file mode 100644
index 0000000..f532f2f
--- /dev/null
+++ b/flink-test-utils-parent/flink-test-utils-junit/src/main/java/org/apache/flink/core/testutils/MultiShotLatch.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.core.testutils;
+
+/**
+ * Latch for synchronizing parts of code in tests. In contrast to
+ * {@link OneShotLatch} this will reset the state once {@link #await()} returns.
+ *
+ * <p>A part of the code that should only run after other code calls {@link #await()}. The call
+ * will only return once the other part is finished and calls {@link #trigger()}.
+ */
+public final class MultiShotLatch {
+
+ private final Object lock = new Object();
+
+ private volatile boolean triggered;
+
+ /**
+ * Fires the latch. Code that is blocked on {@link #await()} will now return.
+ */
+ public void trigger() {
+ synchronized (lock) {
+ triggered = true;
+ lock.notifyAll();
+ }
+ }
+
+ /**
+ * Waits until {@link #trigger())} is called.
+ */
+ public void await() throws InterruptedException {
+ synchronized (lock) {
+ while (!triggered) {
+ lock.wait();
+ }
+ triggered = false;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-test-utils-parent/flink-test-utils-junit/src/main/java/org/apache/flink/core/testutils/OneShotLatch.java
----------------------------------------------------------------------
diff --git a/flink-test-utils-parent/flink-test-utils-junit/src/main/java/org/apache/flink/core/testutils/OneShotLatch.java b/flink-test-utils-parent/flink-test-utils-junit/src/main/java/org/apache/flink/core/testutils/OneShotLatch.java
new file mode 100644
index 0000000..d802860
--- /dev/null
+++ b/flink-test-utils-parent/flink-test-utils-junit/src/main/java/org/apache/flink/core/testutils/OneShotLatch.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.core.testutils;
+
+/**
+ * Latch for synchronizing parts of code in tests. Once the latch has fired once calls to
+ * {@link #await()} will return immediately in the future.
+ *
+ * <p>A part of the code that should only run after other code calls {@link #await()}. The call
+ * will only return once the other part is finished and calls {@link #trigger()}.
+ */
+public final class OneShotLatch {
+
+ private final Object lock = new Object();
+
+ private volatile boolean triggered;
+
+ /**
+ * Fires the latch. Code that is blocked on {@link #await()} will now return.
+ */
+ public void trigger() {
+ synchronized (lock) {
+ triggered = true;
+ lock.notifyAll();
+ }
+ }
+
+ /**
+ * Waits until {@link #trigger())} is called. Once {@code trigger()} has been called this
+ * call will always return immediately.
+ */
+ public void await() throws InterruptedException {
+ synchronized (lock) {
+ while (!triggered) {
+ lock.wait();
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-test-utils-parent/flink-test-utils-junit/src/main/java/org/apache/flink/testutils/junit/RetryOnException.java
----------------------------------------------------------------------
diff --git a/flink-test-utils-parent/flink-test-utils-junit/src/main/java/org/apache/flink/testutils/junit/RetryOnException.java b/flink-test-utils-parent/flink-test-utils-junit/src/main/java/org/apache/flink/testutils/junit/RetryOnException.java
new file mode 100644
index 0000000..fdf409a
--- /dev/null
+++ b/flink-test-utils-parent/flink-test-utils-junit/src/main/java/org/apache/flink/testutils/junit/RetryOnException.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.testutils.junit;
+
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Annotation to use with {@link org.apache.flink.testutils.junit.RetryRule}.
+ *
+ * <p>Add the {@link org.apache.flink.testutils.junit.RetryRule} to your test and
+ * annotate tests with {@link RetryOnException}.
+ *
+ * <pre>
+ * public class YourTest {
+ *
+ * {@literal @}Rule
+ * public RetryRule retryRule = new RetryRule();
+ *
+ * {@literal @}Test
+ * {@literal @}RetryOnException(times=1, exception=IOException.class)
+ * public void yourTest() throws Exception {
+ * // This will be retried 1 time (total runs 2) before failing the test.
+ * throw new IOException("Failing test");
+ * }
+ *
+ * {@literal @}Test
+ * {@literal @}RetryOnException(times=1, exception=IOException.class)
+ * public void yourTest() throws Exception {
+ * // This will not be retried, because it throws the wrong exception
+ * throw new IllegalStateException("Failing test");
+ * }
+ * }
+ * </pre>
+ */
+@Retention(RetentionPolicy.RUNTIME)
+@Target(java.lang.annotation.ElementType.METHOD)
+public @interface RetryOnException {
+
+ int times();
+
+ Class<? extends Throwable> exception();
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-test-utils-parent/flink-test-utils-junit/src/main/java/org/apache/flink/testutils/junit/RetryOnFailure.java
----------------------------------------------------------------------
diff --git a/flink-test-utils-parent/flink-test-utils-junit/src/main/java/org/apache/flink/testutils/junit/RetryOnFailure.java b/flink-test-utils-parent/flink-test-utils-junit/src/main/java/org/apache/flink/testutils/junit/RetryOnFailure.java
new file mode 100644
index 0000000..42b8ef6
--- /dev/null
+++ b/flink-test-utils-parent/flink-test-utils-junit/src/main/java/org/apache/flink/testutils/junit/RetryOnFailure.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.testutils.junit;
+
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Annotation to use with {@link RetryRule}.
+ *
+ * <p>Add the {@link RetryRule} to your test and annotate tests with {@link RetryOnFailure}.
+ *
+ * <pre>
+ * public class YourTest {
+ *
+ * {@literal @}Rule
+ * public RetryRule retryRule = new RetryRule();
+ *
+ * {@literal @}Test
+ * {@literal @}RetryOnFailure(times=1)
+ * public void yourTest() {
+ * // This will be retried 1 time (total runs 2) before failing the test.
+ * throw new Exception("Failing test");
+ * }
+ * }
+ * </pre>
+ */
+@Retention(RetentionPolicy.RUNTIME)
+@Target({ java.lang.annotation.ElementType.METHOD })
+public @interface RetryOnFailure {
+ int times();
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-test-utils-parent/flink-test-utils-junit/src/main/java/org/apache/flink/testutils/junit/RetryRule.java
----------------------------------------------------------------------
diff --git a/flink-test-utils-parent/flink-test-utils-junit/src/main/java/org/apache/flink/testutils/junit/RetryRule.java b/flink-test-utils-parent/flink-test-utils-junit/src/main/java/org/apache/flink/testutils/junit/RetryRule.java
new file mode 100644
index 0000000..fa36b5b
--- /dev/null
+++ b/flink-test-utils-parent/flink-test-utils-junit/src/main/java/org/apache/flink/testutils/junit/RetryRule.java
@@ -0,0 +1,175 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.testutils.junit;
+
+import org.junit.Test;
+import org.junit.rules.TestRule;
+import org.junit.runner.Description;
+import org.junit.runners.model.Statement;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A rule to retry failed tests for a fixed number of times.
+ *
+ * <p>Add the {@link RetryRule} to your test and annotate tests with {@link RetryOnFailure}.
+ *
+ * <pre>
+ * public class YourTest {
+ *
+ * {@literal @}Rule
+ * public RetryRule retryRule = new RetryRule();
+ *
+ * {@literal @}Test
+ * {@literal @}RetryOnFailure(times=1)
+ * public void yourTest() {
+ * // This will be retried 1 time (total runs 2) before failing the test.
+ * throw new Exception("Failing test");
+ * }
+ * }
+ * </pre>
+ */
+public class RetryRule implements TestRule {
+
+ public final static Logger LOG = LoggerFactory.getLogger(RetryRule.class);
+
+ @Override
+ public Statement apply(Statement statement, Description description) {
+ RetryOnFailure retryOnFailure = description.getAnnotation(RetryOnFailure.class);
+ RetryOnException retryOnException = description.getAnnotation(RetryOnException.class);
+
+ // sanity check that we don't use expected exceptions with the RetryOnX annotations
+ if (retryOnFailure != null || retryOnException != null) {
+ Test test = description.getAnnotation(Test.class);
+ if (test.expected() != Test.None.class) {
+ throw new IllegalArgumentException("You cannot combine the RetryOnFailure " +
+ "annotation with the Test(expected) annotation.");
+ }
+ }
+
+ // sanity check that we don't use both annotations
+ if (retryOnFailure != null && retryOnException != null) {
+ throw new IllegalArgumentException(
+ "You cannot combine the RetryOnFailure and RetryOnException annotations.");
+ }
+
+ if (retryOnFailure != null) {
+ return new RetryOnFailureStatement(retryOnFailure.times(), statement);
+ }
+ else if (retryOnException != null) {
+ return new RetryOnExceptionStatement(retryOnException.times(), retryOnException.exception(), statement);
+ }
+ else {
+ return statement;
+ }
+ }
+
+ /**
+ * Retries a test in case of a failure.
+ */
+ private static class RetryOnFailureStatement extends Statement {
+
+ private final int timesOnFailure;
+
+ private int currentRun;
+
+ private final Statement statement;
+
+ private RetryOnFailureStatement(int timesOnFailure, Statement statement) {
+ if (timesOnFailure < 0) {
+ throw new IllegalArgumentException("Negatives number of retries on failure");
+ }
+ this.timesOnFailure = timesOnFailure;
+ this.statement = statement;
+ }
+
+ /**
+ * Retry a test in case of a failure.
+ *
+ * @throws Throwable
+ */
+ @Override
+ public void evaluate() throws Throwable {
+ for (currentRun = 0; currentRun <= timesOnFailure; currentRun++) {
+ try {
+ statement.evaluate();
+ break; // success
+ }
+ catch (Throwable t) {
+ LOG.warn(String.format("Test run failed (%d/%d).",
+ currentRun, timesOnFailure + 1), t);
+
+ // Throw the failure if retried too often
+ if (currentRun == timesOnFailure) {
+ throw t;
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * Retries a test in case of a failure.
+ */
+ private static class RetryOnExceptionStatement extends Statement {
+
+ private final Class<? extends Throwable> exceptionClass;
+ private final int timesOnFailure;
+ private final Statement statement;
+
+ private int currentRun;
+
+ private RetryOnExceptionStatement(int timesOnFailure, Class<? extends Throwable> exceptionClass, Statement statement) {
+ if (timesOnFailure < 0) {
+ throw new IllegalArgumentException("Negatives number of retries on failure");
+ }
+ if (exceptionClass == null) {
+ throw new NullPointerException("exceptionClass");
+ }
+
+ this.exceptionClass = (exceptionClass);
+ this.timesOnFailure = timesOnFailure;
+ this.statement = statement;
+ }
+
+ /**
+ * Retry a test in case of a failure with a specific exception
+ *
+ * @throws Throwable
+ */
+ @Override
+ public void evaluate() throws Throwable {
+ for (currentRun = 0; currentRun <= timesOnFailure; currentRun++) {
+ try {
+ statement.evaluate();
+ break; // success
+ }
+ catch (Throwable t) {
+ LOG.warn(String.format("Test run failed (%d/%d).", currentRun, timesOnFailure + 1), t);
+
+ if (!exceptionClass.isAssignableFrom(t.getClass()) || currentRun >= timesOnFailure) {
+ // Throw the failure if retried too often, or if it is the wrong exception
+ throw t;
+ }
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-test-utils-parent/flink-test-utils-junit/src/main/java/org/apache/flink/util/TestLogger.java
----------------------------------------------------------------------
diff --git a/flink-test-utils-parent/flink-test-utils-junit/src/main/java/org/apache/flink/util/TestLogger.java b/flink-test-utils-parent/flink-test-utils-junit/src/main/java/org/apache/flink/util/TestLogger.java
new file mode 100644
index 0000000..df82d7b
--- /dev/null
+++ b/flink-test-utils-parent/flink-test-utils-junit/src/main/java/org/apache/flink/util/TestLogger.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.util;
+
+import org.junit.Rule;
+import org.junit.rules.TestRule;
+import org.junit.rules.TestWatcher;
+import org.junit.runner.Description;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.PrintWriter;
+import java.io.StringWriter;
+
+/**
+ * Adds automatic test name logging. Every test which wants to log which test is currently
+ * executed and why it failed, simply has to extend this class.
+ */
+public class TestLogger {
+ protected final Logger log = LoggerFactory.getLogger(getClass());
+
+ @Rule
+ public TestRule watchman = new TestWatcher() {
+
+ @Override
+ public void starting(Description description) {
+ log.info("\n================================================================================"
+ + "\nTest {} is running."
+ + "\n--------------------------------------------------------------------------------"
+ , description);
+ }
+
+ @Override
+ public void succeeded(Description description) {
+ log.info("\n--------------------------------------------------------------------------------"
+ + "\nTest {} successfully run."
+ + "\n================================================================================"
+ , description);
+ }
+
+ @Override
+ public void failed(Throwable e, Description description) {
+ log.error("\n--------------------------------------------------------------------------------"
+ + "\nTest {} failed with:\n{}"
+ + "\n================================================================================"
+ , description, exceptionToString(e));
+ }
+ };
+
+ private static String exceptionToString(Throwable t) {
+ if (t == null) {
+ return "(null)";
+ }
+
+ try {
+ StringWriter stm = new StringWriter();
+ PrintWriter wrt = new PrintWriter(stm);
+ t.printStackTrace(wrt);
+ wrt.close();
+ return stm.toString();
+ }
+ catch (Throwable ignored) {
+ return t.getClass().getName() + " (error while printing stack trace)";
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-test-utils-parent/flink-test-utils-junit/src/test/java/org/apache/flink/testutils/junit/RetryOnExceptionTest.java
----------------------------------------------------------------------
diff --git a/flink-test-utils-parent/flink-test-utils-junit/src/test/java/org/apache/flink/testutils/junit/RetryOnExceptionTest.java b/flink-test-utils-parent/flink-test-utils-junit/src/test/java/org/apache/flink/testutils/junit/RetryOnExceptionTest.java
new file mode 100644
index 0000000..a7a6f4b
--- /dev/null
+++ b/flink-test-utils-parent/flink-test-utils-junit/src/test/java/org/apache/flink/testutils/junit/RetryOnExceptionTest.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.testutils.junit;
+
+import org.junit.AfterClass;
+import org.junit.Rule;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class RetryOnExceptionTest {
+
+ @Rule
+ public RetryRule retryRule = new RetryRule();
+
+ private static final int NUMBER_OF_RUNS = 3;
+
+ private static int runsForSuccessfulTest = 0;
+
+ private static int runsForTestWithMatchingException = 0;
+
+ private static int runsForTestWithSubclassException = 0;
+
+ private static int runsForPassAfterOneFailure = 0;
+
+
+ @AfterClass
+ public static void verify() {
+ assertEquals(NUMBER_OF_RUNS + 1, runsForTestWithMatchingException);
+ assertEquals(NUMBER_OF_RUNS + 1, runsForTestWithSubclassException);
+ assertEquals(1, runsForSuccessfulTest);
+ assertEquals(2, runsForPassAfterOneFailure);
+ }
+
+ @Test
+ @RetryOnException(times = NUMBER_OF_RUNS, exception = IllegalArgumentException.class)
+ public void testSuccessfulTest() {
+ runsForSuccessfulTest++;
+ }
+
+ @Test
+ @RetryOnException(times = NUMBER_OF_RUNS, exception = IllegalArgumentException.class)
+ public void testMatchingException() {
+ runsForTestWithMatchingException++;
+ if (runsForTestWithMatchingException <= NUMBER_OF_RUNS) {
+ throw new IllegalArgumentException();
+ }
+ }
+
+ @Test
+ @RetryOnException(times = NUMBER_OF_RUNS, exception = RuntimeException.class)
+ public void testSubclassException() {
+ runsForTestWithSubclassException++;
+ if (runsForTestWithSubclassException <= NUMBER_OF_RUNS) {
+ throw new IllegalArgumentException();
+ }
+ }
+
+ @Test
+ @RetryOnException(times = NUMBER_OF_RUNS, exception = IllegalArgumentException.class)
+ public void testPassAfterOneFailure() {
+ runsForPassAfterOneFailure++;
+ if (runsForPassAfterOneFailure == 1) {
+ throw new IllegalArgumentException();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-test-utils-parent/flink-test-utils-junit/src/test/java/org/apache/flink/testutils/junit/RetryOnFailureTest.java
----------------------------------------------------------------------
diff --git a/flink-test-utils-parent/flink-test-utils-junit/src/test/java/org/apache/flink/testutils/junit/RetryOnFailureTest.java b/flink-test-utils-parent/flink-test-utils-junit/src/test/java/org/apache/flink/testutils/junit/RetryOnFailureTest.java
new file mode 100644
index 0000000..efc7ba4
--- /dev/null
+++ b/flink-test-utils-parent/flink-test-utils-junit/src/test/java/org/apache/flink/testutils/junit/RetryOnFailureTest.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.testutils.junit;
+
+import org.junit.AfterClass;
+import org.junit.Rule;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class RetryOnFailureTest {
+
+ @Rule
+ public RetryRule retryRule = new RetryRule();
+
+ private static final int NUMBER_OF_RUNS = 5;
+
+ private static int numberOfFailedRuns;
+
+ private static int numberOfSuccessfulRuns;
+
+ private static boolean firstRun = true;
+
+ @AfterClass
+ public static void verify() throws Exception {
+ assertEquals(NUMBER_OF_RUNS + 1, numberOfFailedRuns);
+ assertEquals(3, numberOfSuccessfulRuns);
+ }
+
+ @Test
+ @RetryOnFailure(times = NUMBER_OF_RUNS)
+ public void testRetryOnFailure() throws Exception {
+ // All but the (expected) last run should be successful
+ if (numberOfFailedRuns < NUMBER_OF_RUNS) {
+ numberOfFailedRuns++;
+ throw new RuntimeException("Expected test exception");
+ }
+ else {
+ numberOfSuccessfulRuns++;
+ }
+ }
+
+ @Test
+ @RetryOnFailure(times = NUMBER_OF_RUNS)
+ public void testRetryOnceOnFailure() throws Exception {
+ if (firstRun) {
+ numberOfFailedRuns++;
+ firstRun = false;
+ throw new RuntimeException("Expected test exception");
+ }
+ else {
+ numberOfSuccessfulRuns++;
+ }
+ }
+
+ @Test
+ @RetryOnFailure(times = NUMBER_OF_RUNS)
+ public void testDontRetryOnSuccess() throws Exception {
+ numberOfSuccessfulRuns++;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-test-utils-parent/flink-test-utils/pom.xml
----------------------------------------------------------------------
diff --git a/flink-test-utils-parent/flink-test-utils/pom.xml b/flink-test-utils-parent/flink-test-utils/pom.xml
new file mode 100644
index 0000000..3c35cf1
--- /dev/null
+++ b/flink-test-utils-parent/flink-test-utils/pom.xml
@@ -0,0 +1,224 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-test-utils-parent</artifactId>
+ <version>1.1-SNAPSHOT</version>
+ <relativePath>..</relativePath>
+ </parent>
+
+ <artifactId>flink-test-utils_2.10</artifactId>
+ <name>flink-test-utils</name>
+
+ <packaging>jar</packaging>
+
+ <dependencies>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-test-utils-junit</artifactId>
+ <version>${project.version}</version>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-runtime_2.10</artifactId>
+ <version>${project.version}</version>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-clients_2.10</artifactId>
+ <version>${project.version}</version>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>${junit.version}</version>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-test</artifactId>
+ <version>${curator.version}</version>
+ <scope>compile</scope>
+ </dependency>
+
+ </dependencies>
+
+ <build>
+ <plugins>
+ <!-- Scala Compiler -->
+ <plugin>
+ <groupId>net.alchim31.maven</groupId>
+ <artifactId>scala-maven-plugin</artifactId>
+ <version>3.1.4</version>
+ <executions>
+ <!-- Run scala compiler in the process-resources phase, so that dependencies
+ on scala classes can be resolved later in the (Java) compile phase -->
+ <execution>
+ <id>scala-compile-first</id>
+ <phase>process-resources</phase>
+ <goals>
+ <goal>compile</goal>
+ </goals>
+ </execution>
+
+ <!-- Run scala compiler in the process-test-resources phase, so that
+ dependencies on scala classes can be resolved later in the (Java) test-compile
+ phase -->
+ <execution>
+ <id>scala-test-compile</id>
+ <phase>process-test-resources</phase>
+ <goals>
+ <goal>testCompile</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <jvmArgs>
+ <jvmArg>-Xms128m</jvmArg>
+ <jvmArg>-Xmx512m</jvmArg>
+ </jvmArgs>
+ </configuration>
+ </plugin>
+
+ <!-- Eclipse Integration -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-eclipse-plugin</artifactId>
+ <version>2.8</version>
+ <configuration>
+ <downloadSources>true</downloadSources>
+ <projectnatures>
+ <projectnature>org.scala-ide.sdt.core.scalanature</projectnature>
+ <projectnature>org.eclipse.jdt.core.javanature</projectnature>
+ </projectnatures>
+ <buildcommands>
+ <buildcommand>org.scala-ide.sdt.core.scalabuilder</buildcommand>
+ </buildcommands>
+ <classpathContainers>
+ <classpathContainer>org.scala-ide.sdt.launching.SCALA_CONTAINER</classpathContainer>
+ <classpathContainer>org.eclipse.jdt.launching.JRE_CONTAINER</classpathContainer>
+ </classpathContainers>
+ <excludes>
+ <exclude>org.scala-lang:scala-library</exclude>
+ <exclude>org.scala-lang:scala-compiler</exclude>
+ </excludes>
+ <sourceIncludes>
+ <sourceInclude>**/*.scala</sourceInclude>
+ <sourceInclude>**/*.java</sourceInclude>
+ </sourceIncludes>
+ </configuration>
+ </plugin>
+
+ <!-- Adding scala source directories to build path -->
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <version>1.7</version>
+ <executions>
+ <!-- Add src/main/scala to eclipse build path -->
+ <execution>
+ <id>add-source</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>add-source</goal>
+ </goals>
+ <configuration>
+ <sources>
+ <source>src/main/scala</source>
+ </sources>
+ </configuration>
+ </execution>
+ <!-- Add src/test/scala to eclipse build path -->
+ <execution>
+ <id>add-test-source</id>
+ <phase>generate-test-sources</phase>
+ <goals>
+ <goal>add-test-source</goal>
+ </goals>
+ <configuration>
+ <sources>
+ <source>src/test/scala</source>
+ </sources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <!-- Scala Code Style, most of the configuration done via plugin management -->
+ <plugin>
+ <groupId>org.scalastyle</groupId>
+ <artifactId>scalastyle-maven-plugin</artifactId>
+ <configuration>
+ <configLocation>${project.basedir}/../../tools/maven/scalastyle-config.xml</configLocation>
+ </configuration>
+ </plugin>
+
+ </plugins>
+ <pluginManagement>
+ <plugins>
+ <!--This plugin's configuration is used to store Eclipse m2e settings only. It has no influence on the Maven build itself.-->
+ <plugin>
+ <groupId>org.eclipse.m2e</groupId>
+ <artifactId>lifecycle-mapping</artifactId>
+ <version>1.0.0</version>
+ <configuration>
+ <lifecycleMappingMetadata>
+ <pluginExecutions>
+ <pluginExecution>
+ <pluginExecutionFilter>
+ <groupId>
+ net.alchim31.maven
+ </groupId>
+ <artifactId>
+ scala-maven-plugin
+ </artifactId>
+ <versionRange>
+ [3.1.4,)
+ </versionRange>
+ <goals>
+ <goal>compile</goal>
+ <goal>testCompile</goal>
+ </goals>
+ </pluginExecutionFilter>
+ <action>
+ <ignore/>
+ </action>
+ </pluginExecution>
+ </pluginExecutions>
+ </lifecycleMappingMetadata>
+ </configuration>
+ </plugin>
+ </plugins>
+ </pluginManagement>
+ </build>
+</project>
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/testdata/ConnectedComponentsData.java
----------------------------------------------------------------------
diff --git a/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/testdata/ConnectedComponentsData.java b/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/testdata/ConnectedComponentsData.java
new file mode 100644
index 0000000..60cd319
--- /dev/null
+++ b/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/testdata/ConnectedComponentsData.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.test.testdata;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.util.List;
+import java.util.Random;
+import java.util.regex.Pattern;
+
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.junit.Assert;
+
+
+public class ConnectedComponentsData {
+
+ public static final String getEnumeratingVertices(int num) {
+ if (num < 1 || num > 1000000) {
+ throw new IllegalArgumentException();
+ }
+
+ StringBuilder bld = new StringBuilder(3 * num);
+ for (int i = 1; i <= num; i++) {
+ bld.append(i);
+ bld.append('\n');
+ }
+ return bld.toString();
+ }
+
+ /**
+ * Creates random edges such that even numbered vertices are connected with even numbered vertices
+ * and odd numbered vertices only with other odd numbered ones.
+ */
+ public static final String getRandomOddEvenEdges(int numEdges, int numVertices, long seed) {
+ if (numVertices < 2 || numVertices > 1000000 || numEdges < numVertices || numEdges > 1000000) {
+ throw new IllegalArgumentException();
+ }
+
+ StringBuilder bld = new StringBuilder(5 * numEdges);
+
+ // first create the linear edge sequence even -> even and odd -> odd to make sure they are
+ // all in the same component
+ for (int i = 3; i <= numVertices; i++) {
+ bld.append(i - 2).append(' ').append(i).append('\n');
+ }
+
+ numEdges -= numVertices - 2;
+ Random r = new Random(seed);
+
+ for (int i = 1; i <= numEdges; i++) {
+ int evenOdd = r.nextBoolean() ? 1 : 0;
+
+ int source = r.nextInt(numVertices) + 1;
+ if (source % 2 != evenOdd) {
+ source--;
+ if (source < 1) {
+ source = 2;
+ }
+ }
+
+ int target = r.nextInt(numVertices) + 1;
+ if (target % 2 != evenOdd) {
+ target--;
+ if (target < 1) {
+ target = 2;
+ }
+ }
+
+ bld.append(source).append(' ').append(target).append('\n');
+ }
+ return bld.toString();
+ }
+
+ public static void checkOddEvenResult(BufferedReader result) throws IOException {
+ Pattern split = Pattern.compile(" ");
+ String line;
+ while ((line = result.readLine()) != null) {
+ String[] res = split.split(line);
+ Assert.assertEquals("Malformed result: Wrong number of tokens in line.", 2, res.length);
+ try {
+ int vertex = Integer.parseInt(res[0]);
+ int component = Integer.parseInt(res[1]);
+
+ int should = vertex % 2;
+ if (should == 0) {
+ should = 2;
+ }
+ Assert.assertEquals("Vertex is in wrong component.", should, component);
+ } catch (NumberFormatException e) {
+ Assert.fail("Malformed result.");
+ }
+ }
+ }
+
+ public static void checkOddEvenResult(List<Tuple2<Long, Long>> lines) throws IOException {
+ for (Tuple2<Long, Long> line : lines) {
+ try {
+ long vertex = line.f0;
+ long component = line.f1;
+ long should = vertex % 2;
+ if (should == 0) {
+ should = 2;
+ }
+ Assert.assertEquals("Vertex is in wrong component.", should, component);
+ } catch (NumberFormatException e) {
+ Assert.fail("Malformed result.");
+ }
+ }
+ }
+
+ private ConnectedComponentsData() {}
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/testdata/EnumTriangleData.java
----------------------------------------------------------------------
diff --git a/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/testdata/EnumTriangleData.java b/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/testdata/EnumTriangleData.java
new file mode 100644
index 0000000..b170404
--- /dev/null
+++ b/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/testdata/EnumTriangleData.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.test.testdata;
+
+public class EnumTriangleData {
+
+ public static final String EDGES =
+ "1 2\n" +
+ "1 3\n" +
+ "1 4\n" +
+ "1 5\n" +
+ "2 3\n" +
+ "2 5\n" +
+ "3 4\n" +
+ "3 7\n" +
+ "5 6\n" +
+ "3 8\n" +
+ "7 8\n";
+
+ public static final String TRIANGLES_BY_ID =
+ "1,2,3\n" +
+ "1,3,4\n" +
+ "1,2,5\n" +
+ "3,7,8\n";
+
+ public static final String TRIANGLES_BY_DEGREE =
+ "2,1,3\n" +
+ "4,1,3\n" +
+ "2,1,5\n" +
+ "7,3,8\n";
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/testdata/KMeansData.java
----------------------------------------------------------------------
diff --git a/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/testdata/KMeansData.java b/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/testdata/KMeansData.java
new file mode 100644
index 0000000..1319e84
--- /dev/null
+++ b/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/testdata/KMeansData.java
@@ -0,0 +1,257 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.test.testdata;
+
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.StringTokenizer;
+
+import org.junit.Assert;
+
+
+public class KMeansData {
+
+ // --------------------------------------------------------------------------------------------
+ // 3-dimensional data
+ // --------------------------------------------------------------------------------------------
+
+ public final static String DATAPOINTS = "0|50.90|16.20|72.08|\n" + "1|73.65|61.76|62.89|\n" + "2|61.73|49.95|92.74|\n"
+ + "3|1.60|70.11|16.32|\n" + "4|2.43|19.81|89.56|\n" + "5|67.99|9.00|14.48|\n" + "6|87.80|84.49|55.83|\n"
+ + "7|90.26|42.99|53.29|\n" + "8|51.36|6.16|9.35|\n" + "9|12.43|9.52|12.54|\n" + "10|80.01|8.78|29.74|\n"
+ + "11|92.76|2.93|80.07|\n" + "12|46.32|100.00|22.98|\n" + "13|34.11|45.61|58.60|\n"
+ + "14|68.82|16.36|96.60|\n" + "15|81.47|76.45|28.40|\n" + "16|65.55|40.21|43.43|\n"
+ + "17|84.22|88.56|13.31|\n" + "18|36.99|68.36|57.12|\n" + "19|28.87|37.69|91.04|\n"
+ + "20|31.56|13.22|86.00|\n" + "21|18.49|34.45|54.52|\n" + "22|13.33|94.02|92.07|\n"
+ + "23|91.19|81.62|55.06|\n" + "24|85.78|39.02|25.58|\n" + "25|94.41|47.07|78.23|\n"
+ + "26|90.62|10.43|80.20|\n" + "27|31.52|85.81|39.79|\n" + "28|24.65|77.98|26.35|\n"
+ + "29|69.34|75.79|63.96|\n" + "30|22.56|78.61|66.66|\n" + "31|91.74|83.82|73.92|\n"
+ + "32|76.64|89.53|44.66|\n" + "33|36.02|73.01|92.32|\n" + "34|87.86|18.94|10.74|\n"
+ + "35|91.94|34.61|5.20|\n" + "36|12.52|47.01|95.29|\n" + "37|44.01|26.19|78.50|\n"
+ + "38|26.20|73.36|10.08|\n" + "39|15.21|17.37|54.33|\n" + "40|27.96|94.81|44.41|\n"
+ + "41|26.44|44.81|70.88|\n" + "42|53.29|26.69|2.40|\n" + "43|23.94|11.50|1.71|\n"
+ + "44|19.00|25.48|50.80|\n" + "45|82.26|1.88|58.08|\n" + "46|47.56|82.54|82.73|\n"
+ + "47|51.54|35.10|32.95|\n" + "48|86.71|55.51|19.08|\n" + "49|54.16|23.68|32.41|\n"
+ + "50|71.81|32.83|46.66|\n" + "51|20.70|14.19|64.96|\n" + "52|57.17|88.56|55.23|\n"
+ + "53|91.39|49.38|70.55|\n" + "54|47.90|62.07|76.03|\n" + "55|55.70|37.77|30.15|\n"
+ + "56|87.87|74.62|25.95|\n" + "57|95.70|45.04|15.27|\n" + "58|41.61|89.37|24.45|\n"
+ + "59|82.19|20.84|11.13|\n" + "60|49.88|2.62|18.62|\n" + "61|16.42|53.30|74.13|\n"
+ + "62|38.37|72.62|35.16|\n" + "63|43.26|49.59|92.56|\n" + "64|28.96|2.36|78.49|\n"
+ + "65|88.41|91.43|92.55|\n" + "66|98.61|79.58|33.03|\n" + "67|4.94|18.65|30.78|\n"
+ + "68|75.89|79.30|63.90|\n" + "69|93.18|76.26|9.50|\n" + "70|73.43|70.50|76.49|\n"
+ + "71|78.64|90.87|34.49|\n" + "72|58.47|63.07|8.82|\n" + "73|69.74|54.36|64.43|\n"
+ + "74|38.47|36.60|33.39|\n" + "75|51.07|14.75|2.54|\n" + "76|24.18|16.85|15.00|\n"
+ + "77|7.56|50.72|93.45|\n" + "78|64.28|97.01|57.31|\n" + "79|85.30|24.13|76.57|\n"
+ + "80|72.78|30.78|13.11|\n" + "81|18.42|17.45|32.20|\n" + "82|87.44|74.98|87.90|\n"
+ + "83|38.30|17.77|37.33|\n" + "84|63.62|7.90|34.23|\n" + "85|8.84|67.87|30.65|\n"
+ + "86|76.12|51.83|80.12|\n" + "87|32.30|74.79|4.39|\n" + "88|41.73|45.34|18.66|\n"
+ + "89|58.13|18.43|83.38|\n" + "90|98.10|33.46|83.07|\n" + "91|17.76|4.10|88.51|\n"
+ + "92|60.58|18.15|59.96|\n" + "93|50.11|33.25|85.64|\n" + "94|97.74|60.93|38.97|\n"
+ + "95|76.31|52.50|95.43|\n" + "96|7.71|85.85|36.26|\n" + "97|9.32|72.21|42.17|\n"
+ + "98|71.29|51.88|57.62|\n" + "99|31.39|7.27|88.74|";
+
+ public static final String INITIAL_CENTERS =
+ "0|1.96|65.04|20.82|\n" +
+ "1|53.99|84.23|81.59|\n" +
+ "2|97.28|74.50|40.32|\n" +
+ "3|63.57|24.53|87.07|\n" +
+ "4|28.10|43.27|86.53|\n" +
+ "5|99.51|62.70|64.48|\n" +
+ "6|30.31|30.36|80.46|";
+
+ public static final String CENTERS_AFTER_ONE_STEP =
+ "0|28.47|54.80|21.88|\n" +
+ "1|52.74|80.10|73.03|\n" +
+ "2|83.92|60.45|25.17|\n" +
+ "3|70.73|20.18|67.06|\n" +
+ "4|22.51|47.19|86.23|\n" +
+ "5|82.70|53.79|68.68|\n" +
+ "6|29.74|19.17|59.16|";
+
+
+ public static final String CENTERS_AFTER_ONE_STEP_SINGLE_DIGIT =
+ "0|28.5|54.8|21.9|\n" +
+ "1|52.7|80.1|73.0|\n" +
+ "2|83.9|60.5|25.2|\n" +
+ "3|70.7|20.2|67.0|\n" +
+ "4|22.5|47.2|86.2|\n" +
+ "5|82.7|53.8|68.7|\n" +
+ "6|29.7|19.2|59.2|";
+
+
+ public static final String CENTERS_AFTER_20_ITERATIONS_SINGLE_DIGIT =
+ "0|38.3|54.5|19.3|\n" +
+ "1|32.1|83.0|50.4|\n" +
+ "2|87.5|56.6|20.3|\n" +
+ "3|75.4|18.6|67.5|\n" +
+ "4|24.9|29.2|77.6|\n" +
+ "5|78.7|66.1|70.8|\n" +
+ "6|39.5|14.0|18.7|\n";
+
+ public static final String CENTERS_AFTER_20_ITERATIONS_DOUBLE_DIGIT =
+ "0|38.25|54.52|19.34|\n" +
+ "1|32.14|83.04|50.35|\n" +
+ "2|87.48|56.57|20.27|\n" +
+ "3|75.40|18.65|67.49|\n" +
+ "4|24.93|29.25|77.56|\n" +
+ "5|78.67|66.07|70.82|\n" +
+ "6|39.51|14.04|18.74|\n";
+
+ // --------------------------------------------------------------------------------------------
+ // 2-dimensional data
+ // --------------------------------------------------------------------------------------------
+
+ public final static String DATAPOINTS_2D = "0|50.90|16.20|\n" + "1|73.65|61.76|\n" + "2|61.73|49.95|\n"
+ + "3|1.60|70.11|\n" + "4|2.43|19.81|\n" + "5|67.99|9.00|\n" + "6|87.80|84.49|\n"
+ + "7|90.26|42.99|\n" + "8|51.36|6.16|\n" + "9|12.43|9.52|\n" + "10|80.01|8.78|\n"
+ + "11|92.76|2.93|\n" + "12|46.32|100.00|\n"+ "13|34.11|45.61|\n"
+ + "14|68.82|16.36|\n" + "15|81.47|76.45|\n" + "16|65.55|40.21|\n"
+ + "17|84.22|88.56|\n" + "18|36.99|68.36|\n" + "19|28.87|37.69|\n"
+ + "20|31.56|13.22|\n" + "21|18.49|34.45|\n" + "22|13.33|94.02|\n"
+ + "23|91.19|81.62|\n" + "24|85.78|39.02|\n" + "25|94.41|47.07|\n"
+ + "26|90.62|10.43|\n" + "27|31.52|85.81|\n" + "28|24.65|77.98|\n"
+ + "29|69.34|75.79|\n" + "30|22.56|78.61|\n" + "31|91.74|83.82|\n"
+ + "32|76.64|89.53|\n" + "33|36.02|73.01|\n" + "34|87.86|18.94|\n"
+ + "35|91.94|34.61|\n" + "36|12.52|47.01|\n" + "37|44.01|26.19|\n"
+ + "38|26.20|73.36|\n" + "39|15.21|17.37|\n" + "40|27.96|94.81|\n"
+ + "41|26.44|44.81|\n" + "42|53.29|26.69|\n" + "43|23.94|11.50|n"
+ + "44|19.00|25.48|\n" + "45|82.26|1.88|\n" + "46|47.56|82.54|\n"
+ + "47|51.54|35.10|\n" + "48|86.71|55.51|\n" + "49|54.16|23.68|\n"
+ + "50|71.81|32.83|\n" + "51|20.70|14.19|\n" + "52|57.17|88.56|\n"
+ + "53|91.39|49.38|\n" + "54|47.90|62.07|\n" + "55|55.70|37.77|\n"
+ + "56|87.87|74.62|\n" + "57|95.70|45.04|\n" + "58|41.61|89.37|\n"
+ + "59|82.19|20.84|\n" + "60|49.88|2.62|\n" + "61|16.42|53.30|\n"
+ + "62|38.37|72.62|\n" + "63|43.26|49.59|\n" + "64|28.96|2.36|\n"
+ + "65|88.41|91.43|\n" + "66|98.61|79.58|\n" + "67|4.94|18.65|\n"
+ + "68|75.89|79.30|\n" + "69|93.18|76.26|\n" + "70|73.43|70.50|\n"
+ + "71|78.64|90.87|\n" + "72|58.47|63.07|\n" + "73|69.74|54.36|\n"
+ + "74|38.47|36.60|\n" + "75|51.07|14.75|\n" + "76|24.18|16.85|\n"
+ + "77|7.56|50.72|\n" + "78|64.28|97.01|\n" + "79|85.30|24.13|\n"
+ + "80|72.78|30.78|\n" + "81|18.42|17.45|\n" + "82|87.44|74.98|\n"
+ + "83|38.30|17.77|\n" + "84|63.62|7.90|\n" + "85|8.84|67.87|\n"
+ + "86|76.12|51.83|\n" + "87|32.30|74.79|n" + "88|41.73|45.34|\n"
+ + "89|58.13|18.43|\n" + "90|98.10|33.46|\n" + "91|17.76|4.10|\n"
+ + "92|60.58|18.15|\n" + "93|50.11|33.25|\n" + "94|97.74|60.93|\n"
+ + "95|76.31|52.50|\n" + "96|7.71|85.85|\n" + "97|9.32|72.21|\n"
+ + "98|71.29|51.88|\n" + "99|31.39|7.27|";
+
+ public static final String INITIAL_CENTERS_2D =
+ "0|1.96|65.04|\n" +
+ "1|53.99|84.23|\n" +
+ "2|97.28|74.50|\n" +
+ "3|63.57|24.53|\n" +
+ "4|28.10|43.27|\n" +
+ "5|99.51|62.70|\n" +
+ "6|30.31|30.36|";
+
+ public static final String CENTERS_2D_AFTER_SINGLE_ITERATION_DOUBLE_DIGIT =
+ "0|13.53|74.53|\n" +
+ "1|49.12|80.49|\n" +
+ "2|87.20|81.83|\n" +
+ "3|67.39|23.32|\n" +
+ "4|26.94|46.34|\n" +
+ "5|88.39|48.64|\n" +
+ "6|23.20|16.71|";
+
+ public static final String CENTERS_2D_AFTER_20_ITERATIONS_DOUBLE_DIGIT =
+ "0|15.80|79.42|\n" +
+ "1|43.11|78.20|\n" +
+ "2|83.13|82.18|\n" +
+ "3|65.70|18.49|\n" +
+ "4|25.13|44.42|\n" +
+ "5|82.90|48.16|\n" +
+ "6|20.79|13.08|";
+
+ // --------------------------------------------------------------------------------------------
+ // testing / verification
+ // --------------------------------------------------------------------------------------------
+
+ public static void checkResultsWithDelta(String expectedResults, List<String> resultLines, final double maxDelta) {
+
+ Comparator<String> deltaComp = new Comparator<String>() {
+
+ @Override
+ public int compare(String o1, String o2) {
+
+ StringTokenizer st1 = new StringTokenizer(o1, "|");
+ StringTokenizer st2 = new StringTokenizer(o2, "|");
+
+ if(st1.countTokens() != st2.countTokens()) {
+ return st1.countTokens() - st2.countTokens();
+ }
+
+ // first token is ID
+ String t1 = st1.nextToken();
+ String t2 = st2.nextToken();
+ if(!t1.equals(t2)) {
+ return t1.compareTo(t2);
+ }
+
+ while(st1.hasMoreTokens()) {
+ t1 = st1.nextToken();
+ t2 = st2.nextToken();
+
+ double d1 = Double.parseDouble(t1);
+ double d2 = Double.parseDouble(t2);
+
+ if (Math.abs(d1-d2) > maxDelta) {
+ return d1 < d2 ? -1 : 1;
+ }
+ }
+
+ return 0;
+ }
+ };
+
+ // ------- Test results -----------
+
+ Collections.sort(resultLines, deltaComp);
+
+ final String[] should = expectedResults.split("\n");
+ final String[] is = (String[]) resultLines.toArray(new String[resultLines.size()]);
+
+ Assert.assertEquals("Wrong number of result lines.", should.length, is.length);
+
+ for (int i = 0; i < should.length; i++) {
+ StringTokenizer shouldRecord = new StringTokenizer(should[i], "|");
+ StringTokenizer isRecord = new StringTokenizer(is[i], "|");
+
+ Assert.assertEquals("Records don't match.", shouldRecord.countTokens(), isRecord.countTokens());
+
+ // first token is ID
+ String shouldToken = shouldRecord.nextToken();
+ String isToken = isRecord.nextToken();
+
+ Assert.assertEquals("Records don't match.", shouldToken, isToken);
+
+ while (shouldRecord.hasMoreTokens()) {
+ shouldToken = shouldRecord.nextToken();
+ isToken = isRecord.nextToken();
+
+ double shouldDouble = Double.parseDouble(shouldToken);
+ double isDouble = Double.parseDouble(isToken);
+
+ Assert.assertTrue("Value " + isDouble + " is out of range of " + shouldDouble + " +/- " + maxDelta, shouldDouble - maxDelta <= isDouble && shouldDouble + maxDelta >= isDouble);
+ }
+ }
+ }
+
+
+ private KMeansData() {}
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/testdata/PageRankData.java
----------------------------------------------------------------------
diff --git a/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/testdata/PageRankData.java b/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/testdata/PageRankData.java
new file mode 100644
index 0000000..f91e611
--- /dev/null
+++ b/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/testdata/PageRankData.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.test.testdata;
+
+public class PageRankData {
+
+ public static final int NUM_VERTICES = 5;
+
+ public static final String VERTICES = "1\n" +
+ "2\n" +
+ "5\n" +
+ "3\n" +
+ "4";
+
+ public static final String EDGES = "2 1\n" +
+ "5 2\n" +
+ "5 4\n" +
+ "4 3\n" +
+ "4 2\n" +
+ "1 4\n" +
+ "1 2\n" +
+ "1 3\n" +
+ "3 5\n";
+
+
+ public static final String RANKS_AFTER_3_ITERATIONS = "1 0.237\n" +
+ "2 0.248\n" +
+ "3 0.173\n" +
+ "4 0.175\n" +
+ "5 0.165";
+
+
+ public static final String RANKS_AFTER_EPSILON_0_0001_CONVERGENCE = "1 0.238\n" +
+ "2 0.244\n" +
+ "3 0.170\n" +
+ "4 0.171\n" +
+ "5 0.174";
+
+ private PageRankData() {}
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/testdata/TransitiveClosureData.java
----------------------------------------------------------------------
diff --git a/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/testdata/TransitiveClosureData.java b/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/testdata/TransitiveClosureData.java
new file mode 100644
index 0000000..652b195
--- /dev/null
+++ b/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/testdata/TransitiveClosureData.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.test.testdata;
+
+import org.junit.Assert;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.util.regex.Pattern;
+
+public class TransitiveClosureData {
+
+ public static void checkOddEvenResult(BufferedReader result) throws IOException {
+ Pattern split = Pattern.compile(" ");
+ String line;
+ while ((line = result.readLine()) != null) {
+ String[] res = split.split(line);
+ Assert.assertEquals("Malformed result: Wrong number of tokens in line.", 2, res.length);
+ try {
+ int from = Integer.parseInt(res[0]);
+ int to = Integer.parseInt(res[1]);
+
+ Assert.assertEquals("Vertex should not be reachable.", from % 2, to % 2);
+ } catch (NumberFormatException e) {
+ Assert.fail("Malformed result.");
+ }
+ }
+ }
+
+ private TransitiveClosureData() {}
+}
[10/15] flink git commit: [FLINK-3995] [build] flink-test-utils also
contains the streaming test utilities.
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/b9f42e91/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/StreamTaskTimerITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/StreamTaskTimerITCase.java b/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/StreamTaskTimerITCase.java
new file mode 100644
index 0000000..33c8024
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/StreamTaskTimerITCase.java
@@ -0,0 +1,339 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.test.streaming.runtime;
+
+import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
+import org.apache.flink.runtime.client.JobExecutionException;
+import org.apache.flink.streaming.api.TimeCharacteristic;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.functions.source.SourceFunction;
+import org.apache.flink.streaming.api.operators.AbstractStreamOperator;
+import org.apache.flink.streaming.api.operators.ChainingStrategy;
+import org.apache.flink.streaming.api.operators.OneInputStreamOperator;
+import org.apache.flink.streaming.api.operators.TwoInputStreamOperator;
+import org.apache.flink.streaming.api.watermark.Watermark;
+import org.apache.flink.streaming.runtime.operators.Triggerable;
+import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
+import org.apache.flink.streaming.runtime.tasks.TimerException;
+import org.apache.flink.streaming.util.StreamingMultipleProgramsTestBase;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.concurrent.Semaphore;
+
+/**
+ * Tests for the timer service of {@code StreamTask}.
+ *
+ * <p>
+ * These tests ensure that exceptions are properly forwarded from the timer thread to
+ * the task thread and that operator methods are not invoked concurrently.
+ */
+@RunWith(Parameterized.class)
+public class StreamTaskTimerITCase extends StreamingMultipleProgramsTestBase {
+
+ private final TimeCharacteristic timeCharacteristic;
+
+ public StreamTaskTimerITCase(TimeCharacteristic characteristic) {
+ timeCharacteristic = characteristic;
+ }
+
+
+ /**
+ * Note: this test fails if we don't check for exceptions in the source contexts and do not
+ * synchronize in the source contexts.
+ */
+ @Test
+ public void testOperatorChainedToSource() throws Exception {
+
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+ env.setStreamTimeCharacteristic(timeCharacteristic);
+ env.setParallelism(1);
+
+ DataStream<String> source = env.addSource(new InfiniteTestSource());
+
+ source.transform("Custom Operator", BasicTypeInfo.STRING_TYPE_INFO, new TimerOperator(ChainingStrategy.ALWAYS));
+
+ boolean testSuccess = false;
+ try {
+ env.execute("Timer test");
+ } catch (JobExecutionException e) {
+ if (e.getCause() instanceof TimerException) {
+ TimerException te = (TimerException) e.getCause();
+ if (te.getCause() instanceof RuntimeException) {
+ RuntimeException re = (RuntimeException) te.getCause();
+ if (re.getMessage().equals("TEST SUCCESS")) {
+ testSuccess = true;
+ } else {
+ throw e;
+ }
+ } else {
+ throw e;
+ }
+ } else {
+ throw e;
+ }
+ }
+ Assert.assertTrue(testSuccess);
+ }
+
+ /**
+ * Note: this test fails if we don't check for exceptions in the source contexts and do not
+ * synchronize in the source contexts.
+ */
+ @Test
+ public void testOneInputOperatorWithoutChaining() throws Exception {
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+ env.setStreamTimeCharacteristic(timeCharacteristic);
+ env.setParallelism(1);
+
+ DataStream<String> source = env.addSource(new InfiniteTestSource());
+
+ source.transform("Custom Operator", BasicTypeInfo.STRING_TYPE_INFO, new TimerOperator(ChainingStrategy.NEVER));
+
+ boolean testSuccess = false;
+ try {
+ env.execute("Timer test");
+ } catch (JobExecutionException e) {
+ if (e.getCause() instanceof TimerException) {
+ TimerException te = (TimerException) e.getCause();
+ if (te.getCause() instanceof RuntimeException) {
+ RuntimeException re = (RuntimeException) te.getCause();
+ if (re.getMessage().equals("TEST SUCCESS")) {
+ testSuccess = true;
+ } else {
+ throw e;
+ }
+ } else {
+ throw e;
+ }
+ } else {
+ throw e;
+ }
+ }
+ Assert.assertTrue(testSuccess);
+ }
+
+ @Test
+ public void testTwoInputOperatorWithoutChaining() throws Exception {
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+ env.setStreamTimeCharacteristic(timeCharacteristic);
+ env.setParallelism(1);
+
+ DataStream<String> source = env.addSource(new InfiniteTestSource());
+
+ source.connect(source).transform(
+ "Custom Operator",
+ BasicTypeInfo.STRING_TYPE_INFO,
+ new TwoInputTimerOperator(ChainingStrategy.NEVER));
+
+ boolean testSuccess = false;
+ try {
+ env.execute("Timer test");
+ } catch (JobExecutionException e) {
+ if (e.getCause() instanceof TimerException) {
+ TimerException te = (TimerException) e.getCause();
+ if (te.getCause() instanceof RuntimeException) {
+ RuntimeException re = (RuntimeException) te.getCause();
+ if (re.getMessage().equals("TEST SUCCESS")) {
+ testSuccess = true;
+ } else {
+ throw e;
+ }
+ } else {
+ throw e;
+ }
+ } else {
+ throw e;
+ }
+ }
+ Assert.assertTrue(testSuccess);
+ }
+
+ public static class TimerOperator extends AbstractStreamOperator<String> implements OneInputStreamOperator<String, String>, Triggerable {
+ private static final long serialVersionUID = 1L;
+
+ int numTimers = 0;
+ int numElements = 0;
+
+ private boolean first = true;
+
+ private Semaphore semaphore = new Semaphore(1);
+
+ public TimerOperator(ChainingStrategy chainingStrategy) {
+ setChainingStrategy(chainingStrategy);
+ }
+
+ @Override
+ public void processElement(StreamRecord<String> element) throws Exception {
+ if (!semaphore.tryAcquire()) {
+ Assert.fail("Concurrent invocation of operator functions.");
+ }
+
+ if (first) {
+ registerTimer(System.currentTimeMillis() + 100, this);
+ first = false;
+ }
+ numElements++;
+
+ semaphore.release();
+ }
+
+ @Override
+ public void trigger(long time) throws Exception {
+ if (!semaphore.tryAcquire()) {
+ Assert.fail("Concurrent invocation of operator functions.");
+ }
+
+ try {
+ numTimers++;
+ throwIfDone();
+ registerTimer(System.currentTimeMillis() + 1, this);
+ } finally {
+ semaphore.release();
+ }
+ }
+
+ private void throwIfDone() {
+ if (numTimers > 1000 && numElements > 10_000) {
+ throw new RuntimeException("TEST SUCCESS");
+ }
+ }
+
+ @Override
+ public void processWatermark(Watermark mark) throws Exception {
+ if (!semaphore.tryAcquire()) {
+ Assert.fail("Concurrent invocation of operator functions.");
+ }
+ semaphore.release();
+ }
+ }
+
+ public static class TwoInputTimerOperator extends AbstractStreamOperator<String> implements TwoInputStreamOperator<String, String, String>, Triggerable {
+ private static final long serialVersionUID = 1L;
+
+ int numTimers = 0;
+ int numElements = 0;
+
+ private boolean first = true;
+
+ private Semaphore semaphore = new Semaphore(1);
+
+ public TwoInputTimerOperator(ChainingStrategy chainingStrategy) {
+ setChainingStrategy(chainingStrategy);
+ }
+
+ @Override
+ public void processElement1(StreamRecord<String> element) throws Exception {
+ if (!semaphore.tryAcquire()) {
+ Assert.fail("Concurrent invocation of operator functions.");
+ }
+
+ if (first) {
+ registerTimer(System.currentTimeMillis() + 100, this);
+ first = false;
+ }
+ numElements++;
+
+ semaphore.release();
+ }
+
+ @Override
+ public void processElement2(StreamRecord<String> element) throws Exception {
+ if (!semaphore.tryAcquire()) {
+ Assert.fail("Concurrent invocation of operator functions.");
+ }
+
+ if (first) {
+ registerTimer(System.currentTimeMillis() + 100, this);
+ first = false;
+ }
+ numElements++;
+
+ semaphore.release();
+ }
+
+
+ @Override
+ public void trigger(long time) throws Exception {
+ if (!semaphore.tryAcquire()) {
+ Assert.fail("Concurrent invocation of operator functions.");
+ }
+
+ try {
+ numTimers++;
+ throwIfDone();
+ registerTimer(System.currentTimeMillis() + 1, this);
+ } finally {
+ semaphore.release();
+ }
+ }
+
+ private void throwIfDone() {
+ if (numTimers > 1000 && numElements > 10_000) {
+ throw new RuntimeException("TEST SUCCESS");
+ }
+ }
+
+ @Override
+ public void processWatermark1(Watermark mark) throws Exception {
+ //ignore
+ }
+
+ @Override
+ public void processWatermark2(Watermark mark) throws Exception {
+ //ignore
+ }
+ }
+
+
+ private static class InfiniteTestSource implements SourceFunction<String> {
+ private static final long serialVersionUID = 1L;
+ private volatile boolean running = true;
+
+ @Override
+ public void run(SourceContext<String> ctx) throws Exception {
+ while (running) {
+ ctx.collect("hello");
+ }
+ }
+
+ @Override
+ public void cancel() {
+ running = false;
+ }
+ }
+
+ // ------------------------------------------------------------------------
+ // parametrization
+ // ------------------------------------------------------------------------
+
+ @Parameterized.Parameters(name = "Time Characteristic = {0}")
+ public static Collection<Object[]> executionModes() {
+ return Arrays.asList(
+ new Object[] { TimeCharacteristic.ProcessingTime },
+ new Object[] { TimeCharacteristic.IngestionTime },
+ new Object[] { TimeCharacteristic.EventTime });
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/b9f42e91/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/TimestampITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/TimestampITCase.java b/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/TimestampITCase.java
new file mode 100644
index 0000000..d69c140
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/TimestampITCase.java
@@ -0,0 +1,875 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.test.streaming.runtime;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.api.common.functions.MapFunction;
+import org.apache.flink.api.common.functions.ReduceFunction;
+import org.apache.flink.api.common.functions.StoppableFunction;
+import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.configuration.ConfigConstants;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.core.testutils.MultiShotLatch;
+import org.apache.flink.streaming.api.TimeCharacteristic;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.functions.sink.DiscardingSink;
+import org.apache.flink.streaming.api.functions.timestamps.AscendingTimestampExtractor;
+import org.apache.flink.streaming.api.functions.AssignerWithPeriodicWatermarks;
+import org.apache.flink.streaming.api.functions.AssignerWithPunctuatedWatermarks;
+import org.apache.flink.streaming.api.functions.co.CoMapFunction;
+import org.apache.flink.streaming.api.functions.source.SourceFunction;
+import org.apache.flink.streaming.api.operators.AbstractStreamOperator;
+import org.apache.flink.streaming.api.operators.ChainingStrategy;
+import org.apache.flink.streaming.api.operators.OneInputStreamOperator;
+import org.apache.flink.streaming.api.watermark.Watermark;
+import org.apache.flink.streaming.api.windowing.assigners.TumblingEventTimeWindows;
+import org.apache.flink.streaming.api.windowing.time.Time;
+import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
+import org.apache.flink.test.util.ForkableFlinkMiniCluster;
+import org.apache.flink.util.TestLogger;
+
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/**
+ * Tests for timestamps, watermarks, and event-time sources.
+ */
+@SuppressWarnings("serial")
+public class TimestampITCase extends TestLogger {
+
+ private static final int NUM_TASK_MANAGERS = 2;
+ private static final int NUM_TASK_SLOTS = 3;
+ private static final int PARALLELISM = NUM_TASK_MANAGERS * NUM_TASK_SLOTS;
+
+ // this is used in some tests to synchronize
+ static MultiShotLatch latch;
+
+
+ private static ForkableFlinkMiniCluster cluster;
+
+ @Before
+ public void setupLatch() {
+ // ensure that we get a fresh latch for each test
+ latch = new MultiShotLatch();
+ }
+
+
+ @BeforeClass
+ public static void startCluster() {
+ try {
+ Configuration config = new Configuration();
+ config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, NUM_TASK_MANAGERS);
+ config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, NUM_TASK_SLOTS);
+ config.setInteger(ConfigConstants.TASK_MANAGER_MEMORY_SIZE_KEY, 12);
+
+ cluster = new ForkableFlinkMiniCluster(config, false);
+
+ cluster.start();
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail("Failed to start test cluster: " + e.getMessage());
+ }
+ }
+
+ @AfterClass
+ public static void shutdownCluster() {
+ try {
+ cluster.shutdown();
+ cluster = null;
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail("Failed to stop test cluster: " + e.getMessage());
+ }
+ }
+
+ /**
+ * These check whether custom timestamp emission works at sources and also whether timestamps
+ * arrive at operators throughout a topology.
+ *
+ * <p>
+ * This also checks whether watermarks keep propagating if a source closes early.
+ *
+ * <p>
+ * This only uses map to test the workings of watermarks in a complete, running topology. All
+ * tasks and stream operators have dedicated tests that test the watermark propagation
+ * behaviour.
+ */
+ @Test
+ public void testWatermarkPropagation() throws Exception {
+ final int NUM_WATERMARKS = 10;
+
+ long initialTime = 0L;
+
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment(
+ "localhost", cluster.getLeaderRPCPort());
+
+ env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
+ env.setParallelism(PARALLELISM);
+ env.getConfig().disableSysoutLogging();
+
+ DataStream<Integer> source1 = env.addSource(new MyTimestampSource(initialTime, NUM_WATERMARKS));
+ DataStream<Integer> source2 = env.addSource(new MyTimestampSource(initialTime, NUM_WATERMARKS / 2));
+
+ source1.union(source2)
+ .map(new IdentityMap())
+ .connect(source2).map(new IdentityCoMap())
+ .transform("Custom Operator", BasicTypeInfo.INT_TYPE_INFO, new CustomOperator(true))
+ .addSink(new DiscardingSink<Integer>());
+
+ env.execute();
+
+ // verify that all the watermarks arrived at the final custom operator
+ for (int i = 0; i < PARALLELISM; i++) {
+ // we are only guaranteed to see NUM_WATERMARKS / 2 watermarks because the
+ // other source stops emitting after that
+ for (int j = 0; j < NUM_WATERMARKS / 2; j++) {
+ if (!CustomOperator.finalWatermarks[i].get(j).equals(new Watermark(initialTime + j))) {
+ System.err.println("All Watermarks: ");
+ for (int k = 0; k <= NUM_WATERMARKS / 2; k++) {
+ System.err.println(CustomOperator.finalWatermarks[i].get(k));
+ }
+
+ fail("Wrong watermark.");
+ }
+ }
+
+ assertEquals(Watermark.MAX_WATERMARK,
+ CustomOperator.finalWatermarks[i].get(CustomOperator.finalWatermarks[i].size()-1));
+ }
+ }
+
+ @Test
+ public void testWatermarkPropagationNoFinalWatermarkOnStop() throws Exception {
+
+ // for this test to work, we need to be sure that no other jobs are being executed
+ while (!cluster.getCurrentlyRunningJobsJava().isEmpty()) {
+ Thread.sleep(100);
+ }
+
+ final int NUM_WATERMARKS = 10;
+
+ long initialTime = 0L;
+
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment(
+ "localhost", cluster.getLeaderRPCPort());
+
+ env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
+ env.setParallelism(PARALLELISM);
+ env.getConfig().disableSysoutLogging();
+
+ DataStream<Integer> source1 = env.addSource(new MyTimestampSourceInfinite(initialTime, NUM_WATERMARKS));
+ DataStream<Integer> source2 = env.addSource(new MyTimestampSourceInfinite(initialTime, NUM_WATERMARKS / 2));
+
+ source1.union(source2)
+ .map(new IdentityMap())
+ .connect(source2).map(new IdentityCoMap())
+ .transform("Custom Operator", BasicTypeInfo.INT_TYPE_INFO, new CustomOperator(true))
+ .addSink(new DiscardingSink<Integer>());
+
+ new Thread("stopper") {
+ @Override
+ public void run() {
+ try {
+ // try until we get the running jobs
+ List<JobID> running;
+ while ((running = cluster.getCurrentlyRunningJobsJava()).isEmpty()) {
+ Thread.sleep(50);
+ }
+
+ JobID id = running.get(0);
+
+ // send stop until the job is stopped
+ do {
+ cluster.stopJob(id);
+ Thread.sleep(50);
+ } while (!cluster.getCurrentlyRunningJobsJava().isEmpty());
+ }
+ catch (Throwable t) {
+ t.printStackTrace();
+ }
+ }
+ }.start();
+
+ env.execute();
+
+ // verify that all the watermarks arrived at the final custom operator
+ for (List<Watermark> subtaskWatermarks : CustomOperator.finalWatermarks) {
+
+ // we are only guaranteed to see NUM_WATERMARKS / 2 watermarks because the
+ // other source stops emitting after that
+ for (int j = 0; j < subtaskWatermarks.size(); j++) {
+ if (subtaskWatermarks.get(j).getTimestamp() != initialTime + j) {
+ System.err.println("All Watermarks: ");
+ for (int k = 0; k <= NUM_WATERMARKS / 2; k++) {
+ System.err.println(subtaskWatermarks.get(k));
+ }
+
+ fail("Wrong watermark.");
+ }
+ }
+
+ // if there are watermarks, the final one must not be the MAX watermark
+ if (subtaskWatermarks.size() > 0) {
+ assertNotEquals(Watermark.MAX_WATERMARK,
+ subtaskWatermarks.get(subtaskWatermarks.size()-1));
+ }
+ }
+ }
+
+ /**
+ * These check whether timestamps are properly assigned at the sources and handled in
+ * network transmission and between chained operators when timestamps are enabled.
+ */
+ @Test
+ public void testTimestampHandling() throws Exception {
+ final int NUM_ELEMENTS = 10;
+
+
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment(
+ "localhost", cluster.getLeaderRPCPort());
+
+ env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
+ env.setParallelism(PARALLELISM);
+ env.getConfig().disableSysoutLogging();
+
+ DataStream<Integer> source1 = env.addSource(new MyTimestampSource(0L, NUM_ELEMENTS));
+ DataStream<Integer> source2 = env.addSource(new MyTimestampSource(0L, NUM_ELEMENTS));
+
+ source1
+ .map(new IdentityMap())
+ .connect(source2).map(new IdentityCoMap())
+ .transform("Custom Operator", BasicTypeInfo.INT_TYPE_INFO, new TimestampCheckingOperator())
+ .addSink(new DiscardingSink<Integer>());
+
+
+ env.execute();
+ }
+
+ /**
+ * These check whether timestamps are properly ignored when they are disabled.
+ */
+ @Test
+ public void testDisabledTimestamps() throws Exception {
+ final int NUM_ELEMENTS = 10;
+
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment(
+ "localhost", cluster.getLeaderRPCPort());
+
+ env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime);
+ env.setParallelism(PARALLELISM);
+ env.getConfig().disableSysoutLogging();
+
+ DataStream<Integer> source1 = env.addSource(new MyNonWatermarkingSource(NUM_ELEMENTS));
+ DataStream<Integer> source2 = env.addSource(new MyNonWatermarkingSource(NUM_ELEMENTS));
+
+ source1
+ .map(new IdentityMap())
+ .connect(source2).map(new IdentityCoMap())
+ .transform("Custom Operator", BasicTypeInfo.INT_TYPE_INFO, new DisabledTimestampCheckingOperator())
+ .addSink(new DiscardingSink<Integer>());
+
+ env.execute();
+ }
+
+ /**
+ * This tests whether timestamps are properly extracted in the timestamp
+ * extractor and whether watermarks are also correctly forwared from this with the auto watermark
+ * interval.
+ */
+ @Test
+ public void testTimestampExtractorWithAutoInterval() throws Exception {
+ final int NUM_ELEMENTS = 10;
+
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", cluster.getLeaderRPCPort());
+
+ env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
+ env.getConfig().setAutoWatermarkInterval(10);
+ env.setParallelism(1);
+ env.getConfig().disableSysoutLogging();
+
+
+ DataStream<Integer> source1 = env.addSource(new SourceFunction<Integer>() {
+ @Override
+ public void run(SourceContext<Integer> ctx) throws Exception {
+ int index = 1;
+ while (index <= NUM_ELEMENTS) {
+ ctx.collect(index);
+ latch.await();
+ index++;
+ }
+ }
+
+ @Override
+ public void cancel() {}
+ });
+
+ DataStream<Integer> extractOp = source1.assignTimestampsAndWatermarks(
+ new AscendingTimestampExtractor<Integer>() {
+ @Override
+ public long extractAscendingTimestamp(Integer element) {
+ return element;
+ }
+ });
+
+ extractOp
+ .transform("Watermark Check", BasicTypeInfo.INT_TYPE_INFO, new CustomOperator(true))
+ .transform("Timestamp Check",
+ BasicTypeInfo.INT_TYPE_INFO,
+ new TimestampCheckingOperator());
+
+ // verify that extractor picks up source parallelism
+ Assert.assertEquals(extractOp.getTransformation().getParallelism(), source1.getTransformation().getParallelism());
+
+ env.execute();
+
+ // verify that we get NUM_ELEMENTS watermarks
+ for (int j = 0; j < NUM_ELEMENTS; j++) {
+ if (!CustomOperator.finalWatermarks[0].get(j).equals(new Watermark(j))) {
+ long wm = CustomOperator.finalWatermarks[0].get(j).getTimestamp();
+ Assert.fail("Wrong watermark. Expected: " + j + " Found: " + wm + " All: " + CustomOperator.finalWatermarks[0]);
+ }
+ }
+
+ // the input is finite, so it should have a MAX Watermark
+ assertEquals(Watermark.MAX_WATERMARK,
+ CustomOperator.finalWatermarks[0].get(CustomOperator.finalWatermarks[0].size() - 1));
+ }
+
+ /**
+ * This thests whether timestamps are properly extracted in the timestamp
+ * extractor and whether watermark are correctly forwarded from the custom watermark emit
+ * function.
+ */
+ @Test
+ public void testTimestampExtractorWithCustomWatermarkEmit() throws Exception {
+ final int NUM_ELEMENTS = 10;
+
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", cluster.getLeaderRPCPort());
+
+ env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
+ env.getConfig().setAutoWatermarkInterval(10);
+ env.setParallelism(1);
+ env.getConfig().disableSysoutLogging();
+
+
+ DataStream<Integer> source1 = env.addSource(new SourceFunction<Integer>() {
+ @Override
+ public void run(SourceContext<Integer> ctx) throws Exception {
+ int index = 1;
+ while (index <= NUM_ELEMENTS) {
+ ctx.collect(index);
+ latch.await();
+ index++;
+ }
+ }
+
+ @Override
+ public void cancel() {}
+ });
+
+ source1
+ .assignTimestampsAndWatermarks(new AssignerWithPunctuatedWatermarks<Integer>() {
+
+ @Override
+ public long extractTimestamp(Integer element, long currentTimestamp) {
+ return element;
+ }
+
+ @Override
+ public Watermark checkAndGetNextWatermark(Integer element, long extractedTimestamp) {
+ return new Watermark(extractedTimestamp - 1);
+ }
+ })
+ .transform("Watermark Check", BasicTypeInfo.INT_TYPE_INFO, new CustomOperator(true))
+ .transform("Timestamp Check", BasicTypeInfo.INT_TYPE_INFO, new TimestampCheckingOperator());
+
+
+ env.execute();
+
+ // verify that we get NUM_ELEMENTS watermarks
+ for (int j = 0; j < NUM_ELEMENTS; j++) {
+ if (!CustomOperator.finalWatermarks[0].get(j).equals(new Watermark(j))) {
+ Assert.fail("Wrong watermark.");
+ }
+ }
+
+ // the input is finite, so it should have a MAX Watermark
+ assertEquals(Watermark.MAX_WATERMARK,
+ CustomOperator.finalWatermarks[0].get(CustomOperator.finalWatermarks[0].size() - 1));
+ }
+
+ /**
+ * This test verifies that the timestamp extractor does not emit decreasing watermarks even
+ *
+ */
+ @Test
+ public void testTimestampExtractorWithDecreasingCustomWatermarkEmit() throws Exception {
+ final int NUM_ELEMENTS = 10;
+
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", cluster.getLeaderRPCPort());
+
+ env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
+ env.getConfig().setAutoWatermarkInterval(1);
+ env.setParallelism(1);
+ env.getConfig().disableSysoutLogging();
+
+
+ DataStream<Integer> source1 = env.addSource(new SourceFunction<Integer>() {
+ @Override
+ public void run(SourceContext<Integer> ctx) throws Exception {
+ int index = 1;
+ while (index <= NUM_ELEMENTS) {
+ ctx.collect(index);
+ Thread.sleep(100);
+ ctx.collect(index - 1);
+ latch.await();
+ index++;
+ }
+ }
+
+ @Override
+ public void cancel() {}
+ });
+
+ source1
+ .assignTimestampsAndWatermarks(new AssignerWithPunctuatedWatermarks<Integer>() {
+
+ @Override
+ public long extractTimestamp(Integer element, long previousTimestamp) {
+ return element;
+ }
+
+ @Override
+ public Watermark checkAndGetNextWatermark(Integer element, long extractedTimestamp) {
+ return new Watermark(extractedTimestamp - 1);
+ }
+ })
+ .transform("Watermark Check", BasicTypeInfo.INT_TYPE_INFO, new CustomOperator(true))
+ .transform("Timestamp Check", BasicTypeInfo.INT_TYPE_INFO, new TimestampCheckingOperator());
+
+
+ env.execute();
+
+ // verify that we get NUM_ELEMENTS watermarks
+ for (int j = 0; j < NUM_ELEMENTS; j++) {
+ if (!CustomOperator.finalWatermarks[0].get(j).equals(new Watermark(j))) {
+ Assert.fail("Wrong watermark.");
+ }
+ }
+ // the input is finite, so it should have a MAX Watermark
+ assertEquals(Watermark.MAX_WATERMARK,
+ CustomOperator.finalWatermarks[0].get(CustomOperator.finalWatermarks[0].size() - 1));
+ }
+
+ /**
+ * This test verifies that the timestamp extractor forwards Long.MAX_VALUE watermarks.
+ */
+ @Test
+ public void testTimestampExtractorWithLongMaxWatermarkFromSource() throws Exception {
+ final int NUM_ELEMENTS = 10;
+
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", cluster.getLeaderRPCPort());
+
+ env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
+ env.getConfig().setAutoWatermarkInterval(1);
+ env.setParallelism(2);
+ env.getConfig().disableSysoutLogging();
+
+
+ DataStream<Integer> source1 = env.addSource(new SourceFunction<Integer>() {
+ @Override
+ public void run(SourceContext<Integer> ctx) throws Exception {
+ int index = 1;
+ while (index <= NUM_ELEMENTS) {
+ ctx.collectWithTimestamp(index, index);
+ ctx.collectWithTimestamp(index - 1, index - 1);
+ index++;
+ ctx.emitWatermark(new Watermark(index-2));
+ }
+
+ // emit the final Long.MAX_VALUE watermark, do it twice and verify that
+ // we only see one in the result
+ ctx.emitWatermark(new Watermark(Long.MAX_VALUE));
+ ctx.emitWatermark(new Watermark(Long.MAX_VALUE));
+ }
+
+ @Override
+ public void cancel() {}
+ });
+
+ source1
+ .assignTimestampsAndWatermarks(new AssignerWithPunctuatedWatermarks<Integer>() {
+
+ @Override
+ public long extractTimestamp(Integer element, long currentTimestamp) {
+ return element;
+ }
+
+ @Override
+ public Watermark checkAndGetNextWatermark(Integer element, long extractedTimestamp) {
+ return null;
+ }
+ })
+ .transform("Watermark Check", BasicTypeInfo.INT_TYPE_INFO, new CustomOperator(true));
+
+
+ env.execute();
+
+ Assert.assertTrue(CustomOperator.finalWatermarks[0].size() == 1);
+ Assert.assertTrue(CustomOperator.finalWatermarks[0].get(0).getTimestamp() == Long.MAX_VALUE);
+ }
+
+ /**
+ * This test verifies that the timestamp extractor forwards Long.MAX_VALUE watermarks.
+ *
+ * Same test as before, but using a different timestamp extractor
+ */
+ @Test
+ public void testTimestampExtractorWithLongMaxWatermarkFromSource2() throws Exception {
+ final int NUM_ELEMENTS = 10;
+
+ StreamExecutionEnvironment env = StreamExecutionEnvironment
+ .createRemoteEnvironment("localhost", cluster.getLeaderRPCPort());
+
+ env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
+ env.getConfig().setAutoWatermarkInterval(10);
+ env.setParallelism(2);
+ env.getConfig().disableSysoutLogging();
+
+ DataStream<Integer> source1 = env.addSource(new SourceFunction<Integer>() {
+ @Override
+ public void run(SourceContext<Integer> ctx) throws Exception {
+ int index = 1;
+ while (index <= NUM_ELEMENTS) {
+ ctx.collectWithTimestamp(index, index);
+ ctx.collectWithTimestamp(index - 1, index - 1);
+ index++;
+ ctx.emitWatermark(new Watermark(index-2));
+ }
+
+ // emit the final Long.MAX_VALUE watermark, do it twice and verify that
+ // we only see one in the result
+ ctx.emitWatermark(new Watermark(Long.MAX_VALUE));
+ ctx.emitWatermark(new Watermark(Long.MAX_VALUE));
+ }
+
+ @Override
+ public void cancel() {}
+ });
+
+ source1
+ .assignTimestampsAndWatermarks(new AssignerWithPeriodicWatermarks<Integer>() {
+
+ @Override
+ public long extractTimestamp(Integer element, long currentTimestamp) {
+ return element;
+ }
+
+ @Override
+ public Watermark getCurrentWatermark() {
+ return null;
+ }
+ })
+ .transform("Watermark Check", BasicTypeInfo.INT_TYPE_INFO, new CustomOperator(true));
+
+ env.execute();
+
+ Assert.assertTrue(CustomOperator.finalWatermarks[0].size() == 1);
+ Assert.assertTrue(CustomOperator.finalWatermarks[0].get(0).getTimestamp() == Long.MAX_VALUE);
+ }
+
+ /**
+ * This verifies that an event time source works when setting stream time characteristic to
+ * processing time. In this case, the watermarks should just be swallowed.
+ */
+ @Test
+ public void testEventTimeSourceWithProcessingTime() throws Exception {
+ StreamExecutionEnvironment env =
+ StreamExecutionEnvironment.createRemoteEnvironment("localhost", cluster.getLeaderRPCPort());
+
+ env.setParallelism(2);
+ env.getConfig().disableSysoutLogging();
+ env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime);
+
+ DataStream<Integer> source1 = env.addSource(new MyTimestampSource(0, 10));
+
+ source1
+ .map(new IdentityMap())
+ .transform("Watermark Check", BasicTypeInfo.INT_TYPE_INFO, new CustomOperator(false));
+
+ env.execute();
+
+ // verify that we don't get any watermarks, the source is used as watermark source in
+ // other tests, so it normally emits watermarks
+ Assert.assertTrue(CustomOperator.finalWatermarks[0].size() == 0);
+ }
+
+ @Test
+ public void testErrorOnEventTimeOverProcessingTime() {
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment(
+ "localhost", cluster.getLeaderRPCPort());
+
+ env.setParallelism(2);
+ env.getConfig().disableSysoutLogging();
+ env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime);
+
+ DataStream<Tuple2<String, Integer>> source1 =
+ env.fromElements(new Tuple2<>("a", 1), new Tuple2<>("b", 2));
+
+ source1
+ .keyBy(0)
+ .window(TumblingEventTimeWindows.of(Time.seconds(5)))
+ .reduce(new ReduceFunction<Tuple2<String, Integer>>() {
+ @Override
+ public Tuple2<String, Integer> reduce(Tuple2<String, Integer> value1, Tuple2<String, Integer> value2) {
+ return value1;
+ }
+ })
+ .print();
+
+ try {
+ env.execute();
+ fail("this should fail with an exception");
+ } catch (Exception e) {
+ // expected
+ }
+ }
+
+ @Test
+ public void testErrorOnEventTimeWithoutTimestamps() {
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment(
+ "localhost", cluster.getLeaderRPCPort());
+
+ env.setParallelism(2);
+ env.getConfig().disableSysoutLogging();
+ env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
+
+ DataStream<Tuple2<String, Integer>> source1 =
+ env.fromElements(new Tuple2<>("a", 1), new Tuple2<>("b", 2));
+
+ source1
+ .keyBy(0)
+ .window(TumblingEventTimeWindows.of(Time.seconds(5)))
+ .reduce(new ReduceFunction<Tuple2<String, Integer>>() {
+ @Override
+ public Tuple2<String, Integer> reduce(Tuple2<String, Integer> value1, Tuple2<String, Integer> value2) {
+ return value1;
+ }
+ })
+ .print();
+
+ try {
+ env.execute();
+ fail("this should fail with an exception");
+ } catch (Exception e) {
+ // expected
+ }
+ }
+
+ // ------------------------------------------------------------------------
+ // Custom Operators and Functions
+ // ------------------------------------------------------------------------
+
+ @SuppressWarnings("unchecked")
+ public static class CustomOperator extends AbstractStreamOperator<Integer> implements OneInputStreamOperator<Integer, Integer> {
+
+ List<Watermark> watermarks;
+ public static List<Watermark>[] finalWatermarks = new List[PARALLELISM];
+ private final boolean timestampsEnabled;
+
+ public CustomOperator(boolean timestampsEnabled) {
+ setChainingStrategy(ChainingStrategy.ALWAYS);
+ this.timestampsEnabled = timestampsEnabled;
+ }
+
+ @Override
+ public void processElement(StreamRecord<Integer> element) throws Exception {
+ if (timestampsEnabled) {
+ if (element.getTimestamp() != element.getValue()) {
+ Assert.fail("Timestamps are not properly handled.");
+ }
+ }
+ output.collect(element);
+ }
+
+ @Override
+ public void processWatermark(Watermark mark) throws Exception {
+ for (Watermark previousMark: watermarks) {
+ assertTrue(previousMark.getTimestamp() < mark.getTimestamp());
+ }
+ watermarks.add(mark);
+ latch.trigger();
+ output.emitWatermark(mark);
+ }
+
+ @Override
+ public void open() throws Exception {
+ super.open();
+ watermarks = new ArrayList<>();
+ }
+
+ @Override
+ public void close() throws Exception {
+ super.close();
+ finalWatermarks[getRuntimeContext().getIndexOfThisSubtask()] = watermarks;
+ }
+ }
+
+ public static class TimestampCheckingOperator extends AbstractStreamOperator<Integer> implements OneInputStreamOperator<Integer, Integer> {
+
+ public TimestampCheckingOperator() {
+ setChainingStrategy(ChainingStrategy.ALWAYS);
+ }
+
+ @Override
+ public void processElement(StreamRecord<Integer> element) throws Exception {
+ if (element.getTimestamp() != element.getValue()) {
+ Assert.fail("Timestamps are not properly handled.");
+ }
+ output.collect(element);
+ }
+
+ @Override
+ public void processWatermark(Watermark mark) throws Exception {}
+ }
+
+ public static class DisabledTimestampCheckingOperator extends AbstractStreamOperator<Integer> implements OneInputStreamOperator<Integer, Integer> {
+
+ @Override
+ public void processElement(StreamRecord<Integer> element) throws Exception {
+ if (element.hasTimestamp()) {
+ Assert.fail("Timestamps are not properly handled.");
+ }
+ output.collect(element);
+ }
+
+ @Override
+ public void processWatermark(Watermark mark) throws Exception {}
+ }
+
+ public static class IdentityCoMap implements CoMapFunction<Integer, Integer, Integer> {
+ @Override
+ public Integer map1(Integer value) throws Exception {
+ return value;
+ }
+
+ @Override
+ public Integer map2(Integer value) throws Exception {
+ return value;
+ }
+ }
+
+ public static class IdentityMap implements MapFunction<Integer, Integer> {
+ @Override
+ public Integer map(Integer value) throws Exception {
+ return value;
+ }
+ }
+
+ public static class MyTimestampSource implements SourceFunction<Integer> {
+
+ private final long initialTime;
+ private final int numWatermarks;
+
+ public MyTimestampSource(long initialTime, int numWatermarks) {
+ this.initialTime = initialTime;
+ this.numWatermarks = numWatermarks;
+ }
+
+ @Override
+ public void run(SourceContext<Integer> ctx) throws Exception {
+ for (int i = 0; i < numWatermarks; i++) {
+ ctx.collectWithTimestamp(i, initialTime + i);
+ ctx.emitWatermark(new Watermark(initialTime + i));
+ }
+ }
+
+ @Override
+ public void cancel() {}
+ }
+
+ public static class MyTimestampSourceInfinite implements SourceFunction<Integer>, StoppableFunction {
+
+ private final long initialTime;
+ private final int numWatermarks;
+
+ private volatile boolean running = true;
+
+ public MyTimestampSourceInfinite(long initialTime, int numWatermarks) {
+ this.initialTime = initialTime;
+ this.numWatermarks = numWatermarks;
+ }
+
+ @Override
+ public void run(SourceContext<Integer> ctx) throws Exception {
+ for (int i = 0; i < numWatermarks; i++) {
+ ctx.collectWithTimestamp(i, initialTime + i);
+ ctx.emitWatermark(new Watermark(initialTime + i));
+ }
+
+ while (running) {
+ Thread.sleep(20);
+ }
+ }
+
+ @Override
+ public void cancel() {
+ running = false;
+ }
+
+ @Override
+ public void stop() {
+ running = false;
+ }
+ }
+
+ public static class MyNonWatermarkingSource implements SourceFunction<Integer> {
+
+ int numWatermarks;
+
+ public MyNonWatermarkingSource(int numWatermarks) {
+ this.numWatermarks = numWatermarks;
+ }
+
+ @Override
+ public void run(SourceContext<Integer> ctx) throws Exception {
+ for (int i = 0; i < numWatermarks; i++) {
+ ctx.collect(i);
+ }
+ }
+
+ @Override
+ public void cancel() {}
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/b9f42e91/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/WindowFoldITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/WindowFoldITCase.java b/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/WindowFoldITCase.java
new file mode 100644
index 0000000..1e3e3d5
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/WindowFoldITCase.java
@@ -0,0 +1,190 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.test.streaming.runtime;
+
+import org.apache.flink.api.common.functions.FoldFunction;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.streaming.api.TimeCharacteristic;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.functions.AssignerWithPunctuatedWatermarks;
+import org.apache.flink.streaming.api.functions.sink.SinkFunction;
+import org.apache.flink.streaming.api.functions.source.SourceFunction;
+import org.apache.flink.streaming.api.watermark.Watermark;
+import org.apache.flink.streaming.api.windowing.assigners.TumblingEventTimeWindows;
+import org.apache.flink.streaming.api.windowing.time.Time;
+import org.apache.flink.streaming.util.StreamingMultipleProgramsTestBase;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Tests for Folds over windows. These also test whether OutputTypeConfigurable functions
+ * work for windows, because FoldWindowFunction is OutputTypeConfigurable.
+ */
+@SuppressWarnings("serial")
+public class WindowFoldITCase extends StreamingMultipleProgramsTestBase {
+
+ private static List<String> testResults;
+
+ @Test
+ public void testFoldWindow() throws Exception {
+
+ testResults = new ArrayList<>();
+
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+ env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
+ env.setParallelism(1);
+
+ DataStream<Tuple2<String, Integer>> source1 = env.addSource(new SourceFunction<Tuple2<String, Integer>>() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public void run(SourceContext<Tuple2<String, Integer>> ctx) throws Exception {
+ ctx.collect(Tuple2.of("a", 0));
+ ctx.collect(Tuple2.of("a", 1));
+ ctx.collect(Tuple2.of("a", 2));
+
+ ctx.collect(Tuple2.of("b", 3));
+ ctx.collect(Tuple2.of("b", 4));
+ ctx.collect(Tuple2.of("b", 5));
+
+ ctx.collect(Tuple2.of("a", 6));
+ ctx.collect(Tuple2.of("a", 7));
+ ctx.collect(Tuple2.of("a", 8));
+
+ // source is finite, so it will have an implicit MAX watermark when it finishes
+ }
+
+ @Override
+ public void cancel() {}
+
+ }).assignTimestampsAndWatermarks(new Tuple2TimestampExtractor());
+
+ source1
+ .keyBy(0)
+ .window(TumblingEventTimeWindows.of(Time.of(3, TimeUnit.MILLISECONDS)))
+ .fold(Tuple2.of("R:", 0), new FoldFunction<Tuple2<String, Integer>, Tuple2<String, Integer>>() {
+ @Override
+ public Tuple2<String, Integer> fold(Tuple2<String, Integer> accumulator,
+ Tuple2<String, Integer> value) throws Exception {
+ accumulator.f0 += value.f0;
+ accumulator.f1 += value.f1;
+ return accumulator;
+ }
+ })
+ .addSink(new SinkFunction<Tuple2<String, Integer>>() {
+ @Override
+ public void invoke(Tuple2<String, Integer> value) throws Exception {
+ testResults.add(value.toString());
+ }
+ });
+
+ env.execute("Fold Window Test");
+
+ List<String> expectedResult = Arrays.asList(
+ "(R:aaa,3)",
+ "(R:aaa,21)",
+ "(R:bbb,12)");
+
+ Collections.sort(expectedResult);
+ Collections.sort(testResults);
+
+ Assert.assertEquals(expectedResult, testResults);
+ }
+
+ @Test
+ public void testFoldAllWindow() throws Exception {
+
+ testResults = new ArrayList<>();
+
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+ env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
+ env.setParallelism(1);
+
+ DataStream<Tuple2<String, Integer>> source1 = env.addSource(new SourceFunction<Tuple2<String, Integer>>() {
+
+ @Override
+ public void run(SourceContext<Tuple2<String, Integer>> ctx) throws Exception {
+ ctx.collect(Tuple2.of("a", 0));
+ ctx.collect(Tuple2.of("a", 1));
+ ctx.collect(Tuple2.of("a", 2));
+
+ ctx.collect(Tuple2.of("b", 3));
+ ctx.collect(Tuple2.of("a", 3));
+ ctx.collect(Tuple2.of("b", 4));
+ ctx.collect(Tuple2.of("a", 4));
+ ctx.collect(Tuple2.of("b", 5));
+ ctx.collect(Tuple2.of("a", 5));
+
+ // source is finite, so it will have an implicit MAX watermark when it finishes
+ }
+
+ @Override
+ public void cancel() {
+ }
+ }).assignTimestampsAndWatermarks(new Tuple2TimestampExtractor());
+
+ source1
+ .windowAll(TumblingEventTimeWindows.of(Time.of(3, TimeUnit.MILLISECONDS)))
+ .fold(Tuple2.of("R:", 0), new FoldFunction<Tuple2<String, Integer>, Tuple2<String, Integer>>() {
+ @Override
+ public Tuple2<String, Integer> fold(Tuple2<String, Integer> accumulator,
+ Tuple2<String, Integer> value) throws Exception {
+ accumulator.f0 += value.f0;
+ accumulator.f1 += value.f1;
+ return accumulator;
+ }
+ })
+ .addSink(new SinkFunction<Tuple2<String, Integer>>() {
+ @Override
+ public void invoke(Tuple2<String, Integer> value) throws Exception {
+ testResults.add(value.toString());
+ }
+ });
+
+ env.execute("Fold All-Window Test");
+
+ List<String> expectedResult = Arrays.asList(
+ "(R:aaa,3)",
+ "(R:bababa,24)");
+
+ Collections.sort(expectedResult);
+ Collections.sort(testResults);
+
+ Assert.assertEquals(expectedResult, testResults);
+ }
+
+ private static class Tuple2TimestampExtractor implements AssignerWithPunctuatedWatermarks<Tuple2<String, Integer>> {
+
+ @Override
+ public long extractTimestamp(Tuple2<String, Integer> element, long previousTimestamp) {
+ return element.f1;
+ }
+
+ @Override
+ public Watermark checkAndGetNextWatermark(Tuple2<String, Integer> lastElement, long extractedTimestamp) {
+ return new Watermark(lastElement.f1 - 1);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/b9f42e91/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/util/EvenOddOutputSelector.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/util/EvenOddOutputSelector.java b/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/util/EvenOddOutputSelector.java
new file mode 100644
index 0000000..8fc8372
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/util/EvenOddOutputSelector.java
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.test.streaming.runtime.util;
+
+import org.apache.flink.streaming.api.collector.selector.OutputSelector;
+
+import java.util.Collections;
+
+public class EvenOddOutputSelector implements OutputSelector<Integer> {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Iterable<String> select(Integer value) {
+ return value % 2 == 0 ? Collections.singleton("even") : Collections.singleton("odd");
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/b9f42e91/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/util/NoOpIntMap.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/util/NoOpIntMap.java b/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/util/NoOpIntMap.java
new file mode 100644
index 0000000..6667446
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/util/NoOpIntMap.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.test.streaming.runtime.util;
+
+import org.apache.flink.api.common.functions.MapFunction;
+
+public class NoOpIntMap implements MapFunction<Integer, Integer> {
+ private static final long serialVersionUID = 1L;
+
+ public Integer map(Integer value) throws Exception {
+ return value;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/b9f42e91/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/util/ReceiveCheckNoOpSink.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/util/ReceiveCheckNoOpSink.java b/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/util/ReceiveCheckNoOpSink.java
new file mode 100644
index 0000000..21d5294
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/util/ReceiveCheckNoOpSink.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.test.streaming.runtime.util;
+
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.junit.Assert.assertTrue;
+
+public final class ReceiveCheckNoOpSink<T> extends RichSinkFunction<T> {
+ private List<T> received;
+
+ public void invoke(T tuple) {
+ received.add(tuple);
+ }
+
+ public void open(Configuration conf) {
+ received = new ArrayList<T>();
+ }
+
+ public void close() {
+ assertTrue(received.size() > 0);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/b9f42e91/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/util/TestListResultSink.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/util/TestListResultSink.java b/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/util/TestListResultSink.java
new file mode 100644
index 0000000..321d4c5
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/util/TestListResultSink.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.test.streaming.runtime.util;
+
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.TreeSet;
+
+public class TestListResultSink<T> extends RichSinkFunction<T> {
+
+ private static final long serialVersionUID = 1L;
+ private int resultListId;
+
+ public TestListResultSink() {
+ this.resultListId = TestListWrapper.getInstance().createList();
+ }
+
+ @Override
+ public void open(Configuration parameters) throws Exception {
+ super.open(parameters);
+ }
+
+ @Override
+ public void invoke(T value) throws Exception {
+ synchronized (resultList()) {
+ resultList().add(value);
+ }
+ }
+
+ @Override
+ public void close() throws Exception {
+ super.close();
+ }
+
+ @SuppressWarnings("unchecked")
+ private List<T> resultList() {
+ synchronized (TestListWrapper.getInstance()) {
+ return (List<T>) TestListWrapper.getInstance().getList(resultListId);
+ }
+ }
+
+ public List<T> getResult() {
+ synchronized (resultList()) {
+ ArrayList<T> copiedList = new ArrayList<T>(resultList());
+ return copiedList;
+ }
+ }
+
+ public List<T> getSortedResult() {
+ synchronized (resultList()) {
+ TreeSet<T> treeSet = new TreeSet<T>(resultList());
+ ArrayList<T> sortedList = new ArrayList<T>(treeSet);
+ return sortedList;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/b9f42e91/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/util/TestListWrapper.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/util/TestListWrapper.java b/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/util/TestListWrapper.java
new file mode 100644
index 0000000..19ca8eb
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/util/TestListWrapper.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.test.streaming.runtime.util;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+public class TestListWrapper {
+
+ private static TestListWrapper instance;
+
+ @SuppressWarnings("rawtypes")
+ private List<List<? extends Comparable>> lists;
+
+ @SuppressWarnings("rawtypes")
+ private TestListWrapper() {
+ lists = Collections.synchronizedList(new ArrayList<List<? extends Comparable>>());
+ }
+
+ public static TestListWrapper getInstance() {
+ if (instance == null) {
+ instance = new TestListWrapper();
+ }
+ return instance;
+ }
+
+ /**
+ * Creates and stores a list, returns with the id.
+ *
+ * @return The ID of the list.
+ */
+ @SuppressWarnings("rawtypes")
+ public int createList() {
+ lists.add(new ArrayList<Comparable>());
+ return lists.size() - 1;
+ }
+
+ public List<?> getList(int listId) {
+ @SuppressWarnings("rawtypes")
+ List<? extends Comparable> list = lists.get(listId);
+ if (list == null) {
+ throw new RuntimeException("No such list.");
+ }
+
+ return list;
+ }
+
+}
\ No newline at end of file
[08/15] flink git commit: [FLINK-3995] [build] Properly structure
test scopes and dependencies
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-java/src/test/java/org/apache/flink/api/common/io/SequentialFormatTestBase.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/common/io/SequentialFormatTestBase.java b/flink-java/src/test/java/org/apache/flink/api/common/io/SequentialFormatTestBase.java
new file mode 100644
index 0000000..039f4de
--- /dev/null
+++ b/flink-java/src/test/java/org/apache/flink/api/common/io/SequentialFormatTestBase.java
@@ -0,0 +1,300 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.common.io;
+
+import org.apache.flink.api.common.io.statistics.BaseStatistics;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.core.fs.FileInputSplit;
+import org.apache.flink.core.fs.Path;
+import org.apache.flink.core.memory.DataOutputView;
+import org.apache.flink.core.memory.DataOutputViewStreamWrapper;
+import org.apache.flink.util.TestLogger;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runners.Parameterized.Parameters;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.List;
+
+/**
+ * Test base for {@link BinaryInputFormat} and {@link BinaryOutputFormat}.
+ */
+public abstract class SequentialFormatTestBase<T> extends TestLogger {
+
+ public class InputSplitSorter implements Comparator<FileInputSplit> {
+ @Override
+ public int compare(FileInputSplit o1, FileInputSplit o2) {
+ int pathOrder = o1.getPath().getName().compareTo(o2.getPath().getName());
+ return pathOrder == 0 ? Long.signum(o1.getStart() - o2.getStart()) : pathOrder;
+ }
+ }
+
+ private int numberOfTuples;
+
+ protected long blockSize;
+
+ private int parallelism;
+
+ private int[] rawDataSizes;
+
+ protected File tempFile;
+
+ /**
+ * Initializes SequentialFormatTest.
+ */
+ public SequentialFormatTestBase(int numberOfTuples, long blockSize, int parallelism) {
+ this.numberOfTuples = numberOfTuples;
+ this.blockSize = blockSize;
+ this.parallelism = parallelism;
+ this.rawDataSizes = new int[parallelism];
+ }
+
+ /**
+ * Count how many bytes would be written if all records were directly serialized
+ */
+ @Before
+ public void calcRawDataSize() throws IOException {
+ int recordIndex = 0;
+ for (int fileIndex = 0; fileIndex < this.parallelism; fileIndex++) {
+ ByteCounter byteCounter = new ByteCounter();
+
+ for (int fileCount = 0; fileCount < this.getNumberOfTuplesPerFile(fileIndex); fileCount++, recordIndex++) {
+ writeRecord(this.getRecord(recordIndex),
+ new DataOutputViewStreamWrapper(byteCounter));
+ }
+ this.rawDataSizes[fileIndex] = byteCounter.getLength();
+ }
+ }
+
+ /**
+ * Checks if the expected input splits were created
+ */
+ @Test
+ public void checkInputSplits() throws IOException {
+ FileInputSplit[] inputSplits = this.createInputFormat().createInputSplits(0);
+ Arrays.sort(inputSplits, new InputSplitSorter());
+
+ int splitIndex = 0;
+ for (int fileIndex = 0; fileIndex < this.parallelism; fileIndex++) {
+ List<FileInputSplit> sameFileSplits = new ArrayList<FileInputSplit>();
+ Path lastPath = inputSplits[splitIndex].getPath();
+ for (; splitIndex < inputSplits.length; splitIndex++) {
+ if (!inputSplits[splitIndex].getPath().equals(lastPath)) {
+ break;
+ }
+ sameFileSplits.add(inputSplits[splitIndex]);
+ }
+
+ Assert.assertEquals(this.getExpectedBlockCount(fileIndex), sameFileSplits.size());
+
+ long lastBlockLength =
+ this.rawDataSizes[fileIndex] % (this.blockSize - getInfoSize()) + getInfoSize();
+ for (int index = 0; index < sameFileSplits.size(); index++) {
+ Assert.assertEquals(this.blockSize * index, sameFileSplits.get(index).getStart());
+ if (index < sameFileSplits.size() - 1) {
+ Assert.assertEquals(this.blockSize, sameFileSplits.get(index).getLength());
+ }
+ }
+ Assert.assertEquals(lastBlockLength, sameFileSplits.get(sameFileSplits.size() - 1).getLength());
+ }
+ }
+
+ /**
+ * Tests if the expected sequence and amount of data can be read
+ */
+ @Test
+ public void checkRead() throws Exception {
+ BinaryInputFormat<T> input = this.createInputFormat();
+ FileInputSplit[] inputSplits = input.createInputSplits(0);
+ Arrays.sort(inputSplits, new InputSplitSorter());
+
+ int readCount = 0;
+
+ for (FileInputSplit inputSplit : inputSplits) {
+ input.open(inputSplit);
+ input.reopen(inputSplit, input.getCurrentState());
+
+ T record = createInstance();
+
+ while (!input.reachedEnd()) {
+ if (input.nextRecord(record) != null) {
+ this.checkEquals(this.getRecord(readCount), record);
+
+ if (!input.reachedEnd()) {
+ Tuple2<Long, Long> state = input.getCurrentState();
+
+ input = this.createInputFormat();
+ input.reopen(inputSplit, state);
+ }
+ readCount++;
+ }
+ }
+ }
+ Assert.assertEquals(this.numberOfTuples, readCount);
+ }
+
+ /**
+ * Tests the statistics of the given format.
+ */
+ @Test
+ public void checkStatistics() {
+ BinaryInputFormat<T> input = this.createInputFormat();
+ BaseStatistics statistics = input.getStatistics(null);
+ Assert.assertEquals(this.numberOfTuples, statistics.getNumberOfRecords());
+ }
+
+ @After
+ public void cleanup() {
+ this.deleteRecursively(this.tempFile);
+ }
+
+ private void deleteRecursively(File file) {
+ if (file.isDirectory()) {
+ for (File subFile : file.listFiles()) {
+ this.deleteRecursively(subFile);
+ }
+ } else {
+ file.delete();
+ }
+ }
+
+ /**
+ * Write out the tuples in a temporary file and return it.
+ */
+ @Before
+ public void writeTuples() throws IOException {
+ this.tempFile = File.createTempFile("BinaryInputFormat", null);
+ this.tempFile.deleteOnExit();
+ Configuration configuration = new Configuration();
+ configuration.setLong(BinaryOutputFormat.BLOCK_SIZE_PARAMETER_KEY, this.blockSize);
+ if (this.parallelism == 1) {
+ BinaryOutputFormat<T> output = createOutputFormat(this.tempFile.toURI().toString(),
+ configuration);
+ for (int index = 0; index < this.numberOfTuples; index++) {
+ output.writeRecord(this.getRecord(index));
+ }
+ output.close();
+ } else {
+ this.tempFile.delete();
+ this.tempFile.mkdir();
+ int recordIndex = 0;
+ for (int fileIndex = 0; fileIndex < this.parallelism; fileIndex++) {
+ BinaryOutputFormat<T> output = createOutputFormat(this.tempFile.toURI() + "/" +
+ (fileIndex+1), configuration);
+ for (int fileCount = 0; fileCount < this.getNumberOfTuplesPerFile(fileIndex); fileCount++, recordIndex++) {
+ output.writeRecord(this.getRecord(recordIndex));
+ }
+ output.close();
+ }
+ }
+ }
+
+ private int getNumberOfTuplesPerFile(int fileIndex) {
+ return this.numberOfTuples / this.parallelism;
+ }
+
+ /**
+ * Tests if the length of the file matches the expected value.
+ */
+ @Test
+ public void checkLength() {
+ File[] files = this.tempFile.isDirectory() ? this.tempFile.listFiles() : new File[] { this.tempFile };
+ Arrays.sort(files);
+ for (int fileIndex = 0; fileIndex < this.parallelism; fileIndex++) {
+ long lastBlockLength = this.rawDataSizes[fileIndex] % (this.blockSize - getInfoSize());
+ long expectedLength =
+ (this.getExpectedBlockCount(fileIndex) - 1) * this.blockSize + getInfoSize() +
+ lastBlockLength;
+ Assert.assertEquals(expectedLength, files[fileIndex].length());
+ }
+ }
+
+ abstract protected BinaryInputFormat<T> createInputFormat();
+
+ abstract protected BinaryOutputFormat<T> createOutputFormat(final String path, final
+ Configuration configuration)
+ throws IOException;
+
+ abstract protected int getInfoSize();
+
+ /**
+ * Returns the record to write at the given position
+ */
+ abstract protected T getRecord(int index);
+
+ abstract protected T createInstance();
+
+ abstract protected void writeRecord(T record, DataOutputView outputView) throws IOException;
+
+ /**
+ * Checks if both records are equal
+ */
+ abstract protected void checkEquals(T expected, T actual);
+
+ private int getExpectedBlockCount(int fileIndex) {
+ int expectedBlockCount =
+ (int) Math.ceil((double) this.rawDataSizes[fileIndex] / (this.blockSize -
+ getInfoSize()));
+ return expectedBlockCount;
+ }
+
+ @Parameters
+ public static List<Object[]> getParameters() {
+ ArrayList<Object[]> params = new ArrayList<Object[]>();
+ for (int parallelism = 1; parallelism <= 2; parallelism++) {
+ // numberOfTuples, blockSize, parallelism
+ params.add(new Object[] { 100, BinaryOutputFormat.NATIVE_BLOCK_SIZE, parallelism });
+ params.add(new Object[] { 100, 1000, parallelism });
+ params.add(new Object[] { 100, 1 << 20, parallelism });
+ params.add(new Object[] { 10000, 1000, parallelism });
+ params.add(new Object[] { 10000, 1 << 20, parallelism });
+ }
+ return params;
+ }
+
+ /**
+ * Counts the bytes that would be written.
+ *
+ */
+ private static final class ByteCounter extends OutputStream {
+ int length = 0;
+
+ /**
+ * Returns the length.
+ *
+ * @return the length
+ */
+ public int getLength() {
+ return this.length;
+ }
+
+ @Override
+ public void write(int b) throws IOException {
+ this.length++;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-java/src/test/java/org/apache/flink/api/common/io/SerializedFormatTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/common/io/SerializedFormatTest.java b/flink-java/src/test/java/org/apache/flink/api/common/io/SerializedFormatTest.java
new file mode 100644
index 0000000..db2ee8c
--- /dev/null
+++ b/flink-java/src/test/java/org/apache/flink/api/common/io/SerializedFormatTest.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.common.io;
+
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.core.fs.FileSystem;
+import org.apache.flink.core.fs.Path;
+import org.apache.flink.core.memory.DataOutputView;
+import org.apache.flink.types.IntValue;
+import org.apache.flink.types.Record;
+import org.apache.flink.types.StringValue;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.io.IOException;
+
+@RunWith(Parameterized.class)
+public class SerializedFormatTest extends SequentialFormatTestBase<Record> {
+
+ private BlockInfo info;
+
+ public SerializedFormatTest(int numberOfRecords, long blockSize, int parallelism){
+ super(numberOfRecords, blockSize, parallelism);
+ }
+
+ @Before
+ public void setup(){
+ info = createInputFormat().createBlockInfo();
+ }
+
+ @Override
+ protected BinaryInputFormat<Record> createInputFormat() {
+ Configuration configuration = new Configuration();
+ configuration.setLong(BinaryInputFormat.BLOCK_SIZE_PARAMETER_KEY, this.blockSize);
+
+ final SerializedInputFormat<Record> inputFormat = new SerializedInputFormat<Record>();
+ inputFormat.setFilePath(this.tempFile.toURI().toString());
+
+ inputFormat.configure(configuration);
+ return inputFormat;
+ }
+
+
+ @Override
+ protected BinaryOutputFormat<Record> createOutputFormat(String path, Configuration configuration)
+ throws IOException
+ {
+ final SerializedOutputFormat<Record> outputFormat = new SerializedOutputFormat<Record>();
+ outputFormat.setOutputFilePath(new Path(path));
+ outputFormat.setWriteMode(FileSystem.WriteMode.OVERWRITE);
+
+ configuration = configuration == null ? new Configuration() : configuration;
+ outputFormat.configure(configuration);
+ outputFormat.open(0, 1);
+ return outputFormat;
+ }
+
+ @Override
+ protected int getInfoSize() {
+ return info.getInfoSize();
+ }
+
+ @Override
+ protected Record getRecord(int index) {
+ return new Record(new IntValue(index), new StringValue(String.valueOf(index)));
+ }
+
+ @Override
+ protected Record createInstance() {
+ return new Record();
+ }
+
+ @Override
+ protected void writeRecord(Record record, DataOutputView outputView) throws IOException{
+ record.write(outputView);
+ }
+
+ @Override
+ protected void checkEquals(Record expected, Record actual) {
+ Assert.assertEquals(expected.getNumFields(), actual.getNumFields());
+ Assert.assertEquals(expected.getField(0, IntValue.class), actual.getField(0, IntValue.class));
+ Assert.assertEquals(expected.getField(1, StringValue.class), actual.getField(1, StringValue.class));
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-java8/pom.xml
----------------------------------------------------------------------
diff --git a/flink-java8/pom.xml b/flink-java8/pom.xml
index 0635d98..00ace94 100644
--- a/flink-java8/pom.xml
+++ b/flink-java8/pom.xml
@@ -40,36 +40,34 @@ under the License.
</properties>
<dependencies>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-core</artifactId>
- <version>${project.version}</version>
- </dependency>
+
+ <!-- core dependencies -->
<dependency>
<groupId>org.apache.flink</groupId>
- <artifactId>flink-test-utils_2.10</artifactId>
+ <artifactId>flink-java</artifactId>
<version>${project.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
- <artifactId>flink-java</artifactId>
+ <artifactId>flink-streaming-java_2.10</artifactId>
<version>${project.version}</version>
</dependency>
-
+
<dependency>
<groupId>org.apache.flink</groupId>
- <artifactId>flink-streaming-java_2.10</artifactId>
+ <artifactId>flink-examples-batch_2.10</artifactId>
<version>${project.version}</version>
</dependency>
+ <!-- test dependencies -->
+
<dependency>
<groupId>org.apache.flink</groupId>
- <artifactId>flink-examples-batch_2.10</artifactId>
+ <artifactId>flink-test-utils_2.10</artifactId>
<version>${project.version}</version>
+ <scope>test</scope>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-libraries/flink-cep-scala/pom.xml
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-cep-scala/pom.xml b/flink-libraries/flink-cep-scala/pom.xml
index ea1d0e1..878a187 100644
--- a/flink-libraries/flink-cep-scala/pom.xml
+++ b/flink-libraries/flink-cep-scala/pom.xml
@@ -17,50 +17,64 @@ KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
<modelVersion>4.0.0</modelVersion>
+
<parent>
<groupId>org.apache.flink</groupId>
<artifactId>flink-libraries</artifactId>
<version>1.1-SNAPSHOT</version>
<relativePath>..</relativePath>
</parent>
+
<artifactId>flink-cep-scala_2.10</artifactId>
<name>flink-cep-scala</name>
<packaging>jar</packaging>
<dependencies>
+
+ <!-- core dependencies -->
+
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-cep_2.10</artifactId>
<version>${project.version}</version>
</dependency>
+
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-streaming-scala_2.10</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
- <!-- We need to add this explicitly because through shading the dependency on asm seems
- to go away. TODO -->
+
+ <!-- We need to add this explicitly due to shading -->
<dependency>
<groupId>org.ow2.asm</groupId>
<artifactId>asm</artifactId>
<version>${asm.version}</version>
</dependency>
+
+ <!-- test dependencies -->
+
<dependency>
<groupId>org.apache.flink</groupId>
- <artifactId>flink-tests_2.10</artifactId>
+ <artifactId>flink-test-utils_2.10</artifactId>
<version>${project.version}</version>
<scope>test</scope>
- <type>test-jar</type>
</dependency>
+
<dependency>
<groupId>org.apache.flink</groupId>
- <artifactId>flink-test-utils_2.10</artifactId>
+ <artifactId>flink-tests_2.10</artifactId>
<version>${project.version}</version>
<scope>test</scope>
+ <type>test-jar</type>
</dependency>
+
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-streaming-scala_2.10</artifactId>
@@ -68,6 +82,7 @@ under the License.
<scope>test</scope>
<type>test-jar</type>
</dependency>
+
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-cep_2.10</artifactId>
@@ -93,6 +108,7 @@ under the License.
</execution>
</executions>
</plugin>
+
<!-- Scala Code Style, most of the configuration done via plugin management -->
<plugin>
<groupId>org.scalastyle</groupId>
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-libraries/flink-cep/pom.xml
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-cep/pom.xml b/flink-libraries/flink-cep/pom.xml
index 84ed8ac..6b6d431 100644
--- a/flink-libraries/flink-cep/pom.xml
+++ b/flink-libraries/flink-cep/pom.xml
@@ -35,32 +35,32 @@ under the License.
<packaging>jar</packaging>
<dependencies>
+
+ <!-- core dependencies -->
+
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-core</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
+
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-streaming-java_2.10</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- <version>${guava.version}</version>
- </dependency>
+
+ <!-- test dependencies -->
<dependency>
<groupId>org.apache.flink</groupId>
- <artifactId>flink-core</artifactId>
+ <artifactId>flink-test-utils_2.10</artifactId>
<version>${project.version}</version>
- <type>test-jar</type>
<scope>test</scope>
</dependency>
-
+
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-streaming-java_2.10</artifactId>
@@ -68,14 +68,6 @@ under the License.
<type>test-jar</type>
<scope>test</scope>
</dependency>
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-test-utils_2.10</artifactId>
- <version>${project.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-libraries/flink-gelly-examples/pom.xml
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-gelly-examples/pom.xml b/flink-libraries/flink-gelly-examples/pom.xml
index 2cef359..0565e5b 100644
--- a/flink-libraries/flink-gelly-examples/pom.xml
+++ b/flink-libraries/flink-gelly-examples/pom.xml
@@ -32,6 +32,9 @@
<packaging>jar</packaging>
<dependencies>
+
+ <!-- core dependencies -->
+
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-java</artifactId>
@@ -57,11 +60,13 @@
<artifactId>flink-gelly-scala_2.10</artifactId>
<version>${project.version}</version>
</dependency>
+
+ <!-- test dependencies -->
+
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-test-utils_2.10</artifactId>
<version>${project.version}</version>
- <type>test-jar</type>
<scope>test</scope>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-libraries/flink-gelly-scala/pom.xml
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-gelly-scala/pom.xml b/flink-libraries/flink-gelly-scala/pom.xml
index 97d81f6..23b4037 100644
--- a/flink-libraries/flink-gelly-scala/pom.xml
+++ b/flink-libraries/flink-gelly-scala/pom.xml
@@ -58,18 +58,21 @@ under the License.
<artifactId>flink-gelly_2.10</artifactId>
<version>${project.version}</version>
</dependency>
+
+ <!-- test dependencies -->
+
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-tests_2.10</artifactId>
<version>${project.version}</version>
<scope>test</scope>
<type>test-jar</type>
+
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-test-utils_2.10</artifactId>
<version>${project.version}</version>
- <type>test-jar</type>
<scope>test</scope>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-libraries/flink-gelly/pom.xml
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-gelly/pom.xml b/flink-libraries/flink-gelly/pom.xml
index 7e6f712..14e63d2 100644
--- a/flink-libraries/flink-gelly/pom.xml
+++ b/flink-libraries/flink-gelly/pom.xml
@@ -35,25 +35,32 @@ under the License.
<packaging>jar</packaging>
<dependencies>
+
+ <!-- core dependencies -->
+
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-java</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
+
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-clients_2.10</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
+
+ <!-- test dependencies -->
+
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-test-utils_2.10</artifactId>
<version>${project.version}</version>
- <type>test-jar</type>
<scope>test</scope>
</dependency>
+
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-optimizer_2.10</artifactId>
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-libraries/flink-ml/pom.xml
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-ml/pom.xml b/flink-libraries/flink-ml/pom.xml
index 80ac88e..ca286e8 100644
--- a/flink-libraries/flink-ml/pom.xml
+++ b/flink-libraries/flink-ml/pom.xml
@@ -34,6 +34,9 @@
<packaging>jar</packaging>
<dependencies>
+
+ <!-- core dependencies -->
+
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-scala_2.10</artifactId>
@@ -47,26 +50,18 @@
<version>0.12</version>
</dependency>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-clients_2.10</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
+ <!-- test dependencies -->
<dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-clients_2.10</artifactId>
- <version>${project.version}</version>
- <type>test-jar</type>
+ <groupId>org.scalatest</groupId>
+ <artifactId>scalatest_${scala.binary.version}</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
- <artifactId>flink-core</artifactId>
+ <artifactId>flink-clients_2.10</artifactId>
<version>${project.version}</version>
- <type>test-jar</type>
<scope>test</scope>
</dependency>
@@ -74,7 +69,6 @@
<groupId>org.apache.flink</groupId>
<artifactId>flink-test-utils_2.10</artifactId>
<version>${project.version}</version>
- <type>test-jar</type>
<scope>test</scope>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/MLUtilsSuite.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/MLUtilsSuite.scala b/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/MLUtilsSuite.scala
index f02f5ff..cd1a340 100644
--- a/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/MLUtilsSuite.scala
+++ b/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/MLUtilsSuite.scala
@@ -20,16 +20,15 @@ package org.apache.flink.ml
import java.io.File
-import scala.io.Source
-
-import org.scalatest.{FlatSpec, Matchers}
-
import org.apache.flink.api.scala._
-import org.apache.flink.api.scala.ExecutionEnvironment
+import org.apache.flink.core.testutils.CommonTestUtils
import org.apache.flink.ml.common.LabeledVector
import org.apache.flink.ml.math.SparseVector
-import org.apache.flink.test.util.FlinkTestBase
-import org.apache.flink.testutils.TestFileUtils
+import org.apache.flink.ml.util.FlinkTestBase
+
+import org.scalatest.{FlatSpec, Matchers}
+
+import scala.io.Source
class MLUtilsSuite extends FlatSpec with Matchers with FlinkTestBase {
@@ -53,7 +52,7 @@ class MLUtilsSuite extends FlatSpec with Matchers with FlinkTestBase {
LabeledVector(-42.1, SparseVector.fromCOO(10, (1, 2), (3, -6.1), (2, 5.1)))
)
- val inputFilePath = TestFileUtils.createTempFile(content)
+ val inputFilePath = CommonTestUtils.createTempFile(content)
val svmInput = env.readLibSVM(inputFilePath)
@@ -86,10 +85,7 @@ class MLUtilsSuite extends FlatSpec with Matchers with FlinkTestBase {
val labeledVectorsDS = env.fromCollection(labeledVectors)
- val tempDir = new File(System.getProperty("java.io.tmpdir"))
-
- val tempFile = new File(tempDir, TestFileUtils.randomFileName())
-
+ val tempFile = File.createTempFile("flink_test_", ".tmp")
val outputFilePath = tempFile.getAbsolutePath
labeledVectorsDS.writeAsLibSVM(outputFilePath)
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/classification/SVMITSuite.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/classification/SVMITSuite.scala b/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/classification/SVMITSuite.scala
index e6eb873..af17451 100644
--- a/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/classification/SVMITSuite.scala
+++ b/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/classification/SVMITSuite.scala
@@ -18,11 +18,11 @@
package org.apache.flink.ml.classification
+import org.apache.flink.ml.util.FlinkTestBase
import org.scalatest.{FlatSpec, Matchers}
import org.apache.flink.ml.math.DenseVector
import org.apache.flink.api.scala._
-import org.apache.flink.test.util.FlinkTestBase
class SVMITSuite extends FlatSpec with Matchers with FlinkTestBase {
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/common/FlinkMLToolsSuite.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/common/FlinkMLToolsSuite.scala b/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/common/FlinkMLToolsSuite.scala
index 525ba4d..14bb22e 100644
--- a/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/common/FlinkMLToolsSuite.scala
+++ b/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/common/FlinkMLToolsSuite.scala
@@ -20,7 +20,7 @@ package org.apache.flink.ml.common
import org.apache.flink.api.java.typeutils.runtime.kryo.KryoSerializer
import org.apache.flink.api.scala.ExecutionEnvironment
-import org.apache.flink.test.util.FlinkTestBase
+import org.apache.flink.ml.util.FlinkTestBase
import org.scalatest.{FlatSpec, Matchers}
class FlinkMLToolsSuite extends FlatSpec with Matchers with FlinkTestBase {
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/nn/KNNITSuite.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/nn/KNNITSuite.scala b/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/nn/KNNITSuite.scala
index 389d69a..871c1c4 100644
--- a/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/nn/KNNITSuite.scala
+++ b/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/nn/KNNITSuite.scala
@@ -24,7 +24,7 @@ import org.apache.flink.ml.classification.Classification
import org.apache.flink.ml.math.DenseVector
import org.apache.flink.ml.metrics.distances.{ManhattanDistanceMetric,
SquaredEuclideanDistanceMetric}
-import org.apache.flink.test.util.FlinkTestBase
+import org.apache.flink.ml.util.FlinkTestBase
import org.scalatest.{FlatSpec, Matchers}
class KNNITSuite extends FlatSpec with Matchers with FlinkTestBase {
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/nn/QuadTreeSuite.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/nn/QuadTreeSuite.scala b/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/nn/QuadTreeSuite.scala
index a3a415d..086324f 100644
--- a/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/nn/QuadTreeSuite.scala
+++ b/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/nn/QuadTreeSuite.scala
@@ -19,8 +19,8 @@
package org.apache.flink.ml.nn
import org.apache.flink.ml.metrics.distances.EuclideanDistanceMetric
-import org.apache.flink.test.util.FlinkTestBase
import org.apache.flink.ml.math.{Vector, DenseVector}
+import org.apache.flink.ml.util.FlinkTestBase
import org.scalatest.{Matchers, FlatSpec}
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/optimization/GradientDescentITSuite.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/optimization/GradientDescentITSuite.scala b/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/optimization/GradientDescentITSuite.scala
index aed3bfd..728ef57 100644
--- a/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/optimization/GradientDescentITSuite.scala
+++ b/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/optimization/GradientDescentITSuite.scala
@@ -21,10 +21,10 @@ package org.apache.flink.ml.optimization
import org.apache.flink.ml.common.{LabeledVector, WeightVector}
import org.apache.flink.ml.math.DenseVector
import org.apache.flink.ml.regression.RegressionData._
+import org.apache.flink.ml.util.FlinkTestBase
import org.scalatest.{Matchers, FlatSpec}
import org.apache.flink.api.scala._
-import org.apache.flink.test.util.FlinkTestBase
class GradientDescentITSuite extends FlatSpec with Matchers with FlinkTestBase {
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/optimization/LossFunctionITSuite.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/optimization/LossFunctionITSuite.scala b/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/optimization/LossFunctionITSuite.scala
index 4152188..3d538cf 100644
--- a/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/optimization/LossFunctionITSuite.scala
+++ b/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/optimization/LossFunctionITSuite.scala
@@ -20,10 +20,10 @@ package org.apache.flink.ml.optimization
import org.apache.flink.ml.common.{LabeledVector, WeightVector}
import org.apache.flink.ml.math.DenseVector
+import org.apache.flink.ml.util.FlinkTestBase
import org.scalatest.{Matchers, FlatSpec}
import org.apache.flink.api.scala._
-import org.apache.flink.test.util.FlinkTestBase
class LossFunctionITSuite extends FlatSpec with Matchers with FlinkTestBase {
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/optimization/PredictionFunctionITSuite.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/optimization/PredictionFunctionITSuite.scala b/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/optimization/PredictionFunctionITSuite.scala
index 6d2a239..9d4732b 100644
--- a/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/optimization/PredictionFunctionITSuite.scala
+++ b/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/optimization/PredictionFunctionITSuite.scala
@@ -21,7 +21,7 @@ package org.apache.flink.ml.optimization
import org.apache.flink.ml.common.WeightVector
import org.apache.flink.ml.math.DenseVector
import org.apache.flink.api.scala._
-import org.apache.flink.test.util.FlinkTestBase
+import org.apache.flink.ml.util.FlinkTestBase
import org.scalatest.{Matchers, FlatSpec}
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/pipeline/PipelineITSuite.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/pipeline/PipelineITSuite.scala b/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/pipeline/PipelineITSuite.scala
index a3ea086..d8bd51a 100644
--- a/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/pipeline/PipelineITSuite.scala
+++ b/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/pipeline/PipelineITSuite.scala
@@ -24,7 +24,7 @@ import org.apache.flink.ml.common.{ParameterMap, LabeledVector}
import org.apache.flink.ml.math._
import org.apache.flink.ml.preprocessing.{PolynomialFeatures, StandardScaler}
import org.apache.flink.ml.regression.MultipleLinearRegression
-import org.apache.flink.test.util.FlinkTestBase
+import org.apache.flink.ml.util.FlinkTestBase
import org.scalatest.{Matchers, FlatSpec}
class PipelineITSuite extends FlatSpec with Matchers with FlinkTestBase {
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/preprocessing/MinMaxScalerITSuite.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/preprocessing/MinMaxScalerITSuite.scala b/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/preprocessing/MinMaxScalerITSuite.scala
index 75ac442..abfa851 100644
--- a/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/preprocessing/MinMaxScalerITSuite.scala
+++ b/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/preprocessing/MinMaxScalerITSuite.scala
@@ -22,7 +22,7 @@ import org.apache.flink.api.scala._
import org.apache.flink.ml.common.LabeledVector
import org.apache.flink.ml.math.Breeze._
import org.apache.flink.ml.math.{DenseVector, Vector}
-import org.apache.flink.test.util.FlinkTestBase
+import org.apache.flink.ml.util.FlinkTestBase
import org.scalatest.{FlatSpec, Matchers}
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/preprocessing/PolynomialFeaturesITSuite.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/preprocessing/PolynomialFeaturesITSuite.scala b/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/preprocessing/PolynomialFeaturesITSuite.scala
index 006db5f..4dc7cf3 100644
--- a/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/preprocessing/PolynomialFeaturesITSuite.scala
+++ b/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/preprocessing/PolynomialFeaturesITSuite.scala
@@ -21,7 +21,7 @@ package org.apache.flink.ml.preprocessing
import org.apache.flink.api.scala.{ExecutionEnvironment, _}
import org.apache.flink.ml.common.LabeledVector
import org.apache.flink.ml.math.DenseVector
-import org.apache.flink.test.util.FlinkTestBase
+import org.apache.flink.ml.util.FlinkTestBase
import org.scalatest.{FlatSpec, Matchers}
class PolynomialFeaturesITSuite
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/preprocessing/StandardScalerITSuite.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/preprocessing/StandardScalerITSuite.scala b/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/preprocessing/StandardScalerITSuite.scala
index 5cd253d..767de1d 100644
--- a/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/preprocessing/StandardScalerITSuite.scala
+++ b/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/preprocessing/StandardScalerITSuite.scala
@@ -23,8 +23,8 @@ import breeze.numerics.sqrt._
import org.apache.flink.api.scala._
import org.apache.flink.ml.common.LabeledVector
import org.apache.flink.ml.math.{Vector => FlinkVector, DenseVector}
-import org.apache.flink.test.util.FlinkTestBase
import org.apache.flink.ml.math.Breeze._
+import org.apache.flink.ml.util.FlinkTestBase
import org.scalatest._
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/recommendation/ALSITSuite.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/recommendation/ALSITSuite.scala b/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/recommendation/ALSITSuite.scala
index 9c241fd..8dca0c3 100644
--- a/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/recommendation/ALSITSuite.scala
+++ b/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/recommendation/ALSITSuite.scala
@@ -18,12 +18,12 @@
package org.apache.flink.ml.recommendation
+import org.apache.flink.ml.util.FlinkTestBase
import org.scalatest._
import scala.language.postfixOps
import org.apache.flink.api.scala._
-import org.apache.flink.test.util.FlinkTestBase
class ALSITSuite
extends FlatSpec
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/regression/MultipleLinearRegressionITSuite.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/regression/MultipleLinearRegressionITSuite.scala b/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/regression/MultipleLinearRegressionITSuite.scala
index 51214e6..fc3df3a 100644
--- a/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/regression/MultipleLinearRegressionITSuite.scala
+++ b/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/regression/MultipleLinearRegressionITSuite.scala
@@ -21,7 +21,7 @@ package org.apache.flink.ml.regression
import org.apache.flink.api.scala._
import org.apache.flink.ml.common.{ParameterMap, WeightVector}
import org.apache.flink.ml.preprocessing.PolynomialFeatures
-import org.apache.flink.test.util.FlinkTestBase
+import org.apache.flink.ml.util.FlinkTestBase
import org.scalatest.{FlatSpec, Matchers}
class MultipleLinearRegressionITSuite
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/util/FlinkTestBase.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/util/FlinkTestBase.scala b/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/util/FlinkTestBase.scala
new file mode 100644
index 0000000..fb98f24
--- /dev/null
+++ b/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/util/FlinkTestBase.scala
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.ml.util
+
+import org.apache.flink.test.util.{ForkableFlinkMiniCluster, TestBaseUtils, TestEnvironment}
+import org.scalatest.{BeforeAndAfter, Suite}
+
+/** Mixin to start and stop a ForkableFlinkMiniCluster automatically for Scala based tests.
+ * Additionally a TestEnvironment with the started cluster is created and set as the default
+ * [[org.apache.flink.api.java.ExecutionEnvironment]].
+ *
+ * This mixin starts a ForkableFlinkMiniCluster with one TaskManager and a number of slots given
+ * by parallelism. This value can be overridden in a sub class in order to start the cluster
+ * with a different number of slots.
+ *
+ * The cluster is started once before starting the tests and is re-used for the individual tests.
+ * After all tests have been executed, the cluster is shutdown.
+ *
+ * The cluster is used by obtaining the default [[org.apache.flink.api.java.ExecutionEnvironment]].
+ *
+ * @example
+ * {{{
+ * def testSomething: Unit = {
+ * // Obtain TestEnvironment with started ForkableFlinkMiniCluster
+ * val env = ExecutionEnvironment.getExecutionEnvironment
+ *
+ * env.fromCollection(...)
+ *
+ * env.execute
+ * }
+ * }}}
+ *
+ */
+trait FlinkTestBase extends BeforeAndAfter {
+ that: Suite =>
+
+ var cluster: Option[ForkableFlinkMiniCluster] = None
+ val parallelism = 4
+
+ before {
+ val cl = TestBaseUtils.startCluster(
+ 1,
+ parallelism,
+ false,
+ false,
+ true)
+
+ val clusterEnvironment = new TestEnvironment(cl, parallelism)
+ clusterEnvironment.setAsContext()
+
+ cluster = Some(cl)
+ }
+
+ after {
+ cluster.foreach(c => TestBaseUtils.stopCluster(c, TestBaseUtils.DEFAULT_TIMEOUT))
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-libraries/flink-python/pom.xml
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-python/pom.xml b/flink-libraries/flink-python/pom.xml
index 9f7445d..79ec043 100644
--- a/flink-libraries/flink-python/pom.xml
+++ b/flink-libraries/flink-python/pom.xml
@@ -52,6 +52,9 @@ under the License.
</build>
<dependencies>
+
+ <!-- core dependencies -->
+
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-core</artifactId>
@@ -77,11 +80,13 @@ under the License.
<artifactId>flink-clients_2.10</artifactId>
<version>${project.version}</version>
</dependency>
+
+ <!-- test dependencies -->
+
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-test-utils_2.10</artifactId>
<version>${project.version}</version>
- <type>test-jar</type>
<scope>test</scope>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-libraries/flink-python/src/test/java/org/apache/flink/python/api/PythonPlanBinderTest.java
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-python/src/test/java/org/apache/flink/python/api/PythonPlanBinderTest.java b/flink-libraries/flink-python/src/test/java/org/apache/flink/python/api/PythonPlanBinderTest.java
index c53d408..d144298 100644
--- a/flink-libraries/flink-python/src/test/java/org/apache/flink/python/api/PythonPlanBinderTest.java
+++ b/flink-libraries/flink-python/src/test/java/org/apache/flink/python/api/PythonPlanBinderTest.java
@@ -12,17 +12,20 @@
*/
package org.apache.flink.python.api;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
import org.apache.flink.core.fs.FileStatus;
import org.apache.flink.core.fs.FileSystem;
import org.apache.flink.core.fs.Path;
+import org.apache.flink.test.util.JavaProgramTestBase;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
import static org.apache.flink.python.api.PythonPlanBinder.ARGUMENT_PYTHON_2;
import static org.apache.flink.python.api.PythonPlanBinder.ARGUMENT_PYTHON_3;
-import org.apache.flink.test.util.JavaProgramTestBase;
public class PythonPlanBinderTest extends JavaProgramTestBase {
+
@Override
protected boolean skipCollectionExecution() {
return true;
@@ -35,7 +38,7 @@ public class PythonPlanBinderTest extends JavaProgramTestBase {
}
private static List<String> findTestFiles() throws Exception {
- List<String> files = new ArrayList();
+ List<String> files = new ArrayList<>();
FileSystem fs = FileSystem.getLocalFileSystem();
FileStatus[] status = fs.listStatus(
new Path(fs.getWorkingDirectory().toString()
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-libraries/flink-table/pom.xml
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/pom.xml b/flink-libraries/flink-table/pom.xml
index 33e2e57..46be2a1 100644
--- a/flink-libraries/flink-table/pom.xml
+++ b/flink-libraries/flink-table/pom.xml
@@ -34,11 +34,7 @@ under the License.
<dependencies>
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- <version>${guava.version}</version>
- </dependency>
+ <!-- core dependencies -->
<dependency>
<groupId>org.apache.flink</groupId>
@@ -53,20 +49,6 @@ under the License.
</dependency>
<dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-streaming-java_2.10</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-examples-batch_2.10</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-reflect</artifactId>
</dependency>
@@ -82,28 +64,12 @@ under the License.
</dependency>
<dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-test-utils_2.10</artifactId>
- <version>${project.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
-
- <dependency>
<groupId>org.codehaus.janino</groupId>
<artifactId>janino</artifactId>
<version>2.7.5</version>
</dependency>
<dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-tests_2.10</artifactId>
- <version>${project.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
-
- <dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</dependency>
@@ -115,6 +81,29 @@ under the License.
</dependency>
<dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>${guava.version}</version>
+ </dependency>
+
+ <!-- test dependencies -->
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-test-utils_2.10</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-tests_2.10</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-core</artifactId>
<version>${project.version}</version>
@@ -122,6 +111,13 @@ under the License.
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-streaming-java_2.10</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/stream/TableSinkITCase.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/stream/TableSinkITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/stream/TableSinkITCase.scala
index 160d88a..9dfee11 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/stream/TableSinkITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/stream/TableSinkITCase.scala
@@ -20,18 +20,15 @@ package org.apache.flink.api.scala.stream
import java.io.File
-import org.apache.flink.api.common.functions.MapFunction
-import org.apache.flink.api.common.typeinfo.TypeInformation
-import org.apache.flink.api.scala.stream.utils.StreamTestData
import org.apache.flink.api.scala._
+import org.apache.flink.api.scala.stream.utils.StreamTestData
import org.apache.flink.api.scala.table._
-import org.apache.flink.api.table.sinks.{CsvTableSink, StreamTableSink, TableSink}
-import org.apache.flink.api.table.typeutils.RowTypeInfo
-import org.apache.flink.api.table.{Row, TableEnvironment}
-import org.apache.flink.streaming.api.datastream.DataStream
+import org.apache.flink.api.table.TableEnvironment
+import org.apache.flink.api.table.sinks.CsvTableSink
import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment
import org.apache.flink.streaming.util.StreamingMultipleProgramsTestBase
import org.apache.flink.test.util.TestBaseUtils
+
import org.junit.Test
class TableSinkITCase extends StreamingMultipleProgramsTestBase {
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-optimizer/pom.xml
----------------------------------------------------------------------
diff --git a/flink-optimizer/pom.xml b/flink-optimizer/pom.xml
index 2c5a7a7..493475a 100644
--- a/flink-optimizer/pom.xml
+++ b/flink-optimizer/pom.xml
@@ -35,18 +35,13 @@ under the License.
<packaging>jar</packaging>
<dependencies>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-core</artifactId>
- <version>${project.version}</version>
- </dependency>
+
+ <!-- core dependencies -->
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-core</artifactId>
<version>${project.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
</dependency>
<dependency>
@@ -54,7 +49,7 @@ under the License.
<artifactId>flink-runtime_2.10</artifactId>
<version>${project.version}</version>
</dependency>
-
+
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-java</artifactId>
@@ -66,6 +61,15 @@ under the License.
<artifactId>guava</artifactId>
<version>${guava.version}</version>
</dependency>
+
+ <!-- test dependencies -->
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-test-utils-junit</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<!-- Because flink-tests needs the CompilerTestBsae -->
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-runtime-web/pom.xml
----------------------------------------------------------------------
diff --git a/flink-runtime-web/pom.xml b/flink-runtime-web/pom.xml
index e2637e4..8fd0e2f 100644
--- a/flink-runtime-web/pom.xml
+++ b/flink-runtime-web/pom.xml
@@ -34,32 +34,6 @@ under the License.
<packaging>jar</packaging>
- <build>
- <resources>
- <resource>
- <!-- Only include the web folder from the web-dashboard directory -->
- <directory>web-dashboard</directory>
- <includes>
- <include>web/**</include>
- </includes>
- </resource>
- </resources>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <inherited>true</inherited>
- <executions>
- <execution>
- <goals>
- <goal>test-jar</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-
<dependencies>
<!-- ===================================================
@@ -94,51 +68,13 @@ under the License.
</dependency>
<!-- ===================================================
- Dependencies for Actor Calls
- =================================================== -->
-
- <dependency>
- <groupId>org.scala-lang</groupId>
- <artifactId>scala-library</artifactId>
- </dependency>
-
- <dependency>
- <groupId>com.typesafe.akka</groupId>
- <artifactId>akka-actor_${scala.binary.version}</artifactId>
- </dependency>
-
- <dependency>
- <groupId>com.typesafe.akka</groupId>
- <artifactId>akka-remote_${scala.binary.version}</artifactId>
- </dependency>
-
- <!-- ===================================================
- Utilities
- =================================================== -->
-
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- <version>${guava.version}</version>
- </dependency>
-
- <!-- ===================================================
Testing
=================================================== -->
<dependency>
<groupId>org.apache.flink</groupId>
- <artifactId>flink-core</artifactId>
+ <artifactId>flink-test-utils-junit</artifactId>
<version>${project.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-runtime_2.10</artifactId>
- <version>${project.version}</version>
- <type>test-jar</type>
<scope>test</scope>
</dependency>
@@ -154,6 +90,41 @@ under the License.
<artifactId>akka-testkit_${scala.binary.version}</artifactId>
<scope>test</scope>
</dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-runtime_2.10</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
</dependencies>
+ <build>
+ <resources>
+ <resource>
+ <!-- Only include the web folder from the web-dashboard directory -->
+ <directory>web-dashboard</directory>
+ <includes>
+ <include>web/**</include>
+ </includes>
+ </resource>
+ </resources>
+
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
</project>
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/StackTraceSampleCoordinatorTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/StackTraceSampleCoordinatorTest.java b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/StackTraceSampleCoordinatorTest.java
index 29345a6..1791465 100644
--- a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/StackTraceSampleCoordinatorTest.java
+++ b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/StackTraceSampleCoordinatorTest.java
@@ -45,7 +45,6 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.eq;
import static org.mockito.Mockito.mock;
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-runtime/pom.xml
----------------------------------------------------------------------
diff --git a/flink-runtime/pom.xml b/flink-runtime/pom.xml
index bb3ba3b..a96a388 100644
--- a/flink-runtime/pom.xml
+++ b/flink-runtime/pom.xml
@@ -35,18 +35,13 @@ under the License.
<packaging>jar</packaging>
<dependencies>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-core</artifactId>
- <version>${project.version}</version>
- </dependency>
+
+ <!-- core dependencies -->
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-core</artifactId>
<version>${project.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
</dependency>
<dependency>
@@ -112,21 +107,11 @@ under the License.
</dependency>
<dependency>
- <groupId>com.typesafe.akka</groupId>
- <artifactId>akka-testkit_${scala.binary.version}</artifactId>
- </dependency>
-
- <dependency>
<groupId>org.clapper</groupId>
<artifactId>grizzled-slf4j_${scala.binary.version}</artifactId>
</dependency>
<dependency>
- <groupId>org.scalatest</groupId>
- <artifactId>scalatest_${scala.binary.version}</artifactId>
- </dependency>
-
- <dependency>
<groupId>com.github.scopt</groupId>
<artifactId>scopt_${scala.binary.version}</artifactId>
<exclusions>
@@ -176,13 +161,34 @@ under the License.
<artifactId>flink-shaded-curator-recipes</artifactId>
<version>${project.version}</version>
</dependency>
-
+
+ <!-- core dependencies -->
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-test-utils-junit</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-test</artifactId>
<version>${curator.version}</version>
<scope>test</scope>
</dependency>
+
+ <dependency>
+ <groupId>org.scalatest</groupId>
+ <artifactId>scalatest_${scala.binary.version}</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>com.typesafe.akka</groupId>
+ <artifactId>akka-testkit_${scala.binary.version}</artifactId>
+ </dependency>
+
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-runtime/src/main/java/org/apache/flink/runtime/testutils/TestingResourceManager.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/testutils/TestingResourceManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/testutils/TestingResourceManager.java
new file mode 100644
index 0000000..2422925
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/testutils/TestingResourceManager.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.testutils;
+
+import akka.actor.ActorRef;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.runtime.clusterframework.ApplicationStatus;
+import org.apache.flink.runtime.clusterframework.messages.RegisterResourceManagerSuccessful;
+import org.apache.flink.runtime.clusterframework.standalone.StandaloneResourceManager;
+import org.apache.flink.runtime.clusterframework.types.ResourceID;
+import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
+import org.apache.flink.runtime.messages.Messages;
+import org.apache.flink.runtime.testingUtils.TestingMessages;
+
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Set;
+
+
+/**
+ * A testing resource manager which may alter the default standalone resource master's behavior.
+ */
+public class TestingResourceManager extends StandaloneResourceManager {
+
+ /** Set of Actors which want to be informed of a connection to the job manager */
+ private Set<ActorRef> waitForResourceManagerConnected = new HashSet<>();
+
+ /** Set of Actors which want to be informed of a shutdown */
+ private Set<ActorRef> waitForShutdown = new HashSet<>();
+
+ /** Flag to signal a connection to the JobManager */
+ private boolean isConnected = false;
+
+ public TestingResourceManager(Configuration flinkConfig, LeaderRetrievalService leaderRetriever) {
+ super(flinkConfig, leaderRetriever);
+ }
+
+ /**
+ * Overwrite messages here if desired
+ */
+ @Override
+ protected void handleMessage(Object message) {
+
+ if (message instanceof GetRegisteredResources) {
+ sender().tell(new GetRegisteredResourcesReply(getRegisteredTaskManagers()), self());
+ } else if (message instanceof FailResource) {
+ ResourceID resourceID = ((FailResource) message).resourceID;
+ notifyWorkerFailed(resourceID, "Failed for test case.");
+
+ } else if (message instanceof NotifyWhenResourceManagerConnected) {
+ if (isConnected) {
+ sender().tell(
+ Messages.getAcknowledge(),
+ self());
+ } else {
+ waitForResourceManagerConnected.add(sender());
+ }
+ } else if (message instanceof RegisterResourceManagerSuccessful) {
+ super.handleMessage(message);
+
+ isConnected = true;
+
+ for (ActorRef ref : waitForResourceManagerConnected) {
+ ref.tell(
+ Messages.getAcknowledge(),
+ self());
+ }
+ waitForResourceManagerConnected.clear();
+
+ } else if (message instanceof TestingMessages.NotifyOfComponentShutdown$) {
+ waitForShutdown.add(sender());
+ } else if (message instanceof TestingMessages.Alive$) {
+ sender().tell(Messages.getAcknowledge(), self());
+ } else {
+ super.handleMessage(message);
+ }
+ }
+
+ /**
+ * Testing messages
+ */
+ public static class GetRegisteredResources {}
+
+ public static class GetRegisteredResourcesReply {
+
+ public Collection<ResourceID> resources;
+
+ public GetRegisteredResourcesReply(Collection<ResourceID> resources) {
+ this.resources = resources;
+ }
+
+ }
+
+ /**
+ * Fails all resources that the resource manager has registered
+ */
+ public static class FailResource {
+
+ public ResourceID resourceID;
+
+ public FailResource(ResourceID resourceID) {
+ this.resourceID = resourceID;
+ }
+ }
+
+ /**
+ * The sender of this message will be informed of a connection to the Job Manager
+ */
+ public static class NotifyWhenResourceManagerConnected {}
+
+ /**
+ * Inform registered listeners about a shutdown of the application.
+ */
+ @Override
+ protected void shutdownApplication(ApplicationStatus finalStatus, String optionalDiagnostics) {
+ for (ActorRef listener : waitForShutdown) {
+ listener.tell(new TestingMessages.ComponentShutdown(self()), self());
+ }
+ waitForShutdown.clear();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-runtime/src/main/scala/org/apache/flink/runtime/testingUtils/TestingJobManager.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/testingUtils/TestingJobManager.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/testingUtils/TestingJobManager.scala
new file mode 100644
index 0000000..04689c6
--- /dev/null
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/testingUtils/TestingJobManager.scala
@@ -0,0 +1,71 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.flink.runtime.testingUtils
+
+import akka.actor.ActorRef
+
+import org.apache.flink.configuration.Configuration
+import org.apache.flink.metrics.MetricRegistry
+import org.apache.flink.runtime.checkpoint.{SavepointStore, CheckpointRecoveryFactory}
+import org.apache.flink.runtime.execution.librarycache.BlobLibraryCacheManager
+import org.apache.flink.runtime.executiongraph.restart.RestartStrategyFactory
+import org.apache.flink.runtime.instance.InstanceManager
+import org.apache.flink.runtime.jobmanager.scheduler.Scheduler
+import org.apache.flink.runtime.jobmanager.{JobManager, SubmittedJobGraphStore}
+import org.apache.flink.runtime.leaderelection.LeaderElectionService
+
+import scala.concurrent.duration._
+import scala.language.postfixOps
+
+import java.util.concurrent.ExecutorService
+
+/** JobManager implementation extended by testing messages
+ *
+ */
+class TestingJobManager(
+ flinkConfiguration: Configuration,
+ executorService: ExecutorService,
+ instanceManager: InstanceManager,
+ scheduler: Scheduler,
+ libraryCacheManager: BlobLibraryCacheManager,
+ archive: ActorRef,
+ restartStrategyFactory: RestartStrategyFactory,
+ timeout: FiniteDuration,
+ leaderElectionService: LeaderElectionService,
+ submittedJobGraphs : SubmittedJobGraphStore,
+ checkpointRecoveryFactory : CheckpointRecoveryFactory,
+ savepointStore : SavepointStore,
+ jobRecoveryTimeout : FiniteDuration,
+ metricRegistry : Option[MetricRegistry])
+ extends JobManager(
+ flinkConfiguration,
+ executorService,
+ instanceManager,
+ scheduler,
+ libraryCacheManager,
+ archive,
+ restartStrategyFactory,
+ timeout,
+ leaderElectionService,
+ submittedJobGraphs,
+ checkpointRecoveryFactory,
+ savepointStore,
+ jobRecoveryTimeout,
+ metricRegistry)
+ with TestingJobManagerLike {}
http://git-wip-us.apache.org/repos/asf/flink/blob/4b71e0e7/flink-runtime/src/main/scala/org/apache/flink/runtime/testingUtils/TestingJobManagerLike.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/testingUtils/TestingJobManagerLike.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/testingUtils/TestingJobManagerLike.scala
new file mode 100644
index 0000000..5ccebe2
--- /dev/null
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/testingUtils/TestingJobManagerLike.scala
@@ -0,0 +1,405 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.testingUtils
+
+import akka.actor.{ActorRef, Cancellable, Terminated}
+import akka.pattern.{ask, pipe}
+import org.apache.flink.api.common.JobID
+import org.apache.flink.runtime.FlinkActor
+import org.apache.flink.runtime.clusterframework.messages.RegisterResourceSuccessful
+import org.apache.flink.runtime.execution.ExecutionState
+import org.apache.flink.runtime.jobgraph.JobStatus
+import org.apache.flink.runtime.jobmanager.JobManager
+import org.apache.flink.runtime.messages.ExecutionGraphMessages.JobStatusChanged
+import org.apache.flink.runtime.messages.JobManagerMessages.GrantLeadership
+import org.apache.flink.runtime.messages.Messages.{Acknowledge, Disconnect}
+import org.apache.flink.runtime.messages.RegistrationMessages.RegisterTaskManager
+import org.apache.flink.runtime.messages.TaskManagerMessages.Heartbeat
+import org.apache.flink.runtime.testingUtils.TestingJobManagerMessages._
+import org.apache.flink.runtime.testingUtils.TestingMessages._
+import org.apache.flink.runtime.testingUtils.TestingTaskManagerMessages.AccumulatorsChanged
+
+import scala.collection.mutable
+import scala.concurrent.Future
+import scala.concurrent.duration._
+import scala.language.postfixOps
+
+/** This mixin can be used to decorate a JobManager with messages for testing purpose. */
+trait TestingJobManagerLike extends FlinkActor {
+ that: JobManager =>
+
+ import context._
+
+ import scala.collection.JavaConverters._
+
+ val waitForAllVerticesToBeRunning = scala.collection.mutable.HashMap[JobID, Set[ActorRef]]()
+ val waitForTaskManagerToBeTerminated = scala.collection.mutable.HashMap[String, Set[ActorRef]]()
+
+ val waitForAllVerticesToBeRunningOrFinished =
+ scala.collection.mutable.HashMap[JobID, Set[ActorRef]]()
+
+ var periodicCheck: Option[Cancellable] = None
+
+ val waitForJobStatus = scala.collection.mutable.HashMap[JobID,
+ collection.mutable.HashMap[JobStatus, Set[ActorRef]]]()
+
+ val waitForAccumulatorUpdate = scala.collection.mutable.HashMap[JobID, (Boolean, Set[ActorRef])]()
+
+ val waitForLeader = scala.collection.mutable.HashSet[ActorRef]()
+
+ val waitForNumRegisteredTaskManagers = mutable.PriorityQueue.newBuilder(
+ new Ordering[(Int, ActorRef)] {
+ override def compare(x: (Int, ActorRef), y: (Int, ActorRef)): Int = y._1 - x._1
+ })
+
+ val waitForShutdown = scala.collection.mutable.HashSet[ActorRef]()
+
+ var disconnectDisabled = false
+
+ var postStopEnabled = true
+
+ abstract override def postStop(): Unit = {
+ if (postStopEnabled) {
+ super.postStop()
+ } else {
+ // only stop leader election service to revoke the leadership of this JM so that a new JM
+ // can be elected leader
+ leaderElectionService.stop()
+ }
+ }
+
+ abstract override def handleMessage: Receive = {
+ handleTestingMessage orElse super.handleMessage
+ }
+
+ def handleTestingMessage: Receive = {
+ case Alive => sender() ! Acknowledge
+
+ case RequestExecutionGraph(jobID) =>
+ currentJobs.get(jobID) match {
+ case Some((executionGraph, jobInfo)) => sender() ! decorateMessage(
+ ExecutionGraphFound(
+ jobID,
+ executionGraph)
+ )
+
+ case None => archive.tell(decorateMessage(RequestExecutionGraph(jobID)), sender())
+ }
+
+ case WaitForAllVerticesToBeRunning(jobID) =>
+ if(checkIfAllVerticesRunning(jobID)){
+ sender() ! decorateMessage(AllVerticesRunning(jobID))
+ }else{
+ val waiting = waitForAllVerticesToBeRunning.getOrElse(jobID, Set[ActorRef]())
+ waitForAllVerticesToBeRunning += jobID -> (waiting + sender())
+
+ if(periodicCheck.isEmpty){
+ periodicCheck =
+ Some(
+ context.system.scheduler.schedule(
+ 0 seconds,
+ 200 millis,
+ self,
+ decorateMessage(NotifyListeners)
+ )
+ )
+ }
+ }
+ case WaitForAllVerticesToBeRunningOrFinished(jobID) =>
+ if(checkIfAllVerticesRunningOrFinished(jobID)){
+ sender() ! decorateMessage(AllVerticesRunning(jobID))
+ }else{
+ val waiting = waitForAllVerticesToBeRunningOrFinished.getOrElse(jobID, Set[ActorRef]())
+ waitForAllVerticesToBeRunningOrFinished += jobID -> (waiting + sender())
+
+ if(periodicCheck.isEmpty){
+ periodicCheck =
+ Some(
+ context.system.scheduler.schedule(
+ 0 seconds,
+ 200 millis,
+ self,
+ decorateMessage(NotifyListeners)
+ )
+ )
+ }
+ }
+
+ case NotifyListeners =>
+ for(jobID <- currentJobs.keySet){
+ notifyListeners(jobID)
+ }
+
+ if(waitForAllVerticesToBeRunning.isEmpty && waitForAllVerticesToBeRunningOrFinished.isEmpty) {
+ periodicCheck foreach { _.cancel() }
+ periodicCheck = None
+ }
+
+
+ case NotifyWhenJobRemoved(jobID) =>
+ val gateways = instanceManager.getAllRegisteredInstances.asScala.map(_.getActorGateway)
+
+ val responses = gateways.map{
+ gateway => gateway.ask(NotifyWhenJobRemoved(jobID), timeout).mapTo[Boolean]
+ }
+
+ val jobRemovedOnJobManager = (self ? CheckIfJobRemoved(jobID))(timeout).mapTo[Boolean]
+
+ val allFutures = responses ++ Seq(jobRemovedOnJobManager)
+
+ import context.dispatcher
+ Future.fold(allFutures)(true)(_ & _) map(decorateMessage(_)) pipeTo sender()
+
+ case CheckIfJobRemoved(jobID) =>
+ if(currentJobs.contains(jobID)) {
+ context.system.scheduler.scheduleOnce(
+ 200 milliseconds,
+ self,
+ decorateMessage(CheckIfJobRemoved(jobID))
+ )(context.dispatcher, sender())
+ } else {
+ sender() ! decorateMessage(true)
+ }
+
+ case NotifyWhenTaskManagerTerminated(taskManager) =>
+ val waiting = waitForTaskManagerToBeTerminated.getOrElse(taskManager.path.name, Set())
+ waitForTaskManagerToBeTerminated += taskManager.path.name -> (waiting + sender)
+
+ case msg@Terminated(taskManager) =>
+ super.handleMessage(msg)
+
+ waitForTaskManagerToBeTerminated.remove(taskManager.path.name) foreach {
+ _ foreach {
+ listener =>
+ listener ! decorateMessage(TaskManagerTerminated(taskManager))
+ }
+ }
+
+ // see shutdown method for reply
+ case NotifyOfComponentShutdown =>
+ waitForShutdown += sender()
+
+ case NotifyWhenAccumulatorChange(jobID) =>
+
+ val (updated, registered) = waitForAccumulatorUpdate.
+ getOrElse(jobID, (false, Set[ActorRef]()))
+ waitForAccumulatorUpdate += jobID -> (updated, registered + sender)
+ sender ! true
+
+ /**
+ * Notification from the task manager that changed accumulator are transferred on next
+ * Hearbeat. We need to keep this state to notify the listeners on next Heartbeat report.
+ */
+ case AccumulatorsChanged(jobID: JobID) =>
+ waitForAccumulatorUpdate.get(jobID) match {
+ case Some((updated, registered)) =>
+ waitForAccumulatorUpdate.put(jobID, (true, registered))
+ case None =>
+ }
+
+ /**
+ * Disabled async processing of accumulator values and send accumulators to the listeners if
+ * we previously received an [[AccumulatorsChanged]] message.
+ */
+ case msg : Heartbeat =>
+ super.handleMessage(msg)
+
+ waitForAccumulatorUpdate foreach {
+ case (jobID, (updated, actors)) if updated =>
+ currentJobs.get(jobID) match {
+ case Some((graph, jobInfo)) =>
+ val flinkAccumulators = graph.getFlinkAccumulators
+ val userAccumulators = graph.aggregateUserAccumulators
+ actors foreach {
+ actor => actor ! UpdatedAccumulators(jobID, flinkAccumulators, userAccumulators)
+ }
+ case None =>
+ }
+ waitForAccumulatorUpdate.put(jobID, (false, actors))
+ case _ =>
+ }
+
+ case RequestWorkingTaskManager(jobID) =>
+ currentJobs.get(jobID) match {
+ case Some((eg, _)) =>
+ if(eg.getAllExecutionVertices.asScala.isEmpty){
+ sender ! decorateMessage(WorkingTaskManager(None))
+ } else {
+ val resource = eg.getAllExecutionVertices.asScala.head.getCurrentAssignedResource
+
+ if(resource == null){
+ sender ! decorateMessage(WorkingTaskManager(None))
+ } else {
+ sender ! decorateMessage(
+ WorkingTaskManager(
+ Some(resource.getInstance().getActorGateway)
+ )
+ )
+ }
+ }
+ case None => sender ! decorateMessage(WorkingTaskManager(None))
+ }
+
+ case NotifyWhenJobStatus(jobID, state) =>
+ val jobStatusListener = waitForJobStatus.getOrElseUpdate(jobID,
+ scala.collection.mutable.HashMap[JobStatus, Set[ActorRef]]())
+
+ val listener = jobStatusListener.getOrElse(state, Set[ActorRef]())
+
+ jobStatusListener += state -> (listener + sender)
+
+ case msg@JobStatusChanged(jobID, newJobStatus, _, _) =>
+ super.handleMessage(msg)
+
+ val cleanup = waitForJobStatus.get(jobID) match {
+ case Some(stateListener) =>
+ stateListener.remove(newJobStatus) match {
+ case Some(listeners) =>
+ listeners foreach {
+ _ ! decorateMessage(JobStatusIs(jobID, newJobStatus))
+ }
+ case _ =>
+ }
+ stateListener.isEmpty
+
+ case _ => false
+ }
+
+ if (cleanup) {
+ waitForJobStatus.remove(jobID)
+ }
+
+ case DisableDisconnect =>
+ disconnectDisabled = true
+
+ case DisablePostStop =>
+ postStopEnabled = false
+
+ case RequestSavepoint(savepointPath) =>
+ try {
+ val savepoint = savepointStore.getState(savepointPath)
+ sender ! ResponseSavepoint(savepoint)
+ }
+ catch {
+ case e: Exception =>
+ sender ! ResponseSavepoint(null)
+ }
+
+ case msg: Disconnect =>
+ if (!disconnectDisabled) {
+ super.handleMessage(msg)
+
+ val taskManager = sender()
+
+ waitForTaskManagerToBeTerminated.remove(taskManager.path.name) foreach {
+ _ foreach {
+ listener =>
+ listener ! decorateMessage(TaskManagerTerminated(taskManager))
+ }
+ }
+ }
+
+ case NotifyWhenLeader =>
+ if (leaderElectionService.hasLeadership) {
+ sender() ! true
+ } else {
+ waitForLeader += sender()
+ }
+
+ case msg: GrantLeadership =>
+ super.handleMessage(msg)
+
+ waitForLeader.foreach(_ ! true)
+
+ waitForLeader.clear()
+
+ case NotifyWhenAtLeastNumTaskManagerAreRegistered(numRegisteredTaskManager) =>
+ if (that.instanceManager.getNumberOfRegisteredTaskManagers >= numRegisteredTaskManager) {
+ // there are already at least numRegisteredTaskManager registered --> send Acknowledge
+ sender() ! Acknowledge
+ } else {
+ // wait until we see at least numRegisteredTaskManager being registered at the JobManager
+ waitForNumRegisteredTaskManagers += ((numRegisteredTaskManager, sender()))
+ }
+
+ // TaskManager may be registered on these two messages
+ case msg @ (_: RegisterTaskManager | _: RegisterResourceSuccessful) =>
+ super.handleMessage(msg)
+
+ // dequeue all senders which wait for instanceManager.getNumberOfRegisteredTaskManagers or
+ // fewer registered TaskManagers
+ while (waitForNumRegisteredTaskManagers.nonEmpty &&
+ waitForNumRegisteredTaskManagers.head._1 <=
+ instanceManager.getNumberOfRegisteredTaskManagers) {
+ val receiver = waitForNumRegisteredTaskManagers.dequeue()._2
+ receiver ! Acknowledge
+ }
+ }
+
+ def checkIfAllVerticesRunning(jobID: JobID): Boolean = {
+ currentJobs.get(jobID) match {
+ case Some((eg, _)) =>
+ eg.getAllExecutionVertices.asScala.forall( _.getExecutionState == ExecutionState.RUNNING)
+ case None => false
+ }
+ }
+
+ def checkIfAllVerticesRunningOrFinished(jobID: JobID): Boolean = {
+ currentJobs.get(jobID) match {
+ case Some((eg, _)) =>
+ eg.getAllExecutionVertices.asScala.forall {
+ case vertex =>
+ (vertex.getExecutionState == ExecutionState.RUNNING
+ || vertex.getExecutionState == ExecutionState.FINISHED)
+ }
+ case None => false
+ }
+ }
+
+ def notifyListeners(jobID: JobID): Unit = {
+ if(checkIfAllVerticesRunning(jobID)) {
+ waitForAllVerticesToBeRunning.remove(jobID) match {
+ case Some(listeners) =>
+ for (listener <- listeners) {
+ listener ! decorateMessage(AllVerticesRunning(jobID))
+ }
+ case _ =>
+ }
+ }
+
+ if(checkIfAllVerticesRunningOrFinished(jobID)) {
+ waitForAllVerticesToBeRunningOrFinished.remove(jobID) match {
+ case Some(listeners) =>
+ for (listener <- listeners) {
+ listener ! decorateMessage(AllVerticesRunning(jobID))
+ }
+ case _ =>
+ }
+ }
+ }
+
+ /**
+ * No killing of the VM for testing.
+ */
+ override protected def shutdown(): Unit = {
+ log.info("Shutting down TestingJobManager.")
+ waitForShutdown.foreach(_ ! ComponentShutdown(self))
+ waitForShutdown.clear()
+ }
+}