You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nemo.apache.org by sa...@apache.org on 2018/11/28 08:55:57 UTC
[incubator-nemo] branch master updated: [NEMO-244] Organize
integration tests and resources (#146)
This is an automated email from the ASF dual-hosted git repository.
sanha pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-nemo.git
The following commit(s) were added to refs/heads/master by this push:
new 9341b21 [NEMO-244] Organize integration tests and resources (#146)
9341b21 is described below
commit 9341b210b410491015fa8b0aa11e18d4b2d8af1c
Author: Soojeong Kim <so...@snu.ac.kr>
AuthorDate: Wed Nov 28 17:55:52 2018 +0900
[NEMO-244] Organize integration tests and resources (#146)
JIRA: [NEMO-244: Organize integration tests and resources](https://issues.apache.org/jira/browse/NEMO-244)
**Major changes:**
- Singleton global variables (TIMEOUT, fileBasePath)
- Separate directory for each test's resources (inputs, outputs, executors)
**Minor changes to note:**
- N/A
**Tests for the changes:**
- Integration tests for beam and spark
---
.../apache/nemo/common/test/ExampleTestArgs.java | 38 +++++++++++++++++
.../org/apache/nemo/compiler/CompilerTestUtil.java | 10 ++---
.../beam/AlternatingLeastSquareITCase.java | 20 ++++-----
.../nemo/examples/beam/BeamSimpleSumSQLITCase.java | 15 ++++---
.../apache/nemo/examples/beam/BroadcastITCase.java | 21 +++++-----
.../beam/MultinomialLogisticRegressionITCase.java | 9 ++--
.../examples/beam/NetworkTraceAnalysisITCase.java | 27 ++++++------
.../beam/PartitionWordsByLengthITCase.java | 27 ++++++------
.../nemo/examples/beam/PerKeyMedianITCase.java | 19 ++++-----
.../examples/beam/PerPercentileAverageITCase.java | 19 ++++-----
.../examples/beam/WindowedWordCountITCase.java | 42 +++++++++----------
.../apache/nemo/examples/beam/WordCountITCase.java | 33 +++++++--------
.../beam_test_executor_resources.json | 0
.../beam_test_one_executor_resources.json | 0
.../beam_test_poisoned_executor_resources.json | 0
.../spark_test_executor_resources.json | 0
examples/resources/{ => inputs}/test_input_als | 0
.../{ => inputs}/test_input_employees.json | 0
examples/resources/{ => inputs}/test_input_median | 0
examples/resources/{ => inputs}/test_input_mlr | 0
.../resources/{ => inputs}/test_input_network0 | 0
.../resources/{ => inputs}/test_input_network1 | 0
.../resources/{ => inputs}/test_input_partition | 0
.../resources/{ => inputs}/test_input_people.json | 0
.../resources/{ => inputs}/test_input_people.txt | 0
examples/resources/{ => inputs}/test_input_tag | 0
.../{ => inputs}/test_input_windowed_wordcount | 0
.../resources/{ => inputs}/test_input_wordcount | 0
.../{ => inputs}/test_input_wordcount_spark | 0
.../resources/{ => outputs}/expected_output_als | 0
.../{ => outputs}/expected_output_broadcast | 0
.../resources/{ => outputs}/expected_output_median | 0
.../{ => outputs}/expected_output_network | 0
.../{ => outputs}/expected_output_partition_0 | 0
.../{ => outputs}/expected_output_partition_1 | 0
.../{ => outputs}/expected_output_partition_2 | 0
.../{ => outputs}/expected_output_partition_3 | 0
.../{ => outputs}/expected_output_partition_4 | 0
.../{ => outputs}/expected_output_partition_5 | 0
.../{ => outputs}/expected_output_partition_6 | 0
.../{ => outputs}/expected_output_partition_7 | 0
.../{ => outputs}/expected_output_partition_8 | 0
.../{ => outputs}/expected_output_partition_9 | 0
.../expected_output_reversed_wordcount_spark | 0
.../{ => outputs}/expected_output_simplesql | 0
.../expected_output_sliding_windowed_wordcount | 0
.../{ => outputs}/expected_output_tag_long | 0
.../{ => outputs}/expected_output_tag_short | 0
.../{ => outputs}/expected_output_tag_very_long | 0
.../expected_output_tag_very_very_long | 0
.../expected_output_windowed_wordcount | 0
.../expected_output_word_and_line_count | 0
.../{ => outputs}/expected_output_wordcount | 0
.../{ => outputs}/expected_output_wordcount_spark | 0
.../org/apache/nemo/examples/spark/MRJava.java | 33 +++++++--------
.../org/apache/nemo/examples/spark/SparkJava.java | 29 +++++++------
.../org/apache/nemo/examples/spark/SparkScala.java | 49 +++++++++++-----------
57 files changed, 207 insertions(+), 184 deletions(-)
diff --git a/common/src/main/java/org/apache/nemo/common/test/ExampleTestArgs.java b/common/src/main/java/org/apache/nemo/common/test/ExampleTestArgs.java
new file mode 100644
index 0000000..7158617
--- /dev/null
+++ b/common/src/main/java/org/apache/nemo/common/test/ExampleTestArgs.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.nemo.common.test;
+
+/**
+ * Test Arguments for Examples.
+ */
+public final class ExampleTestArgs {
+ public static final int TIMEOUT = 240000;
+ private static String fileBasePath;
+
+ private ExampleTestArgs() {
+
+ }
+
+ public static String getFileBasePath() {
+ if (fileBasePath == null) {
+ fileBasePath = System.getProperty("user.dir") + "/../resources/";
+ }
+ return fileBasePath;
+ }
+}
diff --git a/compiler/test/src/main/java/org/apache/nemo/compiler/CompilerTestUtil.java b/compiler/test/src/main/java/org/apache/nemo/compiler/CompilerTestUtil.java
index 6f34817..2a4d359 100644
--- a/compiler/test/src/main/java/org/apache/nemo/compiler/CompilerTestUtil.java
+++ b/compiler/test/src/main/java/org/apache/nemo/compiler/CompilerTestUtil.java
@@ -93,8 +93,8 @@ public final class CompilerTestUtil {
}
public static DAG<IRVertex, IREdge> compileWordCountDAG() throws Exception {
- final String input = ROOT_DIR + "/examples/resources/test_input_wordcount";
- final String output = ROOT_DIR + "/examples/resources/test_output";
+ final String input = ROOT_DIR + "/examples/resources/inputs/test_input_wordcount";
+ final String output = ROOT_DIR + "/examples/resources/inputs/test_output";
final String main = "org.apache.nemo.examples.beam.WordCount";
final ArgBuilder mrArgBuilder = new ArgBuilder()
@@ -105,7 +105,7 @@ public final class CompilerTestUtil {
}
public static DAG<IRVertex, IREdge> compileALSDAG() throws Exception {
- final String input = ROOT_DIR + "/examples/resources/test_input_als";
+ final String input = ROOT_DIR + "/examples/resources/inputs/test_input_als";
final String numFeatures = "10";
final String numIteration = "3";
final String main = "org.apache.nemo.examples.beam.AlternatingLeastSquare";
@@ -118,7 +118,7 @@ public final class CompilerTestUtil {
}
public static DAG<IRVertex, IREdge> compileALSInefficientDAG() throws Exception {
- final String input = ROOT_DIR + "/examples/resources/test_input_als";
+ final String input = ROOT_DIR + "/examples/resources/inputs/test_input_als";
final String numFeatures = "10";
final String numIteration = "3";
final String main = "org.apache.nemo.examples.beam.AlternatingLeastSquareInefficient";
@@ -131,7 +131,7 @@ public final class CompilerTestUtil {
}
public static DAG<IRVertex, IREdge> compileMLRDAG() throws Exception {
- final String input = ROOT_DIR + "/examples/resources/test_input_mlr";
+ final String input = ROOT_DIR + "/examples/resources/inputs/test_input_mlr";
final String numFeatures = "100";
final String numClasses = "5";
final String numIteration = "3";
diff --git a/examples/beam/src/test/java/org/apache/nemo/examples/beam/AlternatingLeastSquareITCase.java b/examples/beam/src/test/java/org/apache/nemo/examples/beam/AlternatingLeastSquareITCase.java
index 3def350..095104a 100644
--- a/examples/beam/src/test/java/org/apache/nemo/examples/beam/AlternatingLeastSquareITCase.java
+++ b/examples/beam/src/test/java/org/apache/nemo/examples/beam/AlternatingLeastSquareITCase.java
@@ -20,6 +20,7 @@ package org.apache.nemo.examples.beam;
import org.apache.nemo.client.JobLauncher;
import org.apache.nemo.common.test.ArgBuilder;
+import org.apache.nemo.common.test.ExampleTestArgs;
import org.apache.nemo.common.test.ExampleTestUtil;
import org.apache.nemo.compiler.optimizer.policy.DefaultPolicy;
import org.apache.nemo.examples.beam.policy.DefaultPolicyParallelismFive;
@@ -37,16 +38,13 @@ import org.powermock.modules.junit4.PowerMockRunner;
@RunWith(PowerMockRunner.class)
@PrepareForTest(JobLauncher.class)
public final class AlternatingLeastSquareITCase {
- private static final int TIMEOUT = 240 * 1000;
private static ArgBuilder builder;
- private static final String fileBasePath = System.getProperty("user.dir") + "/../resources/";
-
- private static final String input = fileBasePath + "test_input_als";
+ private static final String input = ExampleTestArgs.getFileBasePath() + "inputs/test_input_als";
private static final String outputFileName = "test_output_als";
- private static final String output = fileBasePath + outputFileName;
- private static final String expectedOutputFileName = "expected_output_als";
- private static final String noPoisonResources = fileBasePath + "beam_test_executor_resources.json";
- private static final String poisonedResource = fileBasePath + "beam_test_poisoned_executor_resources.json";
+ private static final String output = ExampleTestArgs.getFileBasePath() + outputFileName;
+ private static final String expectedOutputFileName = "outputs/expected_output_als";
+ private static final String noPoisonResources = ExampleTestArgs.getFileBasePath() + "executors/beam_test_executor_resources.json";
+ private static final String poisonedResource = ExampleTestArgs.getFileBasePath() + "executors/beam_test_poisoned_executor_resources.json";
private static final String numFeatures = "10";
private static final String numIteration = "3";
private static final String lambda = "0.05";
@@ -61,13 +59,13 @@ public final class AlternatingLeastSquareITCase {
@After
public void tearDown() throws Exception {
try {
- ExampleTestUtil.ensureALSOutputValidity(fileBasePath, outputFileName, expectedOutputFileName);
+ ExampleTestUtil.ensureALSOutputValidity(ExampleTestArgs.getFileBasePath(), outputFileName, expectedOutputFileName);
} finally {
- ExampleTestUtil.deleteOutputFile(fileBasePath, outputFileName);
+ ExampleTestUtil.deleteOutputFile(ExampleTestArgs.getFileBasePath(), outputFileName);
}
}
- @Test (timeout = TIMEOUT)
+ @Test (timeout = ExampleTestArgs.TIMEOUT)
public void testDefault() throws Exception {
JobLauncher.main(builder
.addResourceJson(noPoisonResources)
diff --git a/examples/beam/src/test/java/org/apache/nemo/examples/beam/BeamSimpleSumSQLITCase.java b/examples/beam/src/test/java/org/apache/nemo/examples/beam/BeamSimpleSumSQLITCase.java
index af3a3ff..ccb1ca0 100644
--- a/examples/beam/src/test/java/org/apache/nemo/examples/beam/BeamSimpleSumSQLITCase.java
+++ b/examples/beam/src/test/java/org/apache/nemo/examples/beam/BeamSimpleSumSQLITCase.java
@@ -20,6 +20,7 @@ package org.apache.nemo.examples.beam;
import org.apache.nemo.client.JobLauncher;
import org.apache.nemo.common.test.ArgBuilder;
+import org.apache.nemo.common.test.ExampleTestArgs;
import org.apache.nemo.common.test.ExampleTestUtil;
import org.apache.nemo.examples.beam.policy.DefaultPolicyParallelismFive;
import org.junit.After;
@@ -35,14 +36,12 @@ import org.powermock.modules.junit4.PowerMockRunner;
@RunWith(PowerMockRunner.class)
@PrepareForTest(JobLauncher.class)
public final class BeamSimpleSumSQLITCase {
- private static final int TIMEOUT = 180000;
private static ArgBuilder builder;
- private static final String fileBasePath = System.getProperty("user.dir") + "/../resources/";
private static final String outputFileName = "test_output_simplesql";
- private static final String expectedOutputFileName = "expected_output_simplesql";
- private static final String executorResourceFileName = fileBasePath + "beam_test_executor_resources.json";
- private static final String outputFilePath = fileBasePath + outputFileName;
+ private static final String expectedOutputFileName = "outputs/expected_output_simplesql";
+ private static final String executorResourceFileName = ExampleTestArgs.getFileBasePath() + "executors/beam_test_executor_resources.json";
+ private static final String outputFilePath = ExampleTestArgs.getFileBasePath() + outputFileName;
@Before
public void setUp() throws Exception {
@@ -55,13 +54,13 @@ public final class BeamSimpleSumSQLITCase {
@After
public void tearDown() throws Exception {
try {
- ExampleTestUtil.ensureOutputValidity(fileBasePath, outputFileName, expectedOutputFileName);
+ ExampleTestUtil.ensureOutputValidity(ExampleTestArgs.getFileBasePath(), outputFileName, expectedOutputFileName);
} finally {
- ExampleTestUtil.deleteOutputFile(fileBasePath, outputFileName);
+ ExampleTestUtil.deleteOutputFile(ExampleTestArgs.getFileBasePath(), outputFileName);
}
}
- @Test (timeout = TIMEOUT)
+ @Test (timeout = ExampleTestArgs.TIMEOUT)
public void test() throws Exception {
JobLauncher.main(builder
.addJobId(BeamSimpleSumSQLITCase.class.getSimpleName())
diff --git a/examples/beam/src/test/java/org/apache/nemo/examples/beam/BroadcastITCase.java b/examples/beam/src/test/java/org/apache/nemo/examples/beam/BroadcastITCase.java
index e8e8e6c..425eaa1 100644
--- a/examples/beam/src/test/java/org/apache/nemo/examples/beam/BroadcastITCase.java
+++ b/examples/beam/src/test/java/org/apache/nemo/examples/beam/BroadcastITCase.java
@@ -20,6 +20,7 @@ package org.apache.nemo.examples.beam;
import org.apache.nemo.client.JobLauncher;
import org.apache.nemo.common.test.ArgBuilder;
+import org.apache.nemo.common.test.ExampleTestArgs;
import org.apache.nemo.common.test.ExampleTestUtil;
import org.apache.nemo.examples.beam.policy.DefaultPolicyParallelismFive;
import org.apache.nemo.examples.beam.policy.TransientResourcePolicyParallelismFive;
@@ -36,16 +37,14 @@ import org.powermock.modules.junit4.PowerMockRunner;
@RunWith(PowerMockRunner.class)
@PrepareForTest(JobLauncher.class)
public final class BroadcastITCase {
- private static final int TIMEOUT = 180000;
private static ArgBuilder builder;
- private static final String fileBasePath = System.getProperty("user.dir") + "/../resources/";
- private static final String inputFileName = "test_input_wordcount";
+ private static final String inputFileName = "inputs/test_input_wordcount";
private static final String outputFileName = "test_output_broadcast";
- private static final String expectedOutputFileName = "expected_output_broadcast";
- private static final String executorResourceFileName = fileBasePath + "beam_test_executor_resources.json";
- private static final String inputFilePath = fileBasePath + inputFileName;
- private static final String outputFilePath = fileBasePath + outputFileName;
+ private static final String expectedOutputFileName = "outputs/expected_output_broadcast";
+ private static final String executorResourceFileName = ExampleTestArgs.getFileBasePath() + "executors/beam_test_executor_resources.json";
+ private static final String inputFilePath = ExampleTestArgs.getFileBasePath() + inputFileName;
+ private static final String outputFilePath = ExampleTestArgs.getFileBasePath() + outputFileName;
@Before
public void setUp() throws Exception {
@@ -58,13 +57,13 @@ public final class BroadcastITCase {
@After
public void tearDown() throws Exception {
try {
- ExampleTestUtil.ensureOutputValidity(fileBasePath, outputFileName, expectedOutputFileName);
+ ExampleTestUtil.ensureOutputValidity(ExampleTestArgs.getFileBasePath(), outputFileName, expectedOutputFileName);
} finally {
- ExampleTestUtil.deleteOutputFile(fileBasePath, outputFileName);
+ ExampleTestUtil.deleteOutputFile(ExampleTestArgs.getFileBasePath(), outputFileName);
}
}
- @Test (timeout = TIMEOUT)
+ @Test (timeout = ExampleTestArgs.TIMEOUT)
public void test() throws Exception {
JobLauncher.main(builder
.addJobId(BroadcastITCase.class.getSimpleName())
@@ -72,7 +71,7 @@ public final class BroadcastITCase {
.build());
}
- @Test (timeout = TIMEOUT)
+ @Test (timeout = ExampleTestArgs.TIMEOUT)
public void testTransientResource() throws Exception {
JobLauncher.main(builder
.addJobId(BroadcastITCase.class.getSimpleName() + "_transient")
diff --git a/examples/beam/src/test/java/org/apache/nemo/examples/beam/MultinomialLogisticRegressionITCase.java b/examples/beam/src/test/java/org/apache/nemo/examples/beam/MultinomialLogisticRegressionITCase.java
index 60860ce..3d38e5f 100644
--- a/examples/beam/src/test/java/org/apache/nemo/examples/beam/MultinomialLogisticRegressionITCase.java
+++ b/examples/beam/src/test/java/org/apache/nemo/examples/beam/MultinomialLogisticRegressionITCase.java
@@ -20,6 +20,7 @@ package org.apache.nemo.examples.beam;
import org.apache.nemo.client.JobLauncher;
import org.apache.nemo.common.test.ArgBuilder;
+import org.apache.nemo.common.test.ExampleTestArgs;
import org.apache.nemo.compiler.optimizer.policy.DefaultPolicy;
import org.apache.nemo.examples.beam.policy.DefaultPolicyParallelismFive;
import org.junit.Before;
@@ -34,19 +35,17 @@ import org.powermock.modules.junit4.PowerMockRunner;
@RunWith(PowerMockRunner.class)
@PrepareForTest(JobLauncher.class)
public final class MultinomialLogisticRegressionITCase {
- private static final int TIMEOUT = 240000;
private static ArgBuilder builder = new ArgBuilder();
- private static final String fileBasePath = System.getProperty("user.dir") + "/../resources/";
- private static final String executorResourceFileName = fileBasePath + "beam_test_executor_resources.json";
+ private static final String executorResourceFileName = ExampleTestArgs.getFileBasePath() + "executors/beam_test_executor_resources.json";
@Before
public void setUp() throws Exception {
builder = new ArgBuilder();
}
- @Test (timeout = TIMEOUT)
+ @Test (timeout = ExampleTestArgs.TIMEOUT)
public void test() throws Exception {
- final String input = fileBasePath + "test_input_mlr";
+ final String input = ExampleTestArgs.getFileBasePath() + "inputs/test_input_mlr";
final String numFeatures = "100";
final String numClasses = "5";
final String numIteration = "3";
diff --git a/examples/beam/src/test/java/org/apache/nemo/examples/beam/NetworkTraceAnalysisITCase.java b/examples/beam/src/test/java/org/apache/nemo/examples/beam/NetworkTraceAnalysisITCase.java
index 85f714e..4005731 100644
--- a/examples/beam/src/test/java/org/apache/nemo/examples/beam/NetworkTraceAnalysisITCase.java
+++ b/examples/beam/src/test/java/org/apache/nemo/examples/beam/NetworkTraceAnalysisITCase.java
@@ -20,6 +20,7 @@ package org.apache.nemo.examples.beam;
import org.apache.nemo.client.JobLauncher;
import org.apache.nemo.common.test.ArgBuilder;
+import org.apache.nemo.common.test.ExampleTestArgs;
import org.apache.nemo.common.test.ExampleTestUtil;
import org.apache.nemo.examples.beam.policy.DefaultPolicyParallelismFive;
import org.apache.nemo.examples.beam.policy.TransientResourcePolicyParallelismFive;
@@ -34,18 +35,16 @@ import org.powermock.modules.junit4.PowerMockRunner;
@RunWith(PowerMockRunner.class)
@PrepareForTest(JobLauncher.class)
public final class NetworkTraceAnalysisITCase {
- private static final int TIMEOUT = 120000;
private static ArgBuilder builder;
- private static final String fileBasePath = System.getProperty("user.dir") + "/../resources/";
- private static final String inputFileName0 = "test_input_network0";
- private static final String inputFileName1 = "test_input_network1";
+ private static final String inputFileName0 = "inputs/test_input_network0";
+ private static final String inputFileName1 = "inputs/test_input_network1";
private static final String outputFileName = "test_output_network";
- private static final String expectedOutputFileName = "expected_output_network";
- private static final String executorResourceFileName = fileBasePath + "beam_test_executor_resources.json";
- private static final String inputFilePath0 = fileBasePath + inputFileName0;
- private static final String inputFilePath1 = fileBasePath + inputFileName1;
- private static final String outputFilePath = fileBasePath + outputFileName;
+ private static final String expectedOutputFileName = "outputs/expected_output_network";
+ private static final String executorResourceFileName = ExampleTestArgs.getFileBasePath() + "executors/beam_test_executor_resources.json";
+ private static final String inputFilePath0 = ExampleTestArgs.getFileBasePath() + inputFileName0;
+ private static final String inputFilePath1 = ExampleTestArgs.getFileBasePath() + inputFileName1;
+ private static final String outputFilePath = ExampleTestArgs.getFileBasePath() + outputFileName;
@Before
public void setUp() throws Exception {
@@ -58,13 +57,13 @@ public final class NetworkTraceAnalysisITCase {
@After
public void tearDown() throws Exception {
try {
- ExampleTestUtil.ensureOutputValidity(fileBasePath, outputFileName, expectedOutputFileName);
+ ExampleTestUtil.ensureOutputValidity(ExampleTestArgs.getFileBasePath(), outputFileName, expectedOutputFileName);
} finally {
- ExampleTestUtil.deleteOutputFile(fileBasePath, outputFileName);
+ ExampleTestUtil.deleteOutputFile(ExampleTestArgs.getFileBasePath(), outputFileName);
}
}
- @Test(timeout = TIMEOUT)
+ @Test(timeout = ExampleTestArgs.TIMEOUT)
public void test() throws Exception {
JobLauncher.main(builder
.addJobId(NetworkTraceAnalysisITCase.class.getSimpleName())
@@ -72,7 +71,7 @@ public final class NetworkTraceAnalysisITCase {
.build());
}
- @Test (timeout = TIMEOUT)
+ @Test (timeout = ExampleTestArgs.TIMEOUT)
public void testLargeShuffle() throws Exception {
JobLauncher.main(builder
.addJobId(NetworkTraceAnalysisITCase.class.getSimpleName() + "_largeshuffle")
@@ -80,7 +79,7 @@ public final class NetworkTraceAnalysisITCase {
.build());
}
- @Test (timeout = TIMEOUT)
+ @Test (timeout = ExampleTestArgs.TIMEOUT)
public void testTransientResource() throws Exception {
JobLauncher.main(builder
.addJobId(NetworkTraceAnalysisITCase.class.getSimpleName() + "_transient")
diff --git a/examples/beam/src/test/java/org/apache/nemo/examples/beam/PartitionWordsByLengthITCase.java b/examples/beam/src/test/java/org/apache/nemo/examples/beam/PartitionWordsByLengthITCase.java
index aacd204..0939105 100644
--- a/examples/beam/src/test/java/org/apache/nemo/examples/beam/PartitionWordsByLengthITCase.java
+++ b/examples/beam/src/test/java/org/apache/nemo/examples/beam/PartitionWordsByLengthITCase.java
@@ -20,6 +20,7 @@ package org.apache.nemo.examples.beam;
import org.apache.nemo.client.JobLauncher;
import org.apache.nemo.common.test.ArgBuilder;
+import org.apache.nemo.common.test.ExampleTestArgs;
import org.apache.nemo.common.test.ExampleTestUtil;
import org.apache.nemo.examples.beam.policy.DefaultPolicyParallelismFive;
import org.apache.nemo.examples.beam.policy.LargeShufflePolicyParallelismFive;
@@ -36,16 +37,14 @@ import org.powermock.modules.junit4.PowerMockRunner;
@RunWith(PowerMockRunner.class)
@PrepareForTest(JobLauncher.class)
public final class PartitionWordsByLengthITCase {
- private static final int TIMEOUT = 120000;
private static ArgBuilder builder;
- private static final String fileBasePath = System.getProperty("user.dir") + "/../resources/";
- private static final String inputFileName = "test_input_tag";
+ private static final String inputFileName = "inputs/test_input_tag";
private static final String outputFileName = "test_output_tag";
- private static final String expectedOutputFileName = "expected_output_tag";
- private static final String executorResourceFileName = fileBasePath + "beam_test_executor_resources.json";
- private static final String inputFilePath = fileBasePath + inputFileName;
- private static final String outputFilePath = fileBasePath + outputFileName;
+ private static final String expectedOutputFileName = "outputs/expected_output_tag";
+ private static final String executorResourceFileName = ExampleTestArgs.getFileBasePath() + "executors/beam_test_executor_resources.json";
+ private static final String inputFilePath = ExampleTestArgs.getFileBasePath() + inputFileName;
+ private static final String outputFilePath = ExampleTestArgs.getFileBasePath() + outputFileName;
@Before
public void setUp() throws Exception {
@@ -57,16 +56,16 @@ public final class PartitionWordsByLengthITCase {
@After
public void tearDown() throws Exception {
try {
- ExampleTestUtil.ensureOutputValidity(fileBasePath, outputFileName + "_short", expectedOutputFileName + "_short");
- ExampleTestUtil.ensureOutputValidity(fileBasePath, outputFileName + "_long", expectedOutputFileName + "_long");
- ExampleTestUtil.ensureOutputValidity(fileBasePath, outputFileName + "_very_long", expectedOutputFileName + "_very_long");
- ExampleTestUtil.ensureOutputValidity(fileBasePath, outputFileName + "_very_very_long", expectedOutputFileName + "_very_very_long");
+ ExampleTestUtil.ensureOutputValidity(ExampleTestArgs.getFileBasePath(), outputFileName + "_short", expectedOutputFileName + "_short");
+ ExampleTestUtil.ensureOutputValidity(ExampleTestArgs.getFileBasePath(), outputFileName + "_long", expectedOutputFileName + "_long");
+ ExampleTestUtil.ensureOutputValidity(ExampleTestArgs.getFileBasePath(), outputFileName + "_very_long", expectedOutputFileName + "_very_long");
+ ExampleTestUtil.ensureOutputValidity(ExampleTestArgs.getFileBasePath(), outputFileName + "_very_very_long", expectedOutputFileName + "_very_very_long");
} finally {
- ExampleTestUtil.deleteOutputFile(fileBasePath, outputFileName);
+ ExampleTestUtil.deleteOutputFile(ExampleTestArgs.getFileBasePath(), outputFileName);
}
}
- @Test (timeout = TIMEOUT)
+ @Test (timeout = ExampleTestArgs.TIMEOUT)
public void testLargeShuffle() throws Exception {
JobLauncher.main(builder
.addResourceJson(executorResourceFileName)
@@ -75,7 +74,7 @@ public final class PartitionWordsByLengthITCase {
.build());
}
- @Test (timeout = TIMEOUT)
+ @Test (timeout = ExampleTestArgs.TIMEOUT)
public void test() throws Exception {
JobLauncher.main(builder
.addResourceJson(executorResourceFileName)
diff --git a/examples/beam/src/test/java/org/apache/nemo/examples/beam/PerKeyMedianITCase.java b/examples/beam/src/test/java/org/apache/nemo/examples/beam/PerKeyMedianITCase.java
index 41fc6c6..39d8d28 100644
--- a/examples/beam/src/test/java/org/apache/nemo/examples/beam/PerKeyMedianITCase.java
+++ b/examples/beam/src/test/java/org/apache/nemo/examples/beam/PerKeyMedianITCase.java
@@ -20,6 +20,7 @@ package org.apache.nemo.examples.beam;
import org.apache.nemo.client.JobLauncher;
import org.apache.nemo.common.test.ArgBuilder;
+import org.apache.nemo.common.test.ExampleTestArgs;
import org.apache.nemo.common.test.ExampleTestUtil;
import org.apache.nemo.examples.beam.policy.DataSkewPolicyParallelismFive;
import org.junit.After;
@@ -35,16 +36,14 @@ import org.powermock.modules.junit4.PowerMockRunner;
@RunWith(PowerMockRunner.class)
@PrepareForTest(JobLauncher.class)
public final class PerKeyMedianITCase {
- private static final int TIMEOUT = 120 * 1000;
private static ArgBuilder builder;
- private static final String fileBasePath = System.getProperty("user.dir") + "/../resources/";
- private static final String inputFileName = "test_input_median";
+ private static final String inputFileName = "inputs/test_input_median";
private static final String outputFileName = "test_output_median";
- private static final String expectedOutputFileName = "expected_output_median";
- private static final String executorResourceFileName = fileBasePath + "beam_test_executor_resources.json";
- private static final String inputFilePath = fileBasePath + inputFileName;
- private static final String outputFilePath = fileBasePath + outputFileName;
+ private static final String expectedOutputFileName = "outputs/expected_output_median";
+ private static final String executorResourceFileName = ExampleTestArgs.getFileBasePath() + "executors/beam_test_executor_resources.json";
+ private static final String inputFilePath = ExampleTestArgs.getFileBasePath() + inputFileName;
+ private static final String outputFilePath = ExampleTestArgs.getFileBasePath() + outputFileName;
@Before
public void setUp() throws Exception {
@@ -57,9 +56,9 @@ public final class PerKeyMedianITCase {
@After
public void tearDown() throws Exception {
try {
- ExampleTestUtil.ensureOutputValidity(fileBasePath, outputFileName, expectedOutputFileName);
+ ExampleTestUtil.ensureOutputValidity(ExampleTestArgs.getFileBasePath(), outputFileName, expectedOutputFileName);
} finally {
- ExampleTestUtil.deleteOutputFile(fileBasePath, outputFileName);
+ ExampleTestUtil.deleteOutputFile(ExampleTestArgs.getFileBasePath(), outputFileName);
}
}
@@ -67,7 +66,7 @@ public final class PerKeyMedianITCase {
* Testing data skew dynamic optimization.
* @throws Exception exception on the way.
*/
- @Test (timeout = TIMEOUT)
+ @Test (timeout = ExampleTestArgs.TIMEOUT)
public void testDataSkew() throws Exception {
JobLauncher.main(builder
.addJobId(PerKeyMedianITCase.class.getSimpleName())
diff --git a/examples/beam/src/test/java/org/apache/nemo/examples/beam/PerPercentileAverageITCase.java b/examples/beam/src/test/java/org/apache/nemo/examples/beam/PerPercentileAverageITCase.java
index 76e238c..b43a939 100644
--- a/examples/beam/src/test/java/org/apache/nemo/examples/beam/PerPercentileAverageITCase.java
+++ b/examples/beam/src/test/java/org/apache/nemo/examples/beam/PerPercentileAverageITCase.java
@@ -20,6 +20,7 @@ package org.apache.nemo.examples.beam;
import org.apache.nemo.client.JobLauncher;
import org.apache.nemo.common.test.ArgBuilder;
+import org.apache.nemo.common.test.ExampleTestArgs;
import org.apache.nemo.common.test.ExampleTestUtil;
import org.apache.nemo.examples.beam.policy.DefaultPolicyParallelismFive;
import org.junit.After;
@@ -35,16 +36,14 @@ import org.powermock.modules.junit4.PowerMockRunner;
@RunWith(PowerMockRunner.class)
@PrepareForTest(JobLauncher.class)
public final class PerPercentileAverageITCase {
- private static final int TIMEOUT = 120000;
private static ArgBuilder builder;
- private static final String fileBasePath = System.getProperty("user.dir") + "/../resources/";
- private static final String inputFileName = "test_input_partition";
+ private static final String inputFileName = "inputs/test_input_partition";
private static final String outputFileName = "test_output_partition";
- private static final String expectedOutputFileName = "expected_output_partition";
- private static final String executorResourceFileName = fileBasePath + "beam_test_executor_resources.json";
- private static final String inputFilePath = fileBasePath + inputFileName;
- private static final String outputFilePath = fileBasePath + outputFileName;
+ private static final String expectedOutputFileName = "outputs/expected_output_partition";
+ private static final String executorResourceFileName = ExampleTestArgs.getFileBasePath() + "executors/beam_test_executor_resources.json";
+ private static final String inputFilePath = ExampleTestArgs.getFileBasePath() + inputFileName;
+ private static final String outputFilePath = ExampleTestArgs.getFileBasePath() + outputFileName;
@Before
public void setUp() throws Exception {
@@ -58,16 +57,16 @@ public final class PerPercentileAverageITCase {
public void tearDown() throws Exception {
try {
for (int i = 0; i < 10; i++) {
- ExampleTestUtil.ensureOutputValidity(fileBasePath,
+ ExampleTestUtil.ensureOutputValidity(ExampleTestArgs.getFileBasePath(),
outputFileName + "_" + i,
expectedOutputFileName + "_" + i);
}
} finally {
- ExampleTestUtil.deleteOutputFile(fileBasePath, outputFileName);
+ ExampleTestUtil.deleteOutputFile(ExampleTestArgs.getFileBasePath(), outputFileName);
}
}
- @Test (timeout = TIMEOUT)
+ @Test (timeout = ExampleTestArgs.TIMEOUT)
public void test() throws Exception {
JobLauncher.main(builder
.addJobId(PerPercentileAverage.class.getSimpleName())
diff --git a/examples/beam/src/test/java/org/apache/nemo/examples/beam/WindowedWordCountITCase.java b/examples/beam/src/test/java/org/apache/nemo/examples/beam/WindowedWordCountITCase.java
index 7ef0e02..97dafe5 100644
--- a/examples/beam/src/test/java/org/apache/nemo/examples/beam/WindowedWordCountITCase.java
+++ b/examples/beam/src/test/java/org/apache/nemo/examples/beam/WindowedWordCountITCase.java
@@ -20,6 +20,7 @@ package org.apache.nemo.examples.beam;
import org.apache.nemo.client.JobLauncher;
import org.apache.nemo.common.test.ArgBuilder;
+import org.apache.nemo.common.test.ExampleTestArgs;
import org.apache.nemo.common.test.ExampleTestUtil;
import org.apache.nemo.compiler.optimizer.policy.DefaultPolicy;
import org.apache.nemo.examples.beam.policy.*;
@@ -38,20 +39,17 @@ import static org.apache.nemo.examples.beam.WindowedWordCount.INPUT_TYPE_UNBOUND
@RunWith(PowerMockRunner.class)
@PrepareForTest(JobLauncher.class)
public final class WindowedWordCountITCase {
-
- private static final int TIMEOUT = 120000;
private static ArgBuilder builder;
- private static final String fileBasePath = System.getProperty("user.dir") + "/../resources/";
- private static final String inputFileName = "test_input_windowed_wordcount";
+ private static final String inputFileName = "inputs/test_input_windowed_wordcount";
private static final String outputFileName = "test_output_windowed_wordcount";
- private static final String expectedOutputFileName = "expected_output_windowed_wordcount";
- private static final String expectedSlidingWindowOutputFileName = "expected_output_sliding_windowed_wordcount";
- private static final String executorResourceFileName = fileBasePath + "beam_test_executor_resources.json";
- private static final String inputFilePath = fileBasePath + inputFileName;
- private static final String outputFilePath = fileBasePath + outputFileName;
+ private static final String expectedOutputFileName = "outputs/expected_output_windowed_wordcount";
+ private static final String expectedSlidingWindowOutputFileName = "outputs/expected_output_sliding_windowed_wordcount";
+ private static final String executorResourceFileName = ExampleTestArgs.getFileBasePath() + "executors/beam_test_executor_resources.json";
+ private static final String inputFilePath = ExampleTestArgs.getFileBasePath() + inputFileName;
+ private static final String outputFilePath = ExampleTestArgs.getFileBasePath() + outputFileName;
- @Test (timeout = TIMEOUT)
+ @Test (timeout = ExampleTestArgs.TIMEOUT)
public void testBatchFixedWindow() throws Exception {
builder = new ArgBuilder()
.addUserMain(WindowedWordCount.class.getCanonicalName())
@@ -64,14 +62,14 @@ public final class WindowedWordCountITCase {
.build());
try {
- ExampleTestUtil.ensureOutputValidity(fileBasePath, outputFileName, expectedOutputFileName);
+ ExampleTestUtil.ensureOutputValidity(ExampleTestArgs.getFileBasePath(), outputFileName, expectedOutputFileName);
} finally {
- ExampleTestUtil.deleteOutputFile(fileBasePath, outputFileName);
+ ExampleTestUtil.deleteOutputFile(ExampleTestArgs.getFileBasePath(), outputFileName);
}
}
- @Test (timeout = TIMEOUT)
+ @Test (timeout = ExampleTestArgs.TIMEOUT)
public void testBatchSlidingWindow() throws Exception {
builder = new ArgBuilder()
.addUserMain(WindowedWordCount.class.getCanonicalName())
@@ -84,13 +82,13 @@ public final class WindowedWordCountITCase {
.build());
try {
- ExampleTestUtil.ensureOutputValidity(fileBasePath, outputFileName, expectedSlidingWindowOutputFileName);
+ ExampleTestUtil.ensureOutputValidity(ExampleTestArgs.getFileBasePath(), outputFileName, expectedSlidingWindowOutputFileName);
} finally {
- ExampleTestUtil.deleteOutputFile(fileBasePath, outputFileName);
+ ExampleTestUtil.deleteOutputFile(ExampleTestArgs.getFileBasePath(), outputFileName);
}
}
- @Test (timeout = TIMEOUT)
+ @Test (timeout = ExampleTestArgs.TIMEOUT)
public void testStreamingSchedulerAndPipeFixedWindow() throws Exception {
builder = new ArgBuilder()
.addScheduler("org.apache.nemo.runtime.master.scheduler.StreamingScheduler")
@@ -104,14 +102,14 @@ public final class WindowedWordCountITCase {
.build());
try {
- ExampleTestUtil.ensureOutputValidity(fileBasePath, outputFileName, expectedOutputFileName);
+ ExampleTestUtil.ensureOutputValidity(ExampleTestArgs.getFileBasePath(), outputFileName, expectedOutputFileName);
} finally {
- ExampleTestUtil.deleteOutputFile(fileBasePath, outputFileName);
+ ExampleTestUtil.deleteOutputFile(ExampleTestArgs.getFileBasePath(), outputFileName);
}
}
- @Test (timeout = TIMEOUT)
+ @Test (timeout = ExampleTestArgs.TIMEOUT)
public void testStreamingSchedulerAndPipeSlidingWindow() throws Exception {
builder = new ArgBuilder()
.addScheduler("org.apache.nemo.runtime.master.scheduler.StreamingScheduler")
@@ -125,9 +123,9 @@ public final class WindowedWordCountITCase {
.build());
try {
- ExampleTestUtil.ensureOutputValidity(fileBasePath, outputFileName, expectedSlidingWindowOutputFileName);
+ ExampleTestUtil.ensureOutputValidity(ExampleTestArgs.getFileBasePath(), outputFileName, expectedSlidingWindowOutputFileName);
} finally {
- ExampleTestUtil.deleteOutputFile(fileBasePath, outputFileName);
+ ExampleTestUtil.deleteOutputFile(ExampleTestArgs.getFileBasePath(), outputFileName);
}
}
@@ -147,7 +145,7 @@ public final class WindowedWordCountITCase {
.build());
try {
- ExampleTestUtil.ensureOutputValidity(fileBasePath, outputFileName, expectedSlidingWindowOutputFileName);
+ ExampleTestUtil.ensureOutputValidity(ExampleTestArgs.getFileBasePath(), outputFileName, expectedSlidingWindowOutputFileName);
} finally {
}
}
diff --git a/examples/beam/src/test/java/org/apache/nemo/examples/beam/WordCountITCase.java b/examples/beam/src/test/java/org/apache/nemo/examples/beam/WordCountITCase.java
index 052bb32..e31a8c5 100644
--- a/examples/beam/src/test/java/org/apache/nemo/examples/beam/WordCountITCase.java
+++ b/examples/beam/src/test/java/org/apache/nemo/examples/beam/WordCountITCase.java
@@ -20,6 +20,7 @@ package org.apache.nemo.examples.beam;
import org.apache.nemo.client.JobLauncher;
import org.apache.nemo.common.test.ArgBuilder;
+import org.apache.nemo.common.test.ExampleTestArgs;
import org.apache.nemo.common.test.ExampleTestUtil;
import org.apache.nemo.compiler.optimizer.policy.ConditionalLargeShufflePolicy;
import org.apache.nemo.examples.beam.policy.*;
@@ -36,17 +37,15 @@ import org.powermock.modules.junit4.PowerMockRunner;
@RunWith(PowerMockRunner.class)
@PrepareForTest(JobLauncher.class)
public final class WordCountITCase {
- private static final int TIMEOUT = 120000;
private static ArgBuilder builder;
- private static final String fileBasePath = System.getProperty("user.dir") + "/../resources/";
- private static final String inputFileName = "test_input_wordcount";
+ private static final String inputFileName = "inputs/test_input_wordcount";
private static final String outputFileName = "test_output_wordcount";
- private static final String expectedOutputFileName = "expected_output_wordcount";
- private static final String executorResourceFileName = fileBasePath + "beam_test_executor_resources.json";
- private static final String oneExecutorResourceFileName = fileBasePath + "beam_test_one_executor_resources.json";
- private static final String inputFilePath = fileBasePath + inputFileName;
- private static final String outputFilePath = fileBasePath + outputFileName;
+ private static final String expectedOutputFileName = "outputs/expected_output_wordcount";
+ private static final String executorResourceFileName = ExampleTestArgs.getFileBasePath() + "executors/beam_test_executor_resources.json";
+ private static final String oneExecutorResourceFileName = ExampleTestArgs.getFileBasePath() + "executors/beam_test_one_executor_resources.json";
+ private static final String inputFilePath = ExampleTestArgs.getFileBasePath() + inputFileName;
+ private static final String outputFilePath = ExampleTestArgs.getFileBasePath() + outputFileName;
@Before
public void setUp() throws Exception {
@@ -58,13 +57,13 @@ public final class WordCountITCase {
@After
public void tearDown() throws Exception {
try {
- ExampleTestUtil.ensureOutputValidity(fileBasePath, outputFileName, expectedOutputFileName);
+ ExampleTestUtil.ensureOutputValidity(ExampleTestArgs.getFileBasePath(), outputFileName, expectedOutputFileName);
} finally {
- ExampleTestUtil.deleteOutputFile(fileBasePath, outputFileName);
+ ExampleTestUtil.deleteOutputFile(ExampleTestArgs.getFileBasePath(), outputFileName);
}
}
- @Test (timeout = TIMEOUT)
+ @Test (timeout = ExampleTestArgs.TIMEOUT)
public void test() throws Exception {
JobLauncher.main(builder
.addResourceJson(executorResourceFileName)
@@ -73,7 +72,7 @@ public final class WordCountITCase {
.build());
}
- @Test (timeout = TIMEOUT)
+ @Test (timeout = ExampleTestArgs.TIMEOUT)
public void testLargeShuffle() throws Exception {
JobLauncher.main(builder
.addResourceJson(executorResourceFileName)
@@ -82,7 +81,7 @@ public final class WordCountITCase {
.build());
}
- @Test (timeout = TIMEOUT)
+ @Test (timeout = ExampleTestArgs.TIMEOUT)
public void testLargeShuffleInOneExecutor() throws Exception {
JobLauncher.main(builder
.addResourceJson(oneExecutorResourceFileName)
@@ -91,7 +90,7 @@ public final class WordCountITCase {
.build());
}
- @Test (timeout = TIMEOUT)
+ @Test (timeout = ExampleTestArgs.TIMEOUT)
public void testConditionalLargeShuffle() throws Exception {
JobLauncher.main(builder
.addResourceJson(executorResourceFileName)
@@ -100,7 +99,7 @@ public final class WordCountITCase {
.build());
}
- @Test (timeout = TIMEOUT)
+ @Test (timeout = ExampleTestArgs.TIMEOUT)
public void testTransientResource() throws Exception {
JobLauncher.main(builder
.addResourceJson(executorResourceFileName)
@@ -109,7 +108,7 @@ public final class WordCountITCase {
.build());
}
- @Test (timeout = TIMEOUT)
+ @Test (timeout = ExampleTestArgs.TIMEOUT)
public void testClonedScheduling() throws Exception {
JobLauncher.main(builder
.addResourceJson(executorResourceFileName)
@@ -119,7 +118,7 @@ public final class WordCountITCase {
.build());
}
- @Test (timeout = TIMEOUT)
+ @Test (timeout = ExampleTestArgs.TIMEOUT)
public void testSpeculativeExecution() throws Exception {
JobLauncher.main(builder
.addResourceJson(executorResourceFileName)
diff --git a/examples/resources/beam_test_executor_resources.json b/examples/resources/executors/beam_test_executor_resources.json
similarity index 100%
rename from examples/resources/beam_test_executor_resources.json
rename to examples/resources/executors/beam_test_executor_resources.json
diff --git a/examples/resources/beam_test_one_executor_resources.json b/examples/resources/executors/beam_test_one_executor_resources.json
similarity index 100%
rename from examples/resources/beam_test_one_executor_resources.json
rename to examples/resources/executors/beam_test_one_executor_resources.json
diff --git a/examples/resources/beam_test_poisoned_executor_resources.json b/examples/resources/executors/beam_test_poisoned_executor_resources.json
similarity index 100%
rename from examples/resources/beam_test_poisoned_executor_resources.json
rename to examples/resources/executors/beam_test_poisoned_executor_resources.json
diff --git a/examples/resources/spark_test_executor_resources.json b/examples/resources/executors/spark_test_executor_resources.json
similarity index 100%
rename from examples/resources/spark_test_executor_resources.json
rename to examples/resources/executors/spark_test_executor_resources.json
diff --git a/examples/resources/test_input_als b/examples/resources/inputs/test_input_als
similarity index 100%
rename from examples/resources/test_input_als
rename to examples/resources/inputs/test_input_als
diff --git a/examples/resources/test_input_employees.json b/examples/resources/inputs/test_input_employees.json
similarity index 100%
rename from examples/resources/test_input_employees.json
rename to examples/resources/inputs/test_input_employees.json
diff --git a/examples/resources/test_input_median b/examples/resources/inputs/test_input_median
similarity index 100%
rename from examples/resources/test_input_median
rename to examples/resources/inputs/test_input_median
diff --git a/examples/resources/test_input_mlr b/examples/resources/inputs/test_input_mlr
similarity index 100%
rename from examples/resources/test_input_mlr
rename to examples/resources/inputs/test_input_mlr
diff --git a/examples/resources/test_input_network0 b/examples/resources/inputs/test_input_network0
similarity index 100%
rename from examples/resources/test_input_network0
rename to examples/resources/inputs/test_input_network0
diff --git a/examples/resources/test_input_network1 b/examples/resources/inputs/test_input_network1
similarity index 100%
rename from examples/resources/test_input_network1
rename to examples/resources/inputs/test_input_network1
diff --git a/examples/resources/test_input_partition b/examples/resources/inputs/test_input_partition
similarity index 100%
rename from examples/resources/test_input_partition
rename to examples/resources/inputs/test_input_partition
diff --git a/examples/resources/test_input_people.json b/examples/resources/inputs/test_input_people.json
similarity index 100%
rename from examples/resources/test_input_people.json
rename to examples/resources/inputs/test_input_people.json
diff --git a/examples/resources/test_input_people.txt b/examples/resources/inputs/test_input_people.txt
similarity index 100%
rename from examples/resources/test_input_people.txt
rename to examples/resources/inputs/test_input_people.txt
diff --git a/examples/resources/test_input_tag b/examples/resources/inputs/test_input_tag
similarity index 100%
rename from examples/resources/test_input_tag
rename to examples/resources/inputs/test_input_tag
diff --git a/examples/resources/test_input_windowed_wordcount b/examples/resources/inputs/test_input_windowed_wordcount
similarity index 100%
rename from examples/resources/test_input_windowed_wordcount
rename to examples/resources/inputs/test_input_windowed_wordcount
diff --git a/examples/resources/test_input_wordcount b/examples/resources/inputs/test_input_wordcount
similarity index 100%
rename from examples/resources/test_input_wordcount
rename to examples/resources/inputs/test_input_wordcount
diff --git a/examples/resources/test_input_wordcount_spark b/examples/resources/inputs/test_input_wordcount_spark
similarity index 100%
rename from examples/resources/test_input_wordcount_spark
rename to examples/resources/inputs/test_input_wordcount_spark
diff --git a/examples/resources/expected_output_als b/examples/resources/outputs/expected_output_als
similarity index 100%
rename from examples/resources/expected_output_als
rename to examples/resources/outputs/expected_output_als
diff --git a/examples/resources/expected_output_broadcast b/examples/resources/outputs/expected_output_broadcast
similarity index 100%
rename from examples/resources/expected_output_broadcast
rename to examples/resources/outputs/expected_output_broadcast
diff --git a/examples/resources/expected_output_median b/examples/resources/outputs/expected_output_median
similarity index 100%
rename from examples/resources/expected_output_median
rename to examples/resources/outputs/expected_output_median
diff --git a/examples/resources/expected_output_network b/examples/resources/outputs/expected_output_network
similarity index 100%
rename from examples/resources/expected_output_network
rename to examples/resources/outputs/expected_output_network
diff --git a/examples/resources/expected_output_partition_0 b/examples/resources/outputs/expected_output_partition_0
similarity index 100%
rename from examples/resources/expected_output_partition_0
rename to examples/resources/outputs/expected_output_partition_0
diff --git a/examples/resources/expected_output_partition_1 b/examples/resources/outputs/expected_output_partition_1
similarity index 100%
rename from examples/resources/expected_output_partition_1
rename to examples/resources/outputs/expected_output_partition_1
diff --git a/examples/resources/expected_output_partition_2 b/examples/resources/outputs/expected_output_partition_2
similarity index 100%
rename from examples/resources/expected_output_partition_2
rename to examples/resources/outputs/expected_output_partition_2
diff --git a/examples/resources/expected_output_partition_3 b/examples/resources/outputs/expected_output_partition_3
similarity index 100%
rename from examples/resources/expected_output_partition_3
rename to examples/resources/outputs/expected_output_partition_3
diff --git a/examples/resources/expected_output_partition_4 b/examples/resources/outputs/expected_output_partition_4
similarity index 100%
rename from examples/resources/expected_output_partition_4
rename to examples/resources/outputs/expected_output_partition_4
diff --git a/examples/resources/expected_output_partition_5 b/examples/resources/outputs/expected_output_partition_5
similarity index 100%
rename from examples/resources/expected_output_partition_5
rename to examples/resources/outputs/expected_output_partition_5
diff --git a/examples/resources/expected_output_partition_6 b/examples/resources/outputs/expected_output_partition_6
similarity index 100%
rename from examples/resources/expected_output_partition_6
rename to examples/resources/outputs/expected_output_partition_6
diff --git a/examples/resources/expected_output_partition_7 b/examples/resources/outputs/expected_output_partition_7
similarity index 100%
rename from examples/resources/expected_output_partition_7
rename to examples/resources/outputs/expected_output_partition_7
diff --git a/examples/resources/expected_output_partition_8 b/examples/resources/outputs/expected_output_partition_8
similarity index 100%
rename from examples/resources/expected_output_partition_8
rename to examples/resources/outputs/expected_output_partition_8
diff --git a/examples/resources/expected_output_partition_9 b/examples/resources/outputs/expected_output_partition_9
similarity index 100%
rename from examples/resources/expected_output_partition_9
rename to examples/resources/outputs/expected_output_partition_9
diff --git a/examples/resources/expected_output_reversed_wordcount_spark b/examples/resources/outputs/expected_output_reversed_wordcount_spark
similarity index 100%
rename from examples/resources/expected_output_reversed_wordcount_spark
rename to examples/resources/outputs/expected_output_reversed_wordcount_spark
diff --git a/examples/resources/expected_output_simplesql b/examples/resources/outputs/expected_output_simplesql
similarity index 100%
rename from examples/resources/expected_output_simplesql
rename to examples/resources/outputs/expected_output_simplesql
diff --git a/examples/resources/expected_output_sliding_windowed_wordcount b/examples/resources/outputs/expected_output_sliding_windowed_wordcount
similarity index 100%
rename from examples/resources/expected_output_sliding_windowed_wordcount
rename to examples/resources/outputs/expected_output_sliding_windowed_wordcount
diff --git a/examples/resources/expected_output_tag_long b/examples/resources/outputs/expected_output_tag_long
similarity index 100%
rename from examples/resources/expected_output_tag_long
rename to examples/resources/outputs/expected_output_tag_long
diff --git a/examples/resources/expected_output_tag_short b/examples/resources/outputs/expected_output_tag_short
similarity index 100%
rename from examples/resources/expected_output_tag_short
rename to examples/resources/outputs/expected_output_tag_short
diff --git a/examples/resources/expected_output_tag_very_long b/examples/resources/outputs/expected_output_tag_very_long
similarity index 100%
rename from examples/resources/expected_output_tag_very_long
rename to examples/resources/outputs/expected_output_tag_very_long
diff --git a/examples/resources/expected_output_tag_very_very_long b/examples/resources/outputs/expected_output_tag_very_very_long
similarity index 100%
rename from examples/resources/expected_output_tag_very_very_long
rename to examples/resources/outputs/expected_output_tag_very_very_long
diff --git a/examples/resources/expected_output_windowed_wordcount b/examples/resources/outputs/expected_output_windowed_wordcount
similarity index 100%
rename from examples/resources/expected_output_windowed_wordcount
rename to examples/resources/outputs/expected_output_windowed_wordcount
diff --git a/examples/resources/expected_output_word_and_line_count b/examples/resources/outputs/expected_output_word_and_line_count
similarity index 100%
rename from examples/resources/expected_output_word_and_line_count
rename to examples/resources/outputs/expected_output_word_and_line_count
diff --git a/examples/resources/expected_output_wordcount b/examples/resources/outputs/expected_output_wordcount
similarity index 100%
rename from examples/resources/expected_output_wordcount
rename to examples/resources/outputs/expected_output_wordcount
diff --git a/examples/resources/expected_output_wordcount_spark b/examples/resources/outputs/expected_output_wordcount_spark
similarity index 100%
rename from examples/resources/expected_output_wordcount_spark
rename to examples/resources/outputs/expected_output_wordcount_spark
diff --git a/examples/spark/src/test/java/org/apache/nemo/examples/spark/MRJava.java b/examples/spark/src/test/java/org/apache/nemo/examples/spark/MRJava.java
index d89ef8b..39ecae5 100644
--- a/examples/spark/src/test/java/org/apache/nemo/examples/spark/MRJava.java
+++ b/examples/spark/src/test/java/org/apache/nemo/examples/spark/MRJava.java
@@ -20,6 +20,7 @@ package org.apache.nemo.examples.spark;
import org.apache.nemo.client.JobLauncher;
import org.apache.nemo.common.test.ArgBuilder;
+import org.apache.nemo.common.test.ExampleTestArgs;
import org.apache.nemo.common.test.ExampleTestUtil;
import org.apache.nemo.compiler.optimizer.policy.DefaultPolicy;
import org.junit.Before;
@@ -36,10 +37,8 @@ import org.powermock.modules.junit4.PowerMockRunner;
@PrepareForTest(JobLauncher.class)
@PowerMockIgnore("javax.management.*")
public final class MRJava {
- private static final int TIMEOUT = 180000;
private static ArgBuilder builder;
- private static final String fileBasePath = System.getProperty("user.dir") + "/../resources/";
- private static final String executorResourceFileName = fileBasePath + "spark_test_executor_resources.json";
+ private static final String executorResourceFileName = ExampleTestArgs.getFileBasePath() + "/executors/spark_test_executor_resources.json";
@Before
public void setUp() {
@@ -47,13 +46,13 @@ public final class MRJava {
.addResourceJson(executorResourceFileName);
}
- @Test(timeout = TIMEOUT)
+ @Test(timeout = ExampleTestArgs.TIMEOUT)
public void testSparkWordCount() throws Exception {
- final String inputFileName = "test_input_wordcount_spark";
+ final String inputFileName = "/inputs/test_input_wordcount_spark";
final String outputFileName = "test_output_wordcount_spark";
- final String expectedOutputFilename = "expected_output_wordcount_spark";
- final String inputFilePath = fileBasePath + inputFileName;
- final String outputFilePath = fileBasePath + outputFileName;
+ final String expectedOutputFilename = "/outputs/expected_output_wordcount_spark";
+ final String inputFilePath = ExampleTestArgs.getFileBasePath() + inputFileName;
+ final String outputFilePath = ExampleTestArgs.getFileBasePath() + outputFileName;
JobLauncher.main(builder
.addJobId(JavaWordCount.class.getSimpleName() + "_test")
@@ -63,19 +62,19 @@ public final class MRJava {
.build());
try {
- ExampleTestUtil.ensureOutputValidity(fileBasePath, outputFileName, expectedOutputFilename);
+ ExampleTestUtil.ensureOutputValidity(ExampleTestArgs.getFileBasePath(), outputFileName, expectedOutputFilename);
} finally {
- ExampleTestUtil.deleteOutputFile(fileBasePath, outputFileName);
+ ExampleTestUtil.deleteOutputFile(ExampleTestArgs.getFileBasePath(), outputFileName);
}
}
- @Test(timeout = TIMEOUT)
+ @Test(timeout = ExampleTestArgs.TIMEOUT)
public void testSparkWordAndLineCount() throws Exception {
- final String inputFileName = "test_input_wordcount_spark";
+ final String inputFileName = "/inputs/test_input_wordcount_spark";
final String outputFileName = "test_output_word_and_line_count";
- final String expectedOutputFilename = "expected_output_word_and_line_count";
- final String inputFilePath = fileBasePath + inputFileName;
- final String outputFilePath = fileBasePath + outputFileName;
+ final String expectedOutputFilename = "/outputs/expected_output_word_and_line_count";
+ final String inputFilePath = ExampleTestArgs.getFileBasePath() + inputFileName;
+ final String outputFilePath = ExampleTestArgs.getFileBasePath() + outputFileName;
JobLauncher.main(builder
.addJobId(JavaWordAndLineCount.class.getSimpleName() + "_test")
@@ -85,9 +84,9 @@ public final class MRJava {
.build());
try {
- ExampleTestUtil.ensureOutputValidity(fileBasePath, outputFileName, expectedOutputFilename);
+ ExampleTestUtil.ensureOutputValidity(ExampleTestArgs.getFileBasePath(), outputFileName, expectedOutputFilename);
} finally {
- ExampleTestUtil.deleteOutputFile(fileBasePath, outputFileName);
+ ExampleTestUtil.deleteOutputFile(ExampleTestArgs.getFileBasePath(), outputFileName);
}
}
}
diff --git a/examples/spark/src/test/java/org/apache/nemo/examples/spark/SparkJava.java b/examples/spark/src/test/java/org/apache/nemo/examples/spark/SparkJava.java
index b09da38..1ebbefc 100644
--- a/examples/spark/src/test/java/org/apache/nemo/examples/spark/SparkJava.java
+++ b/examples/spark/src/test/java/org/apache/nemo/examples/spark/SparkJava.java
@@ -20,6 +20,7 @@ package org.apache.nemo.examples.spark;
import org.apache.nemo.client.JobLauncher;
import org.apache.nemo.common.test.ArgBuilder;
+import org.apache.nemo.common.test.ExampleTestArgs;
import org.apache.nemo.common.test.ExampleTestUtil;
import org.apache.nemo.compiler.optimizer.policy.DefaultPolicy;
import org.apache.nemo.examples.spark.sql.JavaUserDefinedTypedAggregation;
@@ -38,10 +39,8 @@ import org.powermock.modules.junit4.PowerMockRunner;
@PrepareForTest(JobLauncher.class)
@PowerMockIgnore("javax.management.*")
public final class SparkJava {
- private static final int TIMEOUT = 180000;
private static ArgBuilder builder;
- private static final String fileBasePath = System.getProperty("user.dir") + "/../resources/";
- private static final String executorResourceFileName = fileBasePath + "spark_test_executor_resources.json";
+ private static final String executorResourceFileName = ExampleTestArgs.getFileBasePath() + "inputs/spark_test_executor_resources.json";
@Before
public void setUp() {
@@ -49,7 +48,7 @@ public final class SparkJava {
.addResourceJson(executorResourceFileName);
}
- @Test(timeout = TIMEOUT)
+ @Test(timeout = ExampleTestArgs.TIMEOUT)
public void testSparkPi() throws Exception {
final String numParallelism = "3";
@@ -61,10 +60,10 @@ public final class SparkJava {
.build());
}
- @Test(timeout = TIMEOUT)
+ @Test(timeout = ExampleTestArgs.TIMEOUT)
public void testSparkSQLUserDefinedTypedAggregation() throws Exception {
- final String inputFileName = "test_input_employees.json";
- final String inputFilePath = fileBasePath + inputFileName;
+ final String inputFileName = "inputs/test_input_employees.json";
+ final String inputFilePath = ExampleTestArgs.getFileBasePath() + inputFileName;
JobLauncher.main(builder
.addJobId(JavaUserDefinedTypedAggregation.class.getSimpleName() + "_test")
@@ -74,10 +73,10 @@ public final class SparkJava {
.build());
}
- @Test(timeout = TIMEOUT)
+ @Test(timeout = ExampleTestArgs.TIMEOUT)
public void testSparkSQLUserDefinedUntypedAggregation() throws Exception {
- final String inputFileName = "test_input_employees.json";
- final String inputFilePath = fileBasePath + inputFileName;
+ final String inputFileName = "inputs/test_input_employees.json";
+ final String inputFilePath = ExampleTestArgs.getFileBasePath() + inputFileName;
JobLauncher.main(builder
.addJobId(JavaUserDefinedUntypedAggregation.class.getSimpleName() + "_test")
@@ -87,12 +86,12 @@ public final class SparkJava {
.build());
}
- @Test(timeout = TIMEOUT)
+ @Test(timeout = ExampleTestArgs.TIMEOUT)
public void testSparkSQLExample() throws Exception {
- final String peopleJson = "test_input_people.json";
- final String peopleTxt = "test_input_people.txt";
- final String inputFileJson = fileBasePath + peopleJson;
- final String inputFileTxt = fileBasePath + peopleTxt;
+ final String peopleJson = "inputs/test_input_people.json";
+ final String peopleTxt = "inputs/test_input_people.txt";
+ final String inputFileJson = ExampleTestArgs.getFileBasePath() + peopleJson;
+ final String inputFileTxt = ExampleTestArgs.getFileBasePath() + peopleTxt;
// TODO#12: Frontend support for Scala Spark.
// JobLauncher.main(builder
diff --git a/examples/spark/src/test/java/org/apache/nemo/examples/spark/SparkScala.java b/examples/spark/src/test/java/org/apache/nemo/examples/spark/SparkScala.java
index 3dcf490..0ef2f15 100644
--- a/examples/spark/src/test/java/org/apache/nemo/examples/spark/SparkScala.java
+++ b/examples/spark/src/test/java/org/apache/nemo/examples/spark/SparkScala.java
@@ -20,6 +20,7 @@ package org.apache.nemo.examples.spark;
import org.apache.nemo.client.JobLauncher;
import org.apache.nemo.common.test.ArgBuilder;
+import org.apache.nemo.common.test.ExampleTestArgs;
import org.apache.nemo.common.test.ExampleTestUtil;
import org.apache.nemo.compiler.optimizer.policy.DefaultPolicy;
import org.junit.Before;
@@ -36,10 +37,8 @@ import org.powermock.modules.junit4.PowerMockRunner;
@PrepareForTest(JobLauncher.class)
@PowerMockIgnore("javax.management.*")
public final class SparkScala {
- private static final int TIMEOUT = 120000;
private static ArgBuilder builder;
- private static final String fileBasePath = System.getProperty("user.dir") + "/../resources/";
- private static final String executorResourceFileName = fileBasePath + "spark_test_executor_resources.json";
+ private static final String executorResourceFileName = ExampleTestArgs.getFileBasePath() + "executors/spark_test_executor_resources.json";
@Before
public void setUp() {
@@ -47,7 +46,7 @@ public final class SparkScala {
.addResourceJson(executorResourceFileName);
}
- @Test(timeout = TIMEOUT)
+ @Test(timeout = ExampleTestArgs.TIMEOUT)
public void testPi() throws Exception {
final String numParallelism = "3";
@@ -59,13 +58,13 @@ public final class SparkScala {
.build());
}
- @Test(timeout = TIMEOUT)
+ @Test(timeout = ExampleTestArgs.TIMEOUT)
public void testWordCount() throws Exception {
- final String inputFileName = "test_input_wordcount_spark";
- final String outputFileName = "test_output_wordcount_spark";
- final String expectedOutputFilename = "expected_output_wordcount_spark";
- final String inputFilePath = fileBasePath + inputFileName;
- final String outputFilePath = fileBasePath + outputFileName;
+ final String inputFileName = "inputs/test_input_wordcount_spark";
+ final String outputFileName = "inputs/test_output_wordcount_spark";
+ final String expectedOutputFilename = "/outputs/expected_output_wordcount_spark";
+ final String inputFilePath = ExampleTestArgs.getFileBasePath() + inputFileName;
+ final String outputFilePath = ExampleTestArgs.getFileBasePath() + outputFileName;
JobLauncher.main(builder
.addJobId(SparkWordCount.class.getSimpleName() + "_test")
@@ -75,22 +74,22 @@ public final class SparkScala {
.build());
try {
- ExampleTestUtil.ensureOutputValidity(fileBasePath, outputFileName, expectedOutputFilename);
+ ExampleTestUtil.ensureOutputValidity(ExampleTestArgs.getFileBasePath(), outputFileName, expectedOutputFilename);
} finally {
- ExampleTestUtil.deleteOutputFile(fileBasePath, outputFileName);
+ ExampleTestUtil.deleteOutputFile(ExampleTestArgs.getFileBasePath(), outputFileName);
}
}
- @Test(timeout = TIMEOUT)
+ @Test(timeout = ExampleTestArgs.TIMEOUT)
public void testCachingWordCount() throws Exception {
- final String inputFileName = "test_input_wordcount_spark";
+ final String inputFileName = "inputs/test_input_wordcount_spark";
final String outputFileName1 = "test_output_wordcount_spark";
final String outputFileName2 = "test_output_reversed_wordcount_spark";
- final String expectedOutputFilename1 = "expected_output_wordcount_spark";
- final String expectedOutputFilename2 = "expected_output_reversed_wordcount_spark";
- final String inputFilePath = fileBasePath + inputFileName;
- final String outputFilePath1 = fileBasePath + outputFileName1;
- final String outputFilePath2 = fileBasePath + outputFileName2;
+ final String expectedOutputFilename1 = "outputs/expected_output_wordcount_spark";
+ final String expectedOutputFilename2 = "outputs/expected_output_reversed_wordcount_spark";
+ final String inputFilePath = ExampleTestArgs.getFileBasePath() + inputFileName;
+ final String outputFilePath1 = ExampleTestArgs.getFileBasePath() + outputFileName1;
+ final String outputFilePath2 = ExampleTestArgs.getFileBasePath() + outputFileName2;
JobLauncher.main(builder
.addJobId(SparkCachingWordCount.class.getSimpleName() + "_test")
@@ -100,15 +99,15 @@ public final class SparkScala {
.build());
try {
- ExampleTestUtil.ensureOutputValidity(fileBasePath, outputFileName1, expectedOutputFilename1);
- ExampleTestUtil.ensureOutputValidity(fileBasePath, outputFileName2, expectedOutputFilename2);
+ ExampleTestUtil.ensureOutputValidity(ExampleTestArgs.getFileBasePath(), outputFileName1, expectedOutputFilename1);
+ ExampleTestUtil.ensureOutputValidity(ExampleTestArgs.getFileBasePath(), outputFileName2, expectedOutputFilename2);
} finally {
- ExampleTestUtil.deleteOutputFile(fileBasePath, outputFileName1);
- ExampleTestUtil.deleteOutputFile(fileBasePath, outputFileName2);
+ ExampleTestUtil.deleteOutputFile(ExampleTestArgs.getFileBasePath(), outputFileName1);
+ ExampleTestUtil.deleteOutputFile(ExampleTestArgs.getFileBasePath(), outputFileName2);
}
}
- @Test(timeout = TIMEOUT)
+ @Test(timeout = ExampleTestArgs.TIMEOUT)
public void testALS() throws Exception {
JobLauncher.main(builder
.addJobId(SparkALS.class.getSimpleName() + "_test")
@@ -118,7 +117,7 @@ public final class SparkScala {
.build());
}
- @Test(timeout = TIMEOUT)
+ @Test(timeout = ExampleTestArgs.TIMEOUT)
public void testALSEmptyUserArgs() throws Exception {
JobLauncher.main(builder
.addJobId(SparkALS.class.getSimpleName() + "_test")