You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@nemo.apache.org by GitBox <gi...@apache.org> on 2018/08/06 05:24:21 UTC
[GitHub] jeongyooneo closed pull request #84: [NEMO-171] Fix confusing
ITCase input/output file names
jeongyooneo closed pull request #84: [NEMO-171] Fix confusing ITCase input/output file names
URL: https://github.com/apache/incubator-nemo/pull/84
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/.gitignore b/.gitignore
index be82d4592..5def0cdfc 100644
--- a/.gitignore
+++ b/.gitignore
@@ -15,8 +15,8 @@ build
# ----------------------------------------------------------------------
# Files generated by OutputService during runtime
# ----------------------------------------------------------------------
-.sample_output*
-sample_output*
+.test_output*
+test_output*
runtime/dag/*
*.log
#
diff --git a/README.md b/README.md
index 3817f9fdb..76c875e5f 100644
--- a/README.md
+++ b/README.md
@@ -77,19 +77,19 @@ Please refer to the [Contribution guideline](.github/CONTRIBUTING.md) to contrib
## MapReduce example
./bin/run_beam.sh \
-job_id mr_default \
- -executor_json `pwd`/examples/resources/beam_sample_executor_resources.json \
+ -executor_json `pwd`/examples/resources/beam_test_executor_resources.json \
-optimization_policy edu.snu.nemo.compiler.optimizer.policy.DefaultPolicy \
-user_main edu.snu.nemo.examples.beam.WordCount \
- -user_args "`pwd`/examples/resources/sample_input_wordcount `pwd`/examples/resources/sample_output_wordcount"
+ -user_args "`pwd`/examples/resources/test_input_wordcount `pwd`/examples/resources/test_output_wordcount"
## YARN cluster example
./bin/run_beam.sh \
-deploy_mode yarn \
- -job_id mr_transient \
- -executor_json `pwd`/examples/resources/beam_sample_executor_resources.json \
- -user_main edu.snu.nemo.examples.beam.WordCount \
- -optimization_policy edu.snu.nemo.compiler.optimizer.policy.TransientResourcePolicy \
- -user_args "hdfs://v-m:9000/sample_input_wordcount hdfs://v-m:9000/sample_output_wordcount"
+ -job_id mr_transient \
+ -executor_json `pwd`/examples/resources/beam_test_executor_resources.json \
+ -user_main edu.snu.nemo.examples.beam.WordCount \
+ -optimization_policy edu.snu.nemo.compiler.optimizer.policy.TransientResourcePolicy \
+ -user_args "hdfs://v-m:9000/test_input_wordcount hdfs://v-m:9000/test_output_wordcount"
```
## Resource Configuration
`-executor_json` command line option can be used to provide a path to the JSON file that describes resource configuration for executors. Its default value is `config/default.json`, which initializes one of each `Transient`, `Reserved`, and `Compute` executor, each of which has one core and 1024MB memory.
@@ -133,11 +133,11 @@ Nemo Compiler and Engine can store JSON representation of intermediate DAGs.
```bash
./bin/run_beam.sh \
-job_id als \
- -executor_json `pwd`/examples/resources/beam_sample_executor_resources.json \
+ -executor_json `pwd`/examples/resources/beam_test_executor_resources.json \
-user_main edu.snu.nemo.examples.beam.AlternatingLeastSquare \
-optimization_policy edu.snu.nemo.compiler.optimizer.policy.TransientResourcePolicy \
-dag_dir "./dag/als" \
- -user_args "`pwd`/examples/resources/sample_input_als 10 3"
+ -user_args "`pwd`/examples/resources/test_input_als 10 3"
```
## Speeding up builds
diff --git a/common/src/test/java/edu/snu/nemo/common/ContextImplTest.java b/common/src/test/java/edu/snu/nemo/common/ContextImplTest.java
index b98489e3c..aaf71517a 100644
--- a/common/src/test/java/edu/snu/nemo/common/ContextImplTest.java
+++ b/common/src/test/java/edu/snu/nemo/common/ContextImplTest.java
@@ -46,7 +46,7 @@ public void testContextImpl() {
assertEquals(this.sideInputs, this.context.getSideInputs());
assertEquals(this.taggedOutputs, this.context.getAdditionalTagOutputs());
- final String sampleText = "sample_text";
+ final String sampleText = "test_text";
assertFalse(this.context.getSerializedData().isPresent());
diff --git a/compiler/test/src/main/java/edu/snu/nemo/compiler/CompilerTestUtil.java b/compiler/test/src/main/java/edu/snu/nemo/compiler/CompilerTestUtil.java
index ec5a77048..aece5f328 100644
--- a/compiler/test/src/main/java/edu/snu/nemo/compiler/CompilerTestUtil.java
+++ b/compiler/test/src/main/java/edu/snu/nemo/compiler/CompilerTestUtil.java
@@ -90,8 +90,8 @@ private static String findRoot(final String curDir) {
}
public static DAG<IRVertex, IREdge> compileWordCountDAG() throws Exception {
- final String input = ROOT_DIR + "/examples/resources/sample_input_wordcount";
- final String output = ROOT_DIR + "/examples/resources/sample_output";
+ final String input = ROOT_DIR + "/examples/resources/test_input_wordcount";
+ final String output = ROOT_DIR + "/examples/resources/test_output";
final String main = "edu.snu.nemo.examples.beam.WordCount";
final ArgBuilder mrArgBuilder = new ArgBuilder()
@@ -102,7 +102,7 @@ private static String findRoot(final String curDir) {
}
public static DAG<IRVertex, IREdge> compileALSDAG() throws Exception {
- final String input = ROOT_DIR + "/examples/resources/sample_input_als";
+ final String input = ROOT_DIR + "/examples/resources/test_input_als";
final String numFeatures = "10";
final String numIteration = "3";
final String main = "edu.snu.nemo.examples.beam.AlternatingLeastSquare";
@@ -115,7 +115,7 @@ private static String findRoot(final String curDir) {
}
public static DAG<IRVertex, IREdge> compileALSInefficientDAG() throws Exception {
- final String input = ROOT_DIR + "/examples/resources/sample_input_als";
+ final String input = ROOT_DIR + "/examples/resources/test_input_als";
final String numFeatures = "10";
final String numIteration = "3";
final String main = "edu.snu.nemo.examples.beam.AlternatingLeastSquareInefficient";
@@ -128,7 +128,7 @@ private static String findRoot(final String curDir) {
}
public static DAG<IRVertex, IREdge> compileMLRDAG() throws Exception {
- final String input = ROOT_DIR + "/examples/resources/sample_input_mlr";
+ final String input = ROOT_DIR + "/examples/resources/test_input_mlr";
final String numFeatures = "100";
final String numClasses = "5";
final String numIteration = "3";
diff --git a/conf/src/main/java/edu/snu/nemo/conf/JobConf.java b/conf/src/main/java/edu/snu/nemo/conf/JobConf.java
index 8d016aff8..871bf4411 100644
--- a/conf/src/main/java/edu/snu/nemo/conf/JobConf.java
+++ b/conf/src/main/java/edu/snu/nemo/conf/JobConf.java
@@ -155,7 +155,7 @@
* Path to the JSON file that specifies resource layout.
*/
@NamedParameter(doc = "Path to the JSON file that specifies resources for executors", short_name = "executor_json",
- default_value = "examples/resources/sample_executor_resources.json")
+ default_value = "examples/resources/test_executor_resources.json")
public final class ExecutorJSONPath implements Name<String> {
}
diff --git a/deploy/README.md b/deploy/README.md
index 46f2204cd..ca555a5f4 100644
--- a/deploy/README.md
+++ b/deploy/README.md
@@ -29,7 +29,7 @@
* git clone Nemo on v-m and install
* Upload a local input file to HDFS with `hdfs -put`
* Launch a Nemo job with `-deploy_mode yarn`, and hdfs paths as the input/output
-* Example: `./bin/run.sh -deploy_mode yarn -job_id mr -user_main edu.snu.nemo.examples.beam.WordCount -user_args "hdfs://v-m:9000/sample_input_mr hdfs://v-m:9000/sample_output_mr"`
+* Example: `./bin/run.sh -deploy_mode yarn -job_id mr -user_main edu.snu.nemo.examples.beam.WordCount -user_args "hdfs://v-m:9000/test_input_wordcount hdfs://v-m:9000/test_output_wordcount"`
## And you're all set.....?
* I hope so
diff --git a/examples/beam/src/test/java/edu/snu/nemo/examples/beam/AlternatingLeastSquareITCase.java b/examples/beam/src/test/java/edu/snu/nemo/examples/beam/AlternatingLeastSquareITCase.java
index 27331e9a4..1ad85aa9a 100644
--- a/examples/beam/src/test/java/edu/snu/nemo/examples/beam/AlternatingLeastSquareITCase.java
+++ b/examples/beam/src/test/java/edu/snu/nemo/examples/beam/AlternatingLeastSquareITCase.java
@@ -37,12 +37,12 @@
private static ArgBuilder builder;
private static final String fileBasePath = System.getProperty("user.dir") + "/../resources/";
- private static final String input = fileBasePath + "sample_input_als";
- private static final String outputFileName = "sample_output_als";
+ private static final String input = fileBasePath + "test_input_als";
+ private static final String outputFileName = "test_output_als";
private static final String output = fileBasePath + outputFileName;
- private static final String testResourceFileName = "test_output_als";
- private static final String noPoisonResources = fileBasePath + "beam_sample_executor_resources.json";
- private static final String poisonedResource = fileBasePath + "beam_sample_poisoned_executor_resources.json";
+ private static final String expectedOutputFileName = "expected_output_als";
+ private static final String noPoisonResources = fileBasePath + "beam_test_executor_resources.json";
+ private static final String poisonedResource = fileBasePath + "beam_test_poisoned_executor_resources.json";
private static final String numFeatures = "10";
private static final String numIteration = "3";
private static final String lambda = "0.05";
@@ -57,7 +57,7 @@ public void setUp() throws Exception {
@After
public void tearDown() throws Exception {
try {
- ExampleTestUtil.ensureALSOutputValidity(fileBasePath, outputFileName, testResourceFileName);
+ ExampleTestUtil.ensureALSOutputValidity(fileBasePath, outputFileName, expectedOutputFileName);
} finally {
ExampleTestUtil.deleteOutputFile(fileBasePath, outputFileName);
}
diff --git a/examples/beam/src/test/java/edu/snu/nemo/examples/beam/BroadcastITCase.java b/examples/beam/src/test/java/edu/snu/nemo/examples/beam/BroadcastITCase.java
index d813640b9..2ddfba44e 100644
--- a/examples/beam/src/test/java/edu/snu/nemo/examples/beam/BroadcastITCase.java
+++ b/examples/beam/src/test/java/edu/snu/nemo/examples/beam/BroadcastITCase.java
@@ -37,10 +37,10 @@
private static ArgBuilder builder;
private static final String fileBasePath = System.getProperty("user.dir") + "/../resources/";
- private static final String inputFileName = "sample_input_wordcount";
- private static final String outputFileName = "sample_output_broadcast";
- private static final String testResourceFileName = "test_output_broadcast";
- private static final String executorResourceFileName = fileBasePath + "beam_sample_executor_resources.json";
+ private static final String inputFileName = "test_input_wordcount";
+ private static final String outputFileName = "test_output_broadcast";
+ private static final String expectedOutputFileName = "expected_output_broadcast";
+ private static final String executorResourceFileName = fileBasePath + "beam_test_executor_resources.json";
private static final String inputFilePath = fileBasePath + inputFileName;
private static final String outputFilePath = fileBasePath + outputFileName;
@@ -55,7 +55,7 @@ public void setUp() throws Exception {
@After
public void tearDown() throws Exception {
try {
- ExampleTestUtil.ensureOutputValidity(fileBasePath, outputFileName, testResourceFileName);
+ ExampleTestUtil.ensureOutputValidity(fileBasePath, outputFileName, expectedOutputFileName);
} finally {
ExampleTestUtil.deleteOutputFile(fileBasePath, outputFileName);
}
diff --git a/examples/beam/src/test/java/edu/snu/nemo/examples/beam/MultinomialLogisticRegressionITCase.java b/examples/beam/src/test/java/edu/snu/nemo/examples/beam/MultinomialLogisticRegressionITCase.java
index a11f2fc25..c81f2418d 100644
--- a/examples/beam/src/test/java/edu/snu/nemo/examples/beam/MultinomialLogisticRegressionITCase.java
+++ b/examples/beam/src/test/java/edu/snu/nemo/examples/beam/MultinomialLogisticRegressionITCase.java
@@ -33,7 +33,7 @@
private static final int TIMEOUT = 240000;
private static ArgBuilder builder = new ArgBuilder();
private static final String fileBasePath = System.getProperty("user.dir") + "/../resources/";
- private static final String executorResourceFileName = fileBasePath + "beam_sample_executor_resources.json";
+ private static final String executorResourceFileName = fileBasePath + "beam_test_executor_resources.json";
@Before
public void setUp() throws Exception {
@@ -42,7 +42,7 @@ public void setUp() throws Exception {
@Test (timeout = TIMEOUT)
public void test() throws Exception {
- final String input = fileBasePath + "sample_input_mlr";
+ final String input = fileBasePath + "test_input_mlr";
final String numFeatures = "100";
final String numClasses = "5";
final String numIteration = "3";
diff --git a/examples/beam/src/test/java/edu/snu/nemo/examples/beam/NetworkTraceAnalysisITCase.java b/examples/beam/src/test/java/edu/snu/nemo/examples/beam/NetworkTraceAnalysisITCase.java
index d733f66e7..07bc1cde5 100644
--- a/examples/beam/src/test/java/edu/snu/nemo/examples/beam/NetworkTraceAnalysisITCase.java
+++ b/examples/beam/src/test/java/edu/snu/nemo/examples/beam/NetworkTraceAnalysisITCase.java
@@ -35,11 +35,11 @@
private static ArgBuilder builder;
private static final String fileBasePath = System.getProperty("user.dir") + "/../resources/";
- private static final String inputFileName0 = "sample_input_network0";
- private static final String inputFileName1 = "sample_input_network1";
- private static final String outputFileName = "sample_output_network";
- private static final String testResourceFileName = "test_output_network";
- private static final String executorResourceFileName = fileBasePath + "beam_sample_executor_resources.json";
+ private static final String inputFileName0 = "test_input_network0";
+ private static final String inputFileName1 = "test_input_network1";
+ private static final String outputFileName = "test_output_network";
+ private static final String expectedOutputFileName = "expected_output_network";
+ private static final String executorResourceFileName = fileBasePath + "beam_test_executor_resources.json";
private static final String inputFilePath0 = fileBasePath + inputFileName0;
private static final String inputFilePath1 = fileBasePath + inputFileName1;
private static final String outputFilePath = fileBasePath + outputFileName;
@@ -55,7 +55,7 @@ public void setUp() throws Exception {
@After
public void tearDown() throws Exception {
try {
- ExampleTestUtil.ensureOutputValidity(fileBasePath, outputFileName, testResourceFileName);
+ ExampleTestUtil.ensureOutputValidity(fileBasePath, outputFileName, expectedOutputFileName);
} finally {
ExampleTestUtil.deleteOutputFile(fileBasePath, outputFileName);
}
diff --git a/examples/beam/src/test/java/edu/snu/nemo/examples/beam/PartitionWordsByLengthITCase.java b/examples/beam/src/test/java/edu/snu/nemo/examples/beam/PartitionWordsByLengthITCase.java
index 31c40ef90..96c83a74a 100644
--- a/examples/beam/src/test/java/edu/snu/nemo/examples/beam/PartitionWordsByLengthITCase.java
+++ b/examples/beam/src/test/java/edu/snu/nemo/examples/beam/PartitionWordsByLengthITCase.java
@@ -36,10 +36,10 @@
private static ArgBuilder builder;
private static final String fileBasePath = System.getProperty("user.dir") + "/../resources/";
- private static final String inputFileName = "sample_input_tag";
- private static final String outputFileName = "sample_output_tag";
- private static final String testResourceFileName = "test_output_tag";
- private static final String executorResourceFileName = fileBasePath + "beam_sample_executor_resources.json";
+ private static final String inputFileName = "test_input_tag";
+ private static final String outputFileName = "test_output_tag";
+ private static final String expectedOutputFileName = "expected_output_tag";
+ private static final String executorResourceFileName = fileBasePath + "beam_test_executor_resources.json";
private static final String inputFilePath = fileBasePath + inputFileName;
private static final String outputFilePath = fileBasePath + outputFileName;
@@ -54,9 +54,9 @@ public void setUp() throws Exception {
@After
public void tearDown() throws Exception {
try {
- ExampleTestUtil.ensureOutputValidity(fileBasePath, outputFileName + "_short", testResourceFileName + "_short");
- ExampleTestUtil.ensureOutputValidity(fileBasePath, outputFileName + "_long", testResourceFileName + "_long");
- ExampleTestUtil.ensureOutputValidity(fileBasePath, outputFileName + "_very_long", testResourceFileName + "_very_long");
+ ExampleTestUtil.ensureOutputValidity(fileBasePath, outputFileName + "_short", expectedOutputFileName + "_short");
+ ExampleTestUtil.ensureOutputValidity(fileBasePath, outputFileName + "_long", expectedOutputFileName + "_long");
+ ExampleTestUtil.ensureOutputValidity(fileBasePath, outputFileName + "_very_long", expectedOutputFileName + "_very_long");
} finally {
ExampleTestUtil.deleteOutputFile(fileBasePath, outputFileName);
}
diff --git a/examples/beam/src/test/java/edu/snu/nemo/examples/beam/PerKeyMedianITCase.java b/examples/beam/src/test/java/edu/snu/nemo/examples/beam/PerKeyMedianITCase.java
index ddfc05f6a..adf281147 100644
--- a/examples/beam/src/test/java/edu/snu/nemo/examples/beam/PerKeyMedianITCase.java
+++ b/examples/beam/src/test/java/edu/snu/nemo/examples/beam/PerKeyMedianITCase.java
@@ -36,10 +36,10 @@
private static ArgBuilder builder;
private static final String fileBasePath = System.getProperty("user.dir") + "/../resources/";
- private static final String inputFileName = "sample_input_median";
- private static final String outputFileName = "sample_output_median";
- private static final String testResourceFileName = "test_output_median";
- private static final String executorResourceFileName = fileBasePath + "beam_sample_executor_resources.json";
+ private static final String inputFileName = "test_input_median";
+ private static final String outputFileName = "test_output_median";
+ private static final String expectedOutputFileName = "expected_output_median";
+ private static final String executorResourceFileName = fileBasePath + "beam_test_executor_resources.json";
private static final String inputFilePath = fileBasePath + inputFileName;
private static final String outputFilePath = fileBasePath + outputFileName;
@@ -54,7 +54,7 @@ public void setUp() throws Exception {
@After
public void tearDown() throws Exception {
try {
- ExampleTestUtil.ensureOutputValidity(fileBasePath, outputFileName, testResourceFileName);
+ ExampleTestUtil.ensureOutputValidity(fileBasePath, outputFileName, expectedOutputFileName);
} finally {
ExampleTestUtil.deleteOutputFile(fileBasePath, outputFileName);
}
diff --git a/examples/beam/src/test/java/edu/snu/nemo/examples/beam/WordCountITCase.java b/examples/beam/src/test/java/edu/snu/nemo/examples/beam/WordCountITCase.java
index 6527e26e4..36e30e06a 100644
--- a/examples/beam/src/test/java/edu/snu/nemo/examples/beam/WordCountITCase.java
+++ b/examples/beam/src/test/java/edu/snu/nemo/examples/beam/WordCountITCase.java
@@ -36,11 +36,11 @@
private static ArgBuilder builder;
private static final String fileBasePath = System.getProperty("user.dir") + "/../resources/";
- private static final String inputFileName = "sample_input_wordcount";
- private static final String outputFileName = "sample_output_wordcount";
- private static final String testResourceFileName = "test_output_wordcount";
- private static final String executorResourceFileName = fileBasePath + "beam_sample_executor_resources.json";
- private static final String oneExecutorResourceFileName = fileBasePath + "beam_sample_one_executor_resources.json";
+ private static final String inputFileName = "test_input_wordcount";
+ private static final String outputFileName = "test_output_wordcount";
+ private static final String expectedOutputFileName = "expected_output_wordcount";
+ private static final String executorResourceFileName = fileBasePath + "beam_test_executor_resources.json";
+ private static final String oneExecutorResourceFileName = fileBasePath + "beam_test_one_executor_resources.json";
private static final String inputFilePath = fileBasePath + inputFileName;
private static final String outputFilePath = fileBasePath + outputFileName;
@@ -54,7 +54,7 @@ public void setUp() throws Exception {
@After
public void tearDown() throws Exception {
try {
- ExampleTestUtil.ensureOutputValidity(fileBasePath, outputFileName, testResourceFileName);
+ ExampleTestUtil.ensureOutputValidity(fileBasePath, outputFileName, expectedOutputFileName);
} finally {
ExampleTestUtil.deleteOutputFile(fileBasePath, outputFileName);
}
diff --git a/examples/resources/beam_sample_executor_resources.json b/examples/resources/beam_test_executor_resources.json
similarity index 100%
rename from examples/resources/beam_sample_executor_resources.json
rename to examples/resources/beam_test_executor_resources.json
diff --git a/examples/resources/beam_sample_one_executor_resources.json b/examples/resources/beam_test_one_executor_resources.json
similarity index 100%
rename from examples/resources/beam_sample_one_executor_resources.json
rename to examples/resources/beam_test_one_executor_resources.json
diff --git a/examples/resources/beam_sample_poisoned_executor_resources.json b/examples/resources/beam_test_poisoned_executor_resources.json
similarity index 100%
rename from examples/resources/beam_sample_poisoned_executor_resources.json
rename to examples/resources/beam_test_poisoned_executor_resources.json
diff --git a/examples/resources/test_output_als b/examples/resources/expected_output_als
similarity index 100%
rename from examples/resources/test_output_als
rename to examples/resources/expected_output_als
diff --git a/examples/resources/test_output_broadcast b/examples/resources/expected_output_broadcast
similarity index 100%
rename from examples/resources/test_output_broadcast
rename to examples/resources/expected_output_broadcast
diff --git a/examples/resources/test_output_median b/examples/resources/expected_output_median
similarity index 100%
rename from examples/resources/test_output_median
rename to examples/resources/expected_output_median
diff --git a/examples/resources/test_output_network b/examples/resources/expected_output_network
similarity index 100%
rename from examples/resources/test_output_network
rename to examples/resources/expected_output_network
diff --git a/examples/resources/test_output_tag_long b/examples/resources/expected_output_tag_long
similarity index 100%
rename from examples/resources/test_output_tag_long
rename to examples/resources/expected_output_tag_long
diff --git a/examples/resources/test_output_tag_short b/examples/resources/expected_output_tag_short
similarity index 100%
rename from examples/resources/test_output_tag_short
rename to examples/resources/expected_output_tag_short
diff --git a/examples/resources/test_output_tag_very_long b/examples/resources/expected_output_tag_very_long
similarity index 100%
rename from examples/resources/test_output_tag_very_long
rename to examples/resources/expected_output_tag_very_long
diff --git a/examples/resources/test_output_word_and_line_count b/examples/resources/expected_output_word_and_line_count
similarity index 100%
rename from examples/resources/test_output_word_and_line_count
rename to examples/resources/expected_output_word_and_line_count
diff --git a/examples/resources/test_output_wordcount b/examples/resources/expected_output_wordcount
similarity index 100%
rename from examples/resources/test_output_wordcount
rename to examples/resources/expected_output_wordcount
diff --git a/examples/resources/test_output_wordcount_spark b/examples/resources/expected_output_wordcount_spark
similarity index 100%
rename from examples/resources/test_output_wordcount_spark
rename to examples/resources/expected_output_wordcount_spark
diff --git a/examples/resources/spark_sample_executor_resources.json b/examples/resources/spark_test_executor_resources.json
similarity index 100%
rename from examples/resources/spark_sample_executor_resources.json
rename to examples/resources/spark_test_executor_resources.json
diff --git a/examples/resources/sample_input_als b/examples/resources/test_input_als
similarity index 100%
rename from examples/resources/sample_input_als
rename to examples/resources/test_input_als
diff --git a/examples/resources/sample_input_employees.json b/examples/resources/test_input_employees.json
similarity index 100%
rename from examples/resources/sample_input_employees.json
rename to examples/resources/test_input_employees.json
diff --git a/examples/resources/sample_input_median b/examples/resources/test_input_median
similarity index 100%
rename from examples/resources/sample_input_median
rename to examples/resources/test_input_median
diff --git a/examples/resources/sample_input_mlr b/examples/resources/test_input_mlr
similarity index 100%
rename from examples/resources/sample_input_mlr
rename to examples/resources/test_input_mlr
diff --git a/examples/resources/sample_input_network0 b/examples/resources/test_input_network0
similarity index 100%
rename from examples/resources/sample_input_network0
rename to examples/resources/test_input_network0
diff --git a/examples/resources/sample_input_network1 b/examples/resources/test_input_network1
similarity index 100%
rename from examples/resources/sample_input_network1
rename to examples/resources/test_input_network1
diff --git a/examples/resources/sample_input_people.json b/examples/resources/test_input_people.json
similarity index 100%
rename from examples/resources/sample_input_people.json
rename to examples/resources/test_input_people.json
diff --git a/examples/resources/sample_input_people.txt b/examples/resources/test_input_people.txt
similarity index 100%
rename from examples/resources/sample_input_people.txt
rename to examples/resources/test_input_people.txt
diff --git a/examples/resources/sample_input_tag b/examples/resources/test_input_tag
similarity index 100%
rename from examples/resources/sample_input_tag
rename to examples/resources/test_input_tag
diff --git a/examples/resources/sample_input_wordcount b/examples/resources/test_input_wordcount
similarity index 100%
rename from examples/resources/sample_input_wordcount
rename to examples/resources/test_input_wordcount
diff --git a/examples/resources/sample_input_wordcount_spark b/examples/resources/test_input_wordcount_spark
similarity index 100%
rename from examples/resources/sample_input_wordcount_spark
rename to examples/resources/test_input_wordcount_spark
diff --git a/examples/spark/src/test/java/edu/snu/nemo/examples/spark/MRJava.java b/examples/spark/src/test/java/edu/snu/nemo/examples/spark/MRJava.java
index 96ec20c56..e49dd0c47 100644
--- a/examples/spark/src/test/java/edu/snu/nemo/examples/spark/MRJava.java
+++ b/examples/spark/src/test/java/edu/snu/nemo/examples/spark/MRJava.java
@@ -37,7 +37,7 @@
private static final int TIMEOUT = 180000;
private static ArgBuilder builder;
private static final String fileBasePath = System.getProperty("user.dir") + "/../resources/";
- private static final String executorResourceFileName = fileBasePath + "spark_sample_executor_resources.json";
+ private static final String executorResourceFileName = fileBasePath + "spark_test_executor_resources.json";
@Before
public void setUp() {
@@ -47,9 +47,9 @@ public void setUp() {
@Test(timeout = TIMEOUT)
public void testSparkWordCount() throws Exception {
- final String inputFileName = "sample_input_wordcount_spark";
- final String outputFileName = "sample_output_wordcount_spark";
- final String testResourceFilename = "test_output_wordcount_spark";
+ final String inputFileName = "test_input_wordcount_spark";
+ final String outputFileName = "test_output_wordcount_spark";
+ final String expectedOutputFilename = "expected_output_wordcount_spark";
final String inputFilePath = fileBasePath + inputFileName;
final String outputFilePath = fileBasePath + outputFileName;
@@ -61,56 +61,11 @@ public void testSparkWordCount() throws Exception {
.build());
try {
- ExampleTestUtil.ensureOutputValidity(fileBasePath, outputFileName, testResourceFilename);
- } finally {
- ExampleTestUtil.deleteOutputFile(fileBasePath, outputFileName);
- }
- }
- /* TODO #152: enable execution of multiple jobs (call scheduleJob multiple times with caching).
- @Test(timeout = TIMEOUT)
- public void testSparkWordAndLineCount() throws Exception {
- final String inputFileName = "sample_input_wordcount_spark";
- final String outputFileName = "sample_output_word_and_line_count";
- final String testResourceFilename = "test_output_word_and_line_count";
- final String inputFilePath = fileBasePath + inputFileName;
- final String outputFilePath = fileBasePath + outputFileName;
-
- JobLauncher.main(builder
- .addJobId(JavaWordAndLineCount.class.getSimpleName() + "_test")
- .addUserMain(JavaWordAndLineCount.class.getCanonicalName())
- .addUserArgs(inputFilePath, outputFilePath)
- .addOptimizationPolicy(DefaultPolicy.class.getCanonicalName())
- .build());
-
- try {
- ExampleTestUtil.ensureOutputValidity(fileBasePath, outputFileName, testResourceFilename);
+ ExampleTestUtil.ensureOutputValidity(fileBasePath, outputFileName, expectedOutputFilename);
} finally {
ExampleTestUtil.deleteOutputFile(fileBasePath, outputFileName);
}
}
- /* Temporary disabled due to Travis issue
- @Test(timeout = TIMEOUT)
- public void testSparkMapReduce() throws Exception {
- final String inputFileName = "sample_input_wordcount_spark";
- final String outputFileName = "sample_output_mr";
- final String testResourceFilename = "test_output_wordcount_spark";
- final String inputFilePath = fileBasePath + inputFileName;
- final String outputFilePath = fileBasePath + outputFileName;
- final String parallelism = "2";
- final String runOnYarn = "false";
-
- JobLauncher.main(builder
- .addJobId(JavaMapReduce.class.getSimpleName() + "_test")
- .addUserMain(JavaMapReduce.class.getCanonicalName())
- .addUserArgs(inputFilePath, outputFilePath, parallelism, runOnYarn)
- .addOptimizationPolicy(DefaultPolicy.class.getCanonicalName())
- .build());
-
- try {
- ExampleTestUtil.ensureOutputValidity(fileBasePath, outputFileName, testResourceFilename);
- } finally {
- ExampleTestUtil.deleteOutputFile(fileBasePath, outputFileName);
- }
- }*/
+ // TODO #152: enable execution of multiple jobs (call scheduleJob multiple times with caching).
}
diff --git a/examples/spark/src/test/java/edu/snu/nemo/examples/spark/SparkJava.java b/examples/spark/src/test/java/edu/snu/nemo/examples/spark/SparkJava.java
index 132e5170f..d68022daf 100644
--- a/examples/spark/src/test/java/edu/snu/nemo/examples/spark/SparkJava.java
+++ b/examples/spark/src/test/java/edu/snu/nemo/examples/spark/SparkJava.java
@@ -38,7 +38,7 @@
private static final int TIMEOUT = 180000;
private static ArgBuilder builder;
private static final String fileBasePath = System.getProperty("user.dir") + "/../resources/";
- private static final String executorResourceFileName = fileBasePath + "spark_sample_executor_resources.json";
+ private static final String executorResourceFileName = fileBasePath + "spark_test_executor_resources.json";
@Before
public void setUp() {
@@ -60,7 +60,7 @@ public void testSparkPi() throws Exception {
@Test(timeout = TIMEOUT)
public void testSparkSQLUserDefinedTypedAggregation() throws Exception {
- final String inputFileName = "sample_input_employees.json";
+ final String inputFileName = "test_input_employees.json";
final String inputFilePath = fileBasePath + inputFileName;
JobLauncher.main(builder
@@ -73,7 +73,7 @@ public void testSparkSQLUserDefinedTypedAggregation() throws Exception {
@Test(timeout = TIMEOUT)
public void testSparkSQLUserDefinedUntypedAggregation() throws Exception {
- final String inputFileName = "sample_input_employees.json";
+ final String inputFileName = "test_input_employees.json";
final String inputFilePath = fileBasePath + inputFileName;
JobLauncher.main(builder
@@ -86,8 +86,8 @@ public void testSparkSQLUserDefinedUntypedAggregation() throws Exception {
@Test(timeout = TIMEOUT)
public void testSparkSQLExample() throws Exception {
- final String peopleJson = "sample_input_people.json";
- final String peopleTxt = "sample_input_people.txt";
+ final String peopleJson = "test_input_people.json";
+ final String peopleTxt = "test_input_people.txt";
final String inputFileJson = fileBasePath + peopleJson;
final String inputFileTxt = fileBasePath + peopleTxt;
diff --git a/examples/spark/src/test/java/edu/snu/nemo/examples/spark/SparkScala.java b/examples/spark/src/test/java/edu/snu/nemo/examples/spark/SparkScala.java
index 2aaae7c2e..3e1e8e9f0 100644
--- a/examples/spark/src/test/java/edu/snu/nemo/examples/spark/SparkScala.java
+++ b/examples/spark/src/test/java/edu/snu/nemo/examples/spark/SparkScala.java
@@ -36,7 +36,7 @@
private static final int TIMEOUT = 120000;
private static ArgBuilder builder;
private static final String fileBasePath = System.getProperty("user.dir") + "/../resources/";
- private static final String executorResourceFileName = fileBasePath + "spark_sample_executor_resources.json";
+ private static final String executorResourceFileName = fileBasePath + "spark_test_executor_resources.json";
@Before
public void setUp() {
@@ -58,9 +58,9 @@ public void testPi() throws Exception {
@Test(timeout = TIMEOUT)
public void testWordCount() throws Exception {
- final String inputFileName = "sample_input_wordcount_spark";
- final String outputFileName = "sample_output_wordcount_spark";
- final String testResourceFilename = "test_output_wordcount_spark";
+ final String inputFileName = "test_input_wordcount_spark";
+ final String outputFileName = "test_output_wordcount_spark";
+ final String expectedOutputFilename = "expected_output_wordcount_spark";
final String inputFilePath = fileBasePath + inputFileName;
final String outputFilePath = fileBasePath + outputFileName;
@@ -72,7 +72,7 @@ public void testWordCount() throws Exception {
.build());
try {
- ExampleTestUtil.ensureOutputValidity(fileBasePath, outputFileName, testResourceFilename);
+ ExampleTestUtil.ensureOutputValidity(fileBasePath, outputFileName, expectedOutputFilename);
} finally {
ExampleTestUtil.deleteOutputFile(fileBasePath, outputFileName);
}
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services