You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by an...@apache.org on 2018/05/28 11:38:41 UTC
oozie git commit: OOZIE-3228 amend [Spark action] Can't load
properties from spark-defaults.conf (andras.piros)
Repository: oozie
Updated Branches:
refs/heads/master ad660cf64 -> bf5bd91b7
OOZIE-3228 amend [Spark action] Can't load properties from spark-defaults.conf (andras.piros)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/bf5bd91b
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/bf5bd91b
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/bf5bd91b
Branch: refs/heads/master
Commit: bf5bd91b76a4d0f61fa995c5b6cb062856bd3478
Parents: ad660cf
Author: Andras Piros <an...@cloudera.com>
Authored: Mon May 28 13:38:13 2018 +0200
Committer: Andras Piros <an...@cloudera.com>
Committed: Mon May 28 13:38:13 2018 +0200
----------------------------------------------------------------------
release-log.txt | 1 +
.../action/hadoop/TestSparkArgsExtractor.java | 54 ++++++++++++--------
2 files changed, 33 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/bf5bd91b/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index cc36879..211eb2f 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 5.1.0 release (trunk - unreleased)
+OOZIE-3228 amend [Spark action] Can't load properties from spark-defaults.conf (andras.piros)
OOZIE-3257 TestHiveActionExecutor#testHiveAction still fails (pbacsko via andras.piros)
OOZIE-2352 Unportable shebang in shell scripts (dbist13 via andras.piros)
OOZIE-2211 Remove OozieCLI#validateCommandV41 (dbist13 via andras.piros)
http://git-wip-us.apache.org/repos/asf/oozie/blob/bf5bd91b/sharelib/spark/src/test/java/org/apache/oozie/action/hadoop/TestSparkArgsExtractor.java
----------------------------------------------------------------------
diff --git a/sharelib/spark/src/test/java/org/apache/oozie/action/hadoop/TestSparkArgsExtractor.java b/sharelib/spark/src/test/java/org/apache/oozie/action/hadoop/TestSparkArgsExtractor.java
index d75e727..60ab8b9 100644
--- a/sharelib/spark/src/test/java/org/apache/oozie/action/hadoop/TestSparkArgsExtractor.java
+++ b/sharelib/spark/src/test/java/org/apache/oozie/action/hadoop/TestSparkArgsExtractor.java
@@ -29,11 +29,14 @@ import java.io.FileWriter;
import java.io.IOException;
import java.net.URISyntaxException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
import java.util.Properties;
import static org.apache.oozie.action.hadoop.SparkArgsExtractor.SPARK_DEFAULTS_GENERATED_PROPERTIES;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
public class TestSparkArgsExtractor {
@@ -318,37 +321,44 @@ public class TestSparkArgsExtractor {
actionConf.set(SparkActionExecutor.SPARK_JOB_NAME, "Spark Copy File");
actionConf.set(SparkActionExecutor.SPARK_JAR, "/lib/test.jar");
- createTemporaryFileWithContent(SPARK_DEFAULTS_PROPERTIES, "spark.executor.extraClassPath=/etc/hbase/conf:/etc/hive/conf\n" +
+ createTemporaryFileWithContent(SPARK_DEFAULTS_PROPERTIES,
+ "spark.executor.extraClassPath=/etc/hbase/conf:/etc/hive/conf\n" +
"spark.driver.extraClassPath=/etc/hbase/conf:/etc/hive/conf\n" +
"spark.executor.extraJavaOptions=-XX:+UseG1GC -XX:+PrintGC -XX:+UnlockExperimentalVMOptions\n" +
"spark.driver.extraJavaOptions=-XX:+UseG1GC -XX:+PrintGC -XX:+UnlockExperimentalVMOptions");
final List<String> sparkArgs = new SparkArgsExtractor(actionConf).extract(new String[0]);
- assertEquals("Spark args mismatch",
- Lists.newArrayList("--master", "yarn",
- "--deploy-mode", "client",
- "--name", "Spark Copy File",
- "--class", "org.apache.oozie.example.SparkFileCopy",
- "--conf", "spark.executor.extraClassPath=/etc/hbase/conf:/etc/hive/conf:$PWD/*",
- "--conf", "spark.driver.extraClassPath=/etc/hbase/conf:/etc/hive/conf:$PWD/*",
- "--conf", "spark.yarn.security.tokens.hadoopfs.enabled=false",
- "--conf", "spark.yarn.security.tokens.hive.enabled=false",
- "--conf", "spark.yarn.security.tokens.hbase.enabled=false",
- "--conf", "spark.yarn.security.credentials.hadoopfs.enabled=false",
- "--conf", "spark.yarn.security.credentials.hive.enabled=false",
- "--conf", "spark.yarn.security.credentials.hbase.enabled=false",
- "--conf", "spark.executor.extraJavaOptions=-XX:+UseG1GC -XX:+PrintGC -XX:+UnlockExperimentalVMOptions " +
- "-Dlog4j.configuration=spark-log4j.properties",
- "--conf", "spark.driver.extraJavaOptions=-XX:+UseG1GC -XX:+PrintGC -XX:+UnlockExperimentalVMOptions " +
- "-Dlog4j.configuration=spark-log4j.properties",
- "--properties-file", "spark-defaults-oozie-generated.properties",
- "--files", "spark-log4j.properties,hive-site.xml",
- "--conf", "spark.yarn.jar=null",
- "--verbose", "/lib/test.jar"),
+ assertContainsSublist(
+ Lists.newArrayList("--conf", "spark.executor.extraClassPath=/etc/hbase/conf:/etc/hive/conf:$PWD/*"),
+ sparkArgs);
+ assertContainsSublist(
+ Lists.newArrayList("--conf", "spark.driver.extraClassPath=/etc/hbase/conf:/etc/hive/conf:$PWD/*"),
+ sparkArgs);
+ assertContainsSublist(
+ Lists.newArrayList("--conf", "spark.executor.extraJavaOptions=-XX:+UseG1GC -XX:+PrintGC " +
+ "-XX:+UnlockExperimentalVMOptions -Dlog4j.configuration=spark-log4j.properties"),
+ sparkArgs);
+ assertContainsSublist(
+ Lists.newArrayList("--conf", "spark.driver.extraJavaOptions=-XX:+UseG1GC -XX:+PrintGC " +
+ "-XX:+UnlockExperimentalVMOptions -Dlog4j.configuration=spark-log4j.properties"),
sparkArgs);
}
+ private void assertContainsSublist(final List<String> expected, final List<String> actual) {
+ final int sublistSize = expected.size();
+ assertTrue("actual size is below expected size", actual.size() >= sublistSize);
+
+ for (int ixActual = 0; ixActual <= actual.size() - sublistSize; ixActual++) {
+ final List<String> actualSublist = actual.subList(ixActual, ixActual + sublistSize);
+ if (Arrays.deepEquals(expected.toArray(), actualSublist.toArray())) {
+ return;
+ }
+ }
+
+ fail(String.format("actual:\n%s does not contain expected:\n%s", actual, expected));
+ }
+
private Properties readMergedProperties() throws IOException {
final File file = new File(SPARK_DEFAULTS_GENERATED_PROPERTIES);
file.deleteOnExit();