You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by rk...@apache.org on 2017/02/15 00:41:13 UTC
oozie git commit: OOZIE-2792 Hive2 action is not parsing Spark
application ID from log file properly when Hive is on Spark (zhengxb2005 via
rkanter)
Repository: oozie
Updated Branches:
refs/heads/master 0da31f47e -> 4483f6dd9
OOZIE-2792 Hive2 action is not parsing Spark application ID from log file properly when Hive is on Spark (zhengxb2005 via rkanter)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/4483f6dd
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/4483f6dd
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/4483f6dd
Branch: refs/heads/master
Commit: 4483f6dd94daa0aee1bdaf19cbb4b4a68cbefa8c
Parents: 0da31f4
Author: Robert Kanter <rk...@apache.org>
Authored: Tue Feb 14 16:40:46 2017 -0800
Committer: Robert Kanter <rk...@apache.org>
Committed: Tue Feb 14 16:40:46 2017 -0800
----------------------------------------------------------------------
.../oozie/action/hadoop/TestDistcpMain.java | 26 +++++++++-
release-log.txt | 1 +
.../apache/oozie/action/hadoop/DistcpMain.java | 6 ++-
.../apache/oozie/action/hadoop/HiveMain.java | 5 +-
.../oozie/action/hadoop/TestHiveMain.java | 35 +++++++++++--
.../apache/oozie/action/hadoop/Hive2Main.java | 8 ++-
.../oozie/action/hadoop/TestHive2Main.java | 52 ++++++++++++++++++++
.../oozie/action/hadoop/LauncherMain.java | 32 +++++++-----
.../org/apache/oozie/action/hadoop/PigMain.java | 6 ++-
.../apache/oozie/action/hadoop/TestPigMain.java | 23 +++++++++
.../apache/oozie/action/hadoop/SparkMain.java | 3 +-
.../oozie/action/hadoop/TestSparkMain.java | 18 +++++++
.../apache/oozie/action/hadoop/SqoopMain.java | 6 ++-
.../oozie/action/hadoop/TestSqoopMain.java | 49 ++++++++++++++++++
14 files changed, 247 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/4483f6dd/core/src/test/java/org/apache/oozie/action/hadoop/TestDistcpMain.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestDistcpMain.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestDistcpMain.java
index bf08ccb..199962e 100644
--- a/core/src/test/java/org/apache/oozie/action/hadoop/TestDistcpMain.java
+++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestDistcpMain.java
@@ -23,10 +23,13 @@ import org.apache.oozie.util.XConfiguration;
import java.io.File;
import java.io.FileOutputStream;
-import java.io.IOException;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Writer;
+import java.util.ArrayList;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
import org.apache.oozie.action.hadoop.DistcpMain;
@@ -95,4 +98,25 @@ public class TestDistcpMain extends MainTestCase {
new File(getTestCaseDir(), "action.xml").delete();
return null;
}
+
+ public void testJobIDPattern() {
+ List<String> lines = new ArrayList<String>();
+ lines.add("Job complete: job_001");
+ lines.add("Job job_002 completed successfully");
+ lines.add("Submitted application application_003");
+ // Non-matching ones
+ lines.add("Job complete: job004");
+ lines.add("Job complete: (job_005");
+ lines.add("Job abc job_006 completed successfully");
+ lines.add("Submitted application. application_007");
+ Set<String> jobIds = new LinkedHashSet<String>();
+ for (String line : lines) {
+ LauncherMain.extractJobIDs(line, DistcpMain.DISTCP_JOB_IDS_PATTERNS, jobIds);
+ }
+ Set<String> expected = new LinkedHashSet<String>();
+ expected.add("job_001");
+ expected.add("job_002");
+ expected.add("job_003");
+ assertEquals(expected, jobIds);
+ }
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/4483f6dd/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 32e3648..4562524 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.4.0 release (trunk - unreleased)
+OOZIE-2792 Hive2 action is not parsing Spark application ID from log file properly when Hive is on Spark (zhengxb2005 via rkanter)
OOZIE-2788 Fix jobs API servlet mapping for EmbeddedOozieServer (abhishekbafna via rkanter)
OOZIE-2778 Copy only jetty.version related server dependencies to distro (asasvari via abhishekbafna)
OOZIE-2786 Pass Oozie workflow ID and settings to Spark application configuration (zhengxb2005 via rkanter)
http://git-wip-us.apache.org/repos/asf/oozie/blob/4483f6dd/sharelib/distcp/src/main/java/org/apache/oozie/action/hadoop/DistcpMain.java
----------------------------------------------------------------------
diff --git a/sharelib/distcp/src/main/java/org/apache/oozie/action/hadoop/DistcpMain.java b/sharelib/distcp/src/main/java/org/apache/oozie/action/hadoop/DistcpMain.java
index 2b84f92..fcaadef 100644
--- a/sharelib/distcp/src/main/java/org/apache/oozie/action/hadoop/DistcpMain.java
+++ b/sharelib/distcp/src/main/java/org/apache/oozie/action/hadoop/DistcpMain.java
@@ -33,12 +33,16 @@ import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
import org.apache.log4j.PropertyConfigurator;
+import com.google.common.annotations.VisibleForTesting;
+
public class DistcpMain extends JavaMain {
private Constructor<?> construct;
private Object[] constArgs;
private static final String DISTCP_LOG4J_PROPS = "distcp-log4j.properties";
- private static final Pattern[] DISTCP_JOB_IDS_PATTERNS = {
+
+ @VisibleForTesting
+ static final Pattern[] DISTCP_JOB_IDS_PATTERNS = {
Pattern.compile("Job complete: (job_\\S*)"),
Pattern.compile("Job (job_\\S*) completed successfully"),
Pattern.compile("Submitted application (application[0-9_]*)")
http://git-wip-us.apache.org/repos/asf/oozie/blob/4483f6dd/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java
----------------------------------------------------------------------
diff --git a/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java b/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java
index e1c7caa..bc8ab29 100644
--- a/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java
+++ b/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java
@@ -40,8 +40,11 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.cli.CliDriver;
import org.apache.hadoop.hive.conf.HiveConf;
+import com.google.common.annotations.VisibleForTesting;
+
public class HiveMain extends LauncherMain {
- private static final Pattern[] HIVE_JOB_IDS_PATTERNS = {
+ @VisibleForTesting
+ static final Pattern[] HIVE_JOB_IDS_PATTERNS = {
Pattern.compile("Ended Job = (job_\\S*)"),
Pattern.compile("Submitted application (application[0-9_]*)")
};
http://git-wip-us.apache.org/repos/asf/oozie/blob/4483f6dd/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveMain.java
----------------------------------------------------------------------
diff --git a/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveMain.java b/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveMain.java
index 3026311..2ba0da7 100644
--- a/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveMain.java
+++ b/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveMain.java
@@ -18,10 +18,6 @@
package org.apache.oozie.action.hadoop;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.oozie.test.MiniHCatServer;
-import org.apache.oozie.util.XConfiguration;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FileWriter;
@@ -29,16 +25,27 @@ import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.net.URL;
+import java.util.ArrayList;
+import java.util.LinkedHashSet;
+import java.util.List;
import java.util.Properties;
+import java.util.Set;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.oozie.test.MiniHCatServer;
+import org.apache.oozie.util.XConfiguration;
public class TestHiveMain extends MainTestCase {
private SecurityManager SECURITY_MANAGER;
+ @Override
protected void setUp() throws Exception {
super.setUp();
SECURITY_MANAGER = System.getSecurityManager();
}
+ @Override
protected void tearDown() throws Exception {
System.setSecurityManager(SECURITY_MANAGER);
super.tearDown();
@@ -61,6 +68,7 @@ public class TestHiveMain extends MainTestCase {
return buffer.toString();
}
+ @Override
public Void call() throws Exception {
if (System.getenv("HADOOP_HOME") == null) {
System.out.println("WARNING: 'HADOOP_HOME' env var not defined, TestHiveMain test is not running");
@@ -165,4 +173,23 @@ public class TestHiveMain extends MainTestCase {
}
return null;
}
+
+ public void testJobIDPattern() {
+ List<String> lines = new ArrayList<String>();
+ lines.add("Ended Job = job_001");
+ lines.add("Submitted application application_002");
+ // Non-matching ones
+ lines.add("Ended Job = . job_003");
+ lines.add("Ended Job = abc004");
+ lines.add("Submitted application = job_005");
+ lines.add("Submitted application. job_006");
+ Set<String> jobIds = new LinkedHashSet<String>();
+ for (String line : lines) {
+ LauncherMain.extractJobIDs(line, HiveMain.HIVE_JOB_IDS_PATTERNS, jobIds);
+ }
+ Set<String> expected = new LinkedHashSet<String>();
+ expected.add("job_001");
+ expected.add("job_002");
+ assertEquals(expected, jobIds);
+ }
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/4483f6dd/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java
----------------------------------------------------------------------
diff --git a/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java b/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java
index 54f7039..8e214fa 100644
--- a/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java
+++ b/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java
@@ -37,10 +37,14 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hive.beeline.BeeLine;
+import com.google.common.annotations.VisibleForTesting;
+
public class Hive2Main extends LauncherMain {
- private static final Pattern[] HIVE2_JOB_IDS_PATTERNS = {
+ @VisibleForTesting
+ static final Pattern[] HIVE2_JOB_IDS_PATTERNS = {
Pattern.compile("Ended Job = (job_\\S*)"),
- Pattern.compile("Submitted application (application[0-9_]*)")
+ Pattern.compile("Submitted application (application[0-9_]*)"),
+ Pattern.compile("Running with YARN Application = (application_[0-9]+)")
};
private static final Set<String> DISALLOWED_BEELINE_OPTIONS = new HashSet<String>();
http://git-wip-us.apache.org/repos/asf/oozie/blob/4483f6dd/sharelib/hive2/src/test/java/org/apache/oozie/action/hadoop/TestHive2Main.java
----------------------------------------------------------------------
diff --git a/sharelib/hive2/src/test/java/org/apache/oozie/action/hadoop/TestHive2Main.java b/sharelib/hive2/src/test/java/org/apache/oozie/action/hadoop/TestHive2Main.java
new file mode 100644
index 0000000..b94b7c3
--- /dev/null
+++ b/sharelib/hive2/src/test/java/org/apache/oozie/action/hadoop/TestHive2Main.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oozie.action.hadoop;
+
+import java.util.ArrayList;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.oozie.test.XTestCase;
+
+public class TestHive2Main extends XTestCase {
+
+ public void testJobIDPattern() {
+ List<String> lines = new ArrayList<String>();
+ lines.add("Ended Job = job_001");
+ lines.add("Running with YARN Application = application_002");
+ lines.add("Submitted application application_003");
+ // Non-matching ones
+ lines.add("Ended Job = ");
+ lines.add("Ended Job = abc");
+ lines.add("Running with YARN Application = job_004");
+ lines.add("Running with YARN Application = application005");
+ lines.add("Running with YARN Application = application_abc");
+ lines.add("Submitted application job_006");
+ Set<String> jobIds = new LinkedHashSet<String>();
+ for (String line : lines) {
+ LauncherMain.extractJobIDs(line, Hive2Main.HIVE2_JOB_IDS_PATTERNS,
+ jobIds);
+ }
+ Set<String> expected = new LinkedHashSet<String>();
+ expected.add("job_001");
+ expected.add("job_002");
+ expected.add("job_003");
+ assertEquals(expected, jobIds);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/oozie/blob/4483f6dd/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java
----------------------------------------------------------------------
diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java
index 785ca5e..8087178 100644
--- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java
+++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java
@@ -38,6 +38,10 @@ import java.util.regex.Pattern;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.util.Shell;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.mapred.JobConf;
@@ -69,17 +73,7 @@ public abstract class LauncherMain {
try (BufferedReader br = new BufferedReader(new FileReader(logFile))) {
String line = br.readLine();
while (line != null) {
- for (Pattern pattern : patterns) {
- Matcher matcher = pattern.matcher(line);
- if (matcher.find()) {
- String jobId = matcher.group(1);
- if (StringUtils.isEmpty(jobId) || jobId.equalsIgnoreCase("NULL")) {
- continue;
- }
- jobId = jobId.replaceAll("application", "job");
- jobIds.add(jobId);
- }
- }
+ extractJobIDs(line, patterns, jobIds);
line = br.readLine();
}
} catch (IOException e) {
@@ -90,6 +84,22 @@ public abstract class LauncherMain {
return jobIds.isEmpty() ? null : StringUtils.join(jobIds, ",");
}
+ @VisibleForTesting
+ protected static void extractJobIDs(String line, Pattern[] patterns, Set<String> jobIds) {
+ Preconditions.checkNotNull(line);
+ for (Pattern pattern : patterns) {
+ Matcher matcher = pattern.matcher(line);
+ if (matcher.find()) {
+ String jobId = matcher.group(1);
+ if (StringUtils.isEmpty(jobId) || jobId.equalsIgnoreCase("NULL")) {
+ continue;
+ }
+ jobId = jobId.replaceAll("application", "job");
+ jobIds.add(jobId);
+ }
+ }
+ }
+
protected static void writeExternalChildIDs(String logFile, Pattern[] patterns, String name) {
// Harvesting and recording Hadoop Job IDs
String jobIds = getHadoopJobIds(logFile, patterns);
http://git-wip-us.apache.org/repos/asf/oozie/blob/4483f6dd/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java
----------------------------------------------------------------------
diff --git a/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java b/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java
index 7de2e45..0c47ac8 100644
--- a/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java
+++ b/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java
@@ -23,6 +23,9 @@ import org.apache.pig.PigRunner;
import org.apache.pig.scripting.ScriptEngine;
import org.apache.pig.tools.pigstats.JobStats;
import org.apache.pig.tools.pigstats.PigStats;
+
+import com.google.common.annotations.VisibleForTesting;
+
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
@@ -54,7 +57,8 @@ public class PigMain extends LauncherMain {
public static final int STRING_BUFFER_SIZE = 100;
public static final String LOG_EXPANDED_PIG_SCRIPT = LauncherMapper.ACTION_PREFIX + "pig.log.expandedscript";
- private static final Pattern[] PIG_JOB_IDS_PATTERNS = {
+ @VisibleForTesting
+ static final Pattern[] PIG_JOB_IDS_PATTERNS = {
Pattern.compile("HadoopJobId: (job_\\S*)"),
Pattern.compile("Submitted application (application[0-9_]*)")
};
http://git-wip-us.apache.org/repos/asf/oozie/blob/4483f6dd/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigMain.java
----------------------------------------------------------------------
diff --git a/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigMain.java b/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigMain.java
index 081d319..22e7952 100644
--- a/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigMain.java
+++ b/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigMain.java
@@ -31,8 +31,12 @@ import java.io.OutputStreamWriter;
import java.io.Writer;
import java.io.FileWriter;
import java.io.FileReader;
+import java.util.ArrayList;
+import java.util.LinkedHashSet;
+import java.util.List;
import java.util.Map;
import java.util.Properties;
+import java.util.Set;
import java.net.URL;
public class TestPigMain extends PigTestCase {
@@ -157,4 +161,23 @@ public class TestPigMain extends PigTestCase {
return null;
}
+ public void testJobIDPattern() {
+ List<String> lines = new ArrayList<String>();
+ lines.add("HadoopJobId: job_001");
+ lines.add("Submitted application application_002");
+ // Non-matching ones
+ lines.add("HadoopJobId is set. job_003");
+ lines.add("HadoopJobId: abc004");
+ lines.add("Submitted application = job_005");
+ lines.add("Submitted application. job_006");
+ Set<String> jobIds = new LinkedHashSet<String>();
+ for (String line : lines) {
+ LauncherMain.extractJobIDs(line, PigMain.PIG_JOB_IDS_PATTERNS, jobIds);
+ }
+ Set<String> expected = new LinkedHashSet<String>();
+ expected.add("job_001");
+ expected.add("job_002");
+ assertEquals(expected, jobIds);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/4483f6dd/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
----------------------------------------------------------------------
diff --git a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
index 12eb61a..3cc6ccb 100644
--- a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
+++ b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
@@ -66,7 +66,8 @@ public class SparkMain extends LauncherMain {
Pattern.compile("pyspark.zip") };
private static final Pattern SPARK_DEFAULTS_FILE_PATTERN = Pattern.compile("spark-defaults.conf");
private static final String SPARK_LOG4J_PROPS = "spark-log4j.properties";
- private static final Pattern[] SPARK_JOB_IDS_PATTERNS = {
+ @VisibleForTesting
+ static final Pattern[] SPARK_JOB_IDS_PATTERNS = {
Pattern.compile("Submitted application (application[0-9_]*)") };
public static final Pattern SPARK_ASSEMBLY_JAR_PATTERN = Pattern
.compile("^spark-assembly((?:(-|_|(\\d+\\.))\\d+(?:\\.\\d+)*))*\\.jar$");
http://git-wip-us.apache.org/repos/asf/oozie/blob/4483f6dd/sharelib/spark/src/test/java/org/apache/oozie/action/hadoop/TestSparkMain.java
----------------------------------------------------------------------
diff --git a/sharelib/spark/src/test/java/org/apache/oozie/action/hadoop/TestSparkMain.java b/sharelib/spark/src/test/java/org/apache/oozie/action/hadoop/TestSparkMain.java
index bf4adf9..5f6a0cd 100644
--- a/sharelib/spark/src/test/java/org/apache/oozie/action/hadoop/TestSparkMain.java
+++ b/sharelib/spark/src/test/java/org/apache/oozie/action/hadoop/TestSparkMain.java
@@ -26,7 +26,9 @@ import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.util.ArrayList;
+import java.util.LinkedHashSet;
import java.util.List;
+import java.util.Set;
import java.util.regex.Pattern;
import org.apache.hadoop.conf.Configuration;
@@ -137,4 +139,20 @@ public class TestSparkMain extends MainTestCase {
instance.appendOoziePropertiesToSparkConf(sparkArgs, actionConf);
assertEquals(Lists.newArrayList("--conf", "spark.oozie.bar=bar"), sparkArgs);
}
+
+ public void testJobIDPattern() {
+ List<String> lines = new ArrayList<String>();
+ lines.add("Submitted application application_001");
+ // Non-matching ones
+ lines.add("Submitted application job_002");
+ lines.add("HadoopJobId: application_003");
+ lines.add("Submitted application = application_004");
+ Set<String> jobIds = new LinkedHashSet<String>();
+ for (String line : lines) {
+ LauncherMain.extractJobIDs(line, SparkMain.SPARK_JOB_IDS_PATTERNS, jobIds);
+ }
+ Set<String> expected = new LinkedHashSet<String>();
+ expected.add("job_001");
+ assertEquals(expected, jobIds);
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/oozie/blob/4483f6dd/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java
----------------------------------------------------------------------
diff --git a/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java b/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java
index cb65eb8..cb8f56e 100644
--- a/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java
+++ b/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java
@@ -32,11 +32,14 @@ import org.apache.hadoop.fs.Path;
import org.apache.log4j.PropertyConfigurator;
import org.apache.sqoop.Sqoop;
+import com.google.common.annotations.VisibleForTesting;
+
public class SqoopMain extends LauncherMain {
public static final String SQOOP_SITE_CONF = "sqoop-site.xml";
- private static final Pattern[] SQOOP_JOB_IDS_PATTERNS = {
+ @VisibleForTesting
+ static final Pattern[] SQOOP_JOB_IDS_PATTERNS = {
Pattern.compile("Job complete: (job_\\S*)"),
Pattern.compile("Job (job_\\S*) has completed successfully"),
Pattern.compile("Submitted application (application[0-9_]*)")
@@ -153,6 +156,7 @@ public class SqoopMain extends LauncherMain {
return logFile;
}
+ @Override
protected void run(String[] args) throws Exception {
System.out.println();
System.out.println("Oozie Sqoop action configuration");
http://git-wip-us.apache.org/repos/asf/oozie/blob/4483f6dd/sharelib/sqoop/src/test/java/org/apache/oozie/action/hadoop/TestSqoopMain.java
----------------------------------------------------------------------
diff --git a/sharelib/sqoop/src/test/java/org/apache/oozie/action/hadoop/TestSqoopMain.java b/sharelib/sqoop/src/test/java/org/apache/oozie/action/hadoop/TestSqoopMain.java
new file mode 100644
index 0000000..d6f96d5
--- /dev/null
+++ b/sharelib/sqoop/src/test/java/org/apache/oozie/action/hadoop/TestSqoopMain.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oozie.action.hadoop;
+
+import java.util.ArrayList;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.oozie.test.XTestCase;
+
+public class TestSqoopMain extends XTestCase {
+
+ public void testJobIDPattern() {
+ List<String> lines = new ArrayList<String>();
+ lines.add("Job complete: job_001");
+ lines.add("Job job_002 has completed successfully");
+ lines.add("Submitted application application_003");
+ // Non-matching ones
+ lines.add("Job complete: application_004");
+ lines.add("Job application_005 has completed successfully");
+ lines.add("Submitted application job_006");
+ Set<String> jobIds = new LinkedHashSet<String>();
+ for (String line : lines) {
+ LauncherMain.extractJobIDs(line, SqoopMain.SQOOP_JOB_IDS_PATTERNS,
+ jobIds);
+ }
+ Set<String> expected = new LinkedHashSet<String>();
+ expected.add("job_001");
+ expected.add("job_002");
+ expected.add("job_003");
+ assertEquals(expected, jobIds);
+ }
+}
\ No newline at end of file