You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by sa...@apache.org on 2017/12/09 00:51:36 UTC
[01/31] oozie git commit: OOZIE-2690 OOZIE NPE while executing kill()
(abhishekbafna via jaydeepvishwakarma)
Repository: oozie
Updated Branches:
refs/heads/branch-4.3 79fac26ab -> c2d4a14a2
OOZIE-2690 OOZIE NPE while executing kill() (abhishekbafna via jaydeepvishwakarma)
(cherry picked from commit 05337002fc50d78750979dc15ecea5ad80c14931)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/96f57b9e
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/96f57b9e
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/96f57b9e
Branch: refs/heads/branch-4.3
Commit: 96f57b9ee945a66583c563ec2b1d691f119fc297
Parents: 79fac26
Author: jvishwakarma <jv...@walmartlabs.com>
Authored: Mon Nov 21 12:01:39 2016 +0530
Committer: satishsaley <sa...@apache.org>
Committed: Fri Dec 8 16:33:57 2017 -0800
----------------------------------------------------------------------
.../org/apache/oozie/action/hadoop/JavaActionExecutor.java | 6 +++++-
.../apache/oozie/action/hadoop/MapReduceActionExecutor.java | 9 ++++-----
2 files changed, 9 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/96f57b9e/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
index 31d4817..4beeb96 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
@@ -1413,7 +1413,11 @@ public class JavaActionExecutor extends ActionExecutor {
}
protected RunningJob getRunningJob(Context context, WorkflowAction action, JobClient jobClient) throws Exception{
- RunningJob runningJob = jobClient.getJob(JobID.forName(action.getExternalId()));
+ String externalId = action.getExternalId();
+ RunningJob runningJob = null;
+ if (externalId != null) {
+ runningJob = jobClient.getJob(JobID.forName(externalId));
+ }
return runningJob;
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/96f57b9e/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java
index c36a675..55c9372 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java
@@ -335,12 +335,11 @@ public class MapReduceActionExecutor extends JavaActionExecutor {
@Override
protected RunningJob getRunningJob(Context context, WorkflowAction action, JobClient jobClient) throws Exception{
-
- RunningJob runningJob;
+ RunningJob runningJob = null;
String jobId = getActualExternalId(action);
-
- runningJob = jobClient.getJob(JobID.forName(jobId));
-
+ if (jobId != null) {
+ runningJob = jobClient.getJob(JobID.forName(jobId));
+ }
return runningJob;
}
[04/31] oozie git commit: OOZIE-2747 README.txt is out of date (Jan
Hentschel via rkanter)
Posted by sa...@apache.org.
OOZIE-2747 README.txt is out of date (Jan Hentschel via rkanter)
(cherry picked from commit 7b6d2354667dbacb2cb031cf856071d29d195ae6)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/f2320300
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/f2320300
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/f2320300
Branch: refs/heads/branch-4.3
Commit: f23203006a1241da463637e68c90ce5e5e05a666
Parents: e284e63
Author: Robert Kanter <rk...@apache.org>
Authored: Tue Jun 13 17:15:28 2017 -0700
Committer: satishsaley <sa...@apache.org>
Committed: Fri Dec 8 16:34:55 2017 -0800
----------------------------------------------------------------------
README.txt | 6 ++----
release-log.txt | 1 +
2 files changed, 3 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/f2320300/README.txt
----------------------------------------------------------------------
diff --git a/README.txt b/README.txt
index 0026687..86e76ba 100644
--- a/README.txt
+++ b/README.txt
@@ -42,13 +42,13 @@ More inforamtion could be found at:
http://oozie.apache.org/
Oozie Quick Start:
-http://oozie.apache.org/docs/3.2.0-incubating/DG_QuickStart.html
+http://oozie.apache.org/docs/4.3.0/DG_QuickStart.html
Supported Hadoop Versions:
----------------------------
-This version of Oozie was primarily tested against Hadoop 0.20.205.x. This will not work on earlier versions of Hadoop such as 0.20.x. and 0.21.
+This version of Oozie was primarily tested against Hadoop 2.4.x and 2.6.x.
--------------------------------------
@@ -59,5 +59,3 @@ user@oozie.apache.org
Subscribe using the link:
http://oozie.apache.org/mail-lists.html
-
-
http://git-wip-us.apache.org/repos/asf/oozie/blob/f2320300/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 8af2ee2..f831645 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.3.1 release
+OOZIE-2747 README.txt is out of date (Jan Hentschel via rkanter)
OOZIE-2923 Improve Spark options parsing (andras.piros via gezapeti)
OOZIE-2886 Ensure consistent versioning of hadoop jars in sharelibs (dbist13 via rkanter)
OOZIE-2872 Address backward compatibility issue introduced by OOZIE-2748 (pbacsko)
[25/31] oozie git commit: OOZIE-2863
SLACalculatorMemory.loadOnRestart causing delay in server start (puru via
satishsaley)
Posted by sa...@apache.org.
OOZIE-2863 SLACalculatorMemory.loadOnRestart causing delay in server start (puru via satishsaley)
(cherry picked from commit 8df784b5fb15273af31d8c104f0f7c47c78e82a3)
core/src/main/java/org/apache/oozie/sla/SLACalculatorMemory.java
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/f553c4e5
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/f553c4e5
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/f553c4e5
Branch: refs/heads/branch-4.3
Commit: f553c4e565b935d827fe172dba4a29a1499de1ae
Parents: 4b3b30d
Author: satishsaley <sa...@apache.org>
Authored: Wed Apr 19 23:11:29 2017 -0700
Committer: satishsaley <sa...@apache.org>
Committed: Fri Dec 8 16:34:55 2017 -0800
----------------------------------------------------------------------
.../org/apache/oozie/sla/SLACalcStatus.java | 65 ++++---
.../apache/oozie/sla/SLACalculatorMemory.java | 192 ++++++++++---------
.../apache/oozie/sla/service/SLAService.java | 5 +
.../apache/oozie/service/TestHASLAService.java | 3 +
.../oozie/sla/TestSLACalculatorMemory.java | 22 ++-
release-log.txt | 1 +
6 files changed, 166 insertions(+), 122 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/f553c4e5/core/src/main/java/org/apache/oozie/sla/SLACalcStatus.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/sla/SLACalcStatus.java b/core/src/main/java/org/apache/oozie/sla/SLACalcStatus.java
index 3a76dfe..7be16f0 100644
--- a/core/src/main/java/org/apache/oozie/sla/SLACalcStatus.java
+++ b/core/src/main/java/org/apache/oozie/sla/SLACalcStatus.java
@@ -36,6 +36,7 @@ public class SLACalcStatus extends SLAEvent {
public static String SLA_ENTITYKEY_PREFIX = "sla-";
private SLARegistrationBean regBean;
+ private SLASummaryBean summary;
private String jobStatus;
private SLAStatus slaStatus;
private EventStatus eventStatus;
@@ -44,6 +45,7 @@ public class SLACalcStatus extends SLAEvent {
private long actualDuration = -1;
private Date lastModifiedTime;
private byte eventProcessed;
+ private String jobId;
private XLog LOG;
@@ -54,27 +56,13 @@ public class SLACalcStatus extends SLAEvent {
}
public SLACalcStatus(SLASummaryBean summary, SLARegistrationBean regBean) {
+ this(summary);
+ updateSLARegistrationBean(regBean);
+ LOG = LogUtils.setLogPrefix(LOG, this);
+ }
+
+ public SLACalcStatus(SLASummaryBean summary) {
this();
- SLARegistrationBean reg = new SLARegistrationBean();
- reg.setNotificationMsg(regBean.getNotificationMsg());
- reg.setUpstreamApps(regBean.getUpstreamApps());
- reg.setAlertContact(regBean.getAlertContact());
- reg.setAlertEvents(regBean.getAlertEvents());
- reg.setJobData(regBean.getJobData());
- if (regBean.getSLAConfigMap().containsKey(OozieClient.SLA_DISABLE_ALERT)) {
- reg.addToSLAConfigMap(OozieClient.SLA_DISABLE_ALERT,
- regBean.getSLAConfigMap().get(OozieClient.SLA_DISABLE_ALERT));
- }
- reg.setId(summary.getId());
- reg.setAppType(summary.getAppType());
- reg.setUser(summary.getUser());
- reg.setAppName(summary.getAppName());
- reg.setParentId(summary.getParentId());
- reg.setNominalTime(summary.getNominalTime());
- reg.setExpectedStart(summary.getExpectedStart());
- reg.setExpectedEnd(summary.getExpectedEnd());
- reg.setExpectedDuration(summary.getExpectedDuration());
- setSLARegistrationBean(reg);
setActualStart(summary.getActualStart());
setActualEnd(summary.getActualEnd());
setActualDuration(summary.getActualDuration());
@@ -83,7 +71,8 @@ public class SLACalcStatus extends SLAEvent {
setEventStatus(summary.getEventStatus());
setLastModifiedTime(summary.getLastModifiedTime());
setEventProcessed(summary.getEventProcessed());
- LOG = LogUtils.setLogPrefix(LOG, this);
+ setId(summary.getId());
+ this.summary = summary;
}
/**
@@ -112,17 +101,24 @@ public class SLACalcStatus extends SLAEvent {
return regBean;
}
+ public SLASummaryBean getSLASummaryBean() {
+ return summary;
+ }
+
public void setSLARegistrationBean(SLARegistrationBean slaBean) {
+ if (slaBean != null) {
+ this.jobId = slaBean.getId();
+ }
this.regBean = slaBean;
}
@Override
public String getId() {
- return regBean.getId();
+ return jobId;
}
public void setId(String id) {
- regBean.setId(id);
+ this.jobId = id;
}
@Override
@@ -288,4 +284,27 @@ public class SLACalcStatus extends SLAEvent {
return SLA_ENTITYKEY_PREFIX + this.getId();
}
+ public void updateSLARegistrationBean(SLARegistrationBean slaBean) {
+ SLARegistrationBean reg = new SLARegistrationBean();
+ reg.setNotificationMsg(slaBean.getNotificationMsg());
+ reg.setUpstreamApps(slaBean.getUpstreamApps());
+ reg.setAlertContact(slaBean.getAlertContact());
+ reg.setAlertEvents(slaBean.getAlertEvents());
+ reg.setJobData(slaBean.getJobData());
+ if (slaBean.getSLAConfigMap().containsKey(OozieClient.SLA_DISABLE_ALERT)) {
+ reg.addToSLAConfigMap(OozieClient.SLA_DISABLE_ALERT,
+ slaBean.getSLAConfigMap().get(OozieClient.SLA_DISABLE_ALERT));
+ }
+ reg.setId(summary.getId());
+ reg.setAppType(summary.getAppType());
+ reg.setUser(summary.getUser());
+ reg.setAppName(summary.getAppName());
+ reg.setParentId(summary.getParentId());
+ reg.setNominalTime(summary.getNominalTime());
+ reg.setExpectedStart(summary.getExpectedStart());
+ reg.setExpectedEnd(summary.getExpectedEnd());
+ reg.setExpectedDuration(summary.getExpectedDuration());
+ setSLARegistrationBean(reg);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/f553c4e5/core/src/main/java/org/apache/oozie/sla/SLACalculatorMemory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/sla/SLACalculatorMemory.java b/core/src/main/java/org/apache/oozie/sla/SLACalculatorMemory.java
index 3522ffe..7fb3f0e 100644
--- a/core/src/main/java/org/apache/oozie/sla/SLACalculatorMemory.java
+++ b/core/src/main/java/org/apache/oozie/sla/SLACalculatorMemory.java
@@ -134,42 +134,14 @@ public class SLACalculatorMemory implements SLACalculator {
}
private void loadOnRestart() {
- long slaPendingCount = 0;
- long statusPendingCount = 0;
-
try {
- List<SLASummaryBean> summaryBeans = jpaService.execute(new SLASummaryGetRecordsOnRestartJPAExecutor(
- modifiedAfter));
+ List<SLASummaryBean> summaryBeans = jpaService
+ .execute(new SLASummaryGetRecordsOnRestartJPAExecutor(modifiedAfter));
for (SLASummaryBean summaryBean : summaryBeans) {
String jobId = summaryBean.getId();
-
- SLARegistrationBean slaRegBean = SLARegistrationQueryExecutor.getInstance().get(
- SLARegQuery.GET_SLA_REG_ON_RESTART, jobId);
- SLACalcStatus slaCalcStatus = new SLACalcStatus(summaryBean, slaRegBean);
-
- // Processed missed jobs
- try {
- SLAXCommandFactory.getSLAEventXCommand(slaCalcStatus).call();
- }
- catch (Throwable e) {
- LOG.error("Error while updating job {0}", slaCalcStatus.getId(), e);
- }
-
- if (slaCalcStatus.getEventProcessed() == 7) {
- historySet.add(jobId);
- statusPendingCount++;
- LOG.debug("Adding job [{0}] to historySet. EventProcessed is [{1}]", slaCalcStatus,
- slaCalcStatus);
- }
- else if (slaCalcStatus.getEventProcessed() < 7) {
- slaMap.put(jobId, slaCalcStatus);
- slaPendingCount++;
- LOG.debug("Adding job [{0}] to slamap. EventProcessed is [{1}]", slaCalcStatus,
- slaCalcStatus);
-
- }
+ slaMap.put(jobId, new SLACalcStatus(summaryBean));
}
- LOG.info("Loaded SLASummary pendingSLA=" + slaPendingCount + ", pendingStatusUpdate=" + statusPendingCount);
+ LOG.info("Loaded {0} SLASummary object after restart", slaMap.size());
}
catch (Exception e) {
LOG.warn("Failed to retrieve SLASummary records on restart", e);
@@ -198,13 +170,24 @@ public class SLACalculatorMemory implements SLACalculator {
return memObj;
}
- private SLACalcStatus getSLACalcStatus(String jobId) throws JPAExecutorException {
+ /**
+ * Get SLACalcStatus from map if SLARegistration is not null, else create a new SLACalcStatus
+ * This function deosn't update slaMap
+ * @param jobId
+ * @return
+ * @throws JPAExecutorException
+ */
+ private SLACalcStatus getOrCreateSLACalcStatus(String jobId) throws JPAExecutorException {
SLACalcStatus memObj;
memObj = slaMap.get(jobId);
- if (memObj == null) {
- memObj = new SLACalcStatus(SLASummaryQueryExecutor.getInstance()
- .get(SLASummaryQuery.GET_SLA_SUMMARY, jobId), SLARegistrationQueryExecutor.getInstance().get(
- SLARegQuery.GET_SLA_REG_ON_RESTART, jobId));
+ // if the request came from immediately after restart don't use map SLACalcStatus.
+ if (memObj == null || memObj.getSLARegistrationBean() == null) {
+ SLARegistrationBean registrationBean = SLARegistrationQueryExecutor.getInstance()
+ .get(SLARegQuery.GET_SLA_REG_ON_RESTART, jobId);
+ SLASummaryBean summaryBean = memObj == null
+ ? SLASummaryQueryExecutor.getInstance().get(SLASummaryQuery.GET_SLA_SUMMARY, jobId)
+ : memObj.getSLASummaryBean();
+ return new SLACalcStatus(summaryBean, registrationBean);
}
return memObj;
}
@@ -235,52 +218,55 @@ public class SLACalculatorMemory implements SLACalculator {
// job might be processed and removed from map by addJobStatus
return;
}
- synchronized (slaCalc) {
- // get eventProcessed on DB for validation in HA
- SLASummaryBean summaryBean = null;
- try {
- summaryBean = ((SLASummaryQueryExecutor) SLASummaryQueryExecutor.getInstance()).get(
- SLASummaryQuery.GET_SLA_SUMMARY_EVENTPROCESSED_LAST_MODIFIED, jobId);
+ boolean firstCheckAfterRetstart = checkAndUpdateSLACalcAfterRestart(slaCalc);
+ // get eventProcessed on DB for validation in HA
+ SLASummaryBean summaryBean = null;
+ try {
+ summaryBean = ((SLASummaryQueryExecutor) SLASummaryQueryExecutor.getInstance())
+ .get(SLASummaryQuery.GET_SLA_SUMMARY_EVENTPROCESSED_LAST_MODIFIED, jobId);
+ }
+ catch (JPAExecutorException e) {
+ if (e.getErrorCode().equals(ErrorCode.E0604) || e.getErrorCode().equals(ErrorCode.E0605)) {
+ LOG.debug("job [{0}] is is not in DB, removing from Memory", jobId);
+ slaMap.remove(jobId);
+ return;
}
- catch (JPAExecutorException e) {
- if (e.getErrorCode().equals(ErrorCode.E0604) || e.getErrorCode().equals(ErrorCode.E0605)) {
- LOG.debug("job [{0}] is is not in DB, removing from Memory", jobId);
- slaMap.remove(jobId);
- return;
- }
- throw e;
+ throw e;
+ }
+ byte eventProc = summaryBean.getEventProcessed();
+ slaCalc.setEventProcessed(eventProc);
+ if (eventProc >= 7) {
+ if (eventProc == 7) {
+ historySet.add(jobId);
}
- byte eventProc = summaryBean.getEventProcessed();
- slaCalc.setEventProcessed(eventProc);
- if (eventProc >= 7) {
- if (eventProc == 7) {
- historySet.add(jobId);
- }
- slaMap.remove(jobId);
- LOG.trace("Removed Job [{0}] from map as SLA processed", jobId);
+ slaMap.remove(jobId);
+ LOG.trace("Removed Job [{0}] from map as SLA processed", jobId);
+ }
+ else {
+ if (!slaCalc.getLastModifiedTime().equals(summaryBean.getLastModifiedTime())) {
+ // Update last modified time.
+ slaCalc.setLastModifiedTime(summaryBean.getLastModifiedTime());
+ reloadExpectedTimeAndConfig(slaCalc);
+ LOG.debug("Last modified time has changed for job " + jobId + " reloading config from DB");
}
- else {
- if (!slaCalc.getLastModifiedTime().equals(summaryBean.getLastModifiedTime())) {
- // Update last modified time.
- slaCalc.setLastModifiedTime(summaryBean.getLastModifiedTime());
- reloadExpectedTimeAndConfig(slaCalc);
- LOG.debug("Last modified time has changed for job " + jobId + " reloading config from DB");
+ if (firstCheckAfterRetstart || isChanged(slaCalc)) {
+ LOG.debug("{0} job has SLA event change. EventProc = {1}, status = {2}", slaCalc.getId(),
+ slaCalc.getEventProcessed(), slaCalc.getJobStatus());
+ try {
+ SLAXCommandFactory.getSLAEventXCommand(slaCalc).call();
+ checkEventProc(slaCalc);
}
- if (isChanged(slaCalc)) {
- LOG.debug("{0} job has SLA event change. EventProc = {1}, status = {2}", slaCalc.getId(),
- slaCalc.getEventProcessed(), slaCalc.getJobStatus());
- try {
- SLAXCommandFactory.getSLAEventXCommand(slaCalc).call();
- checkEventProc(slaCalc);
+ catch (XException e) {
+ if (e.getErrorCode().equals(ErrorCode.E0604) || e.getErrorCode().equals(ErrorCode.E0605)) {
+ LOG.debug("job [{0}] is is not in DB, removing from Memory", slaCalc.getId());
+ slaMap.remove(jobId);
}
- catch (XException e) {
- if (e.getErrorCode().equals(ErrorCode.E0604) || e.getErrorCode().equals(ErrorCode.E0605)) {
- LOG.debug("job [{0}] is is not in DB, removing from Memory", slaCalc.getId());
- slaMap.remove(jobId);
+ else {
+ if (firstCheckAfterRetstart) {
+ slaCalc.setSLARegistrationBean(null);
}
}
}
-
}
}
}
@@ -473,7 +459,7 @@ public class SLACalculatorMemory implements SLACalculator {
"Received addJobStatus request for job [{0}] jobStatus = [{1}], jobEventStatus = [{2}], startTime = [{3}], "
+ "endTime = [{4}] ", jobId, jobStatus, jobEventStatus, startTime, endTime);
SLACalcStatus slaCalc = slaMap.get(jobId);
-
+ boolean firstCheckAfterRetstart = checkAndUpdateSLACalcAfterRestart(slaCalc);
if (slaCalc == null) {
SLARegistrationBean slaRegBean = SLARegistrationQueryExecutor.getInstance().get(
SLARegQuery.GET_SLA_REG_ALL, jobId);
@@ -505,6 +491,9 @@ public class SLACalculatorMemory implements SLACalculator {
checkEventProc(slaCalc);
}
catch (XException e) {
+ if (firstCheckAfterRetstart) {
+ slaCalc.setSLARegistrationBean(null);
+ }
LOG.error(e);
throw new ServiceException(e);
}
@@ -570,12 +559,10 @@ public class SLACalculatorMemory implements SLACalculator {
@SuppressWarnings("rawtypes")
List<UpdateEntry> updateList = new ArrayList<BatchQueryExecutor.UpdateEntry>();
for (String jobId : jobIds) {
- SLACalcStatus slaCalc = getSLACalcStatus(jobId);
- if (slaCalc != null) {
- slaCalc.getSLARegistrationBean().removeFromSLAConfigMap(OozieClient.SLA_DISABLE_ALERT);
- updateDBSlaConfig(slaCalc, updateList);
- isJobFound = true;
- }
+ SLACalcStatus slaCalc = getOrCreateSLACalcStatus(jobId);
+ slaCalc.getSLARegistrationBean().removeFromSLAConfigMap(OozieClient.SLA_DISABLE_ALERT);
+ updateDBSlaConfig(slaCalc, updateList);
+ isJobFound = true;
}
executeBatchQuery(updateList);
return isJobFound;
@@ -593,13 +580,10 @@ public class SLACalculatorMemory implements SLACalculator {
List<UpdateEntry> updateList = new ArrayList<BatchQueryExecutor.UpdateEntry>();
for (String jobId : jobIds) {
- SLACalcStatus slaCalc = getSLACalcStatus(jobId);
- if (slaCalc != null) {
- slaCalc.getSLARegistrationBean().addToSLAConfigMap(OozieClient.SLA_DISABLE_ALERT,
- Boolean.toString(true));
- updateDBSlaConfig(slaCalc, updateList);
- isJobFound = true;
- }
+ SLACalcStatus slaCalc = getOrCreateSLACalcStatus(jobId);
+ slaCalc.getSLARegistrationBean().addToSLAConfigMap(OozieClient.SLA_DISABLE_ALERT, Boolean.toString(true));
+ updateDBSlaConfig(slaCalc, updateList);
+ isJobFound = true;
}
executeBatchQuery(updateList);
return isJobFound;
@@ -617,12 +601,10 @@ public class SLACalculatorMemory implements SLACalculator {
@SuppressWarnings("rawtypes")
List<UpdateEntry> updateList = new ArrayList<BatchQueryExecutor.UpdateEntry>();
for (Pair<String, Map<String, String>> jobIdSLAPair : jobIdsSLAPair) {
- SLACalcStatus slaCalc = getSLACalcStatus(jobIdSLAPair.getFist());
- if (slaCalc != null) {
- updateParams(slaCalc, jobIdSLAPair.getSecond());
- updateDBSlaExpectedValues(slaCalc, updateList);
- isJobFound = true;
- }
+ SLACalcStatus slaCalc = getOrCreateSLACalcStatus(jobIdSLAPair.getFist());
+ updateParams(slaCalc, jobIdSLAPair.getSecond());
+ updateDBSlaExpectedValues(slaCalc, updateList);
+ isJobFound = true;
}
executeBatchQuery(updateList);
return isJobFound;
@@ -655,4 +637,24 @@ public class SLACalculatorMemory implements SLACalculator {
return childJobIds;
}
+ private boolean checkAndUpdateSLACalcAfterRestart(SLACalcStatus slaCalc) throws JPAExecutorException {
+ if (slaCalc != null && slaCalc.getSLARegistrationBean() == null) {
+ return updateSLARegistartion(slaCalc);
+ }
+ return false;
+ }
+
+ public boolean updateSLARegistartion(SLACalcStatus slaCalc) throws JPAExecutorException {
+ if (slaCalc.getSLARegistrationBean() == null) {
+ synchronized (slaCalc) {
+ if (slaCalc.getSLARegistrationBean() == null) {
+ SLARegistrationBean slaRegBean = SLARegistrationQueryExecutor.getInstance()
+ .get(SLARegQuery.GET_SLA_REG_ON_RESTART, slaCalc.getId());
+ slaCalc.updateSLARegistrationBean(slaRegBean);
+ return true;
+ }
+ }
+ }
+ return false;
+ }
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/f553c4e5/core/src/main/java/org/apache/oozie/sla/service/SLAService.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/sla/service/SLAService.java b/core/src/main/java/org/apache/oozie/sla/service/SLAService.java
index 08cd07e..2d23a22 100644
--- a/core/src/main/java/org/apache/oozie/sla/service/SLAService.java
+++ b/core/src/main/java/org/apache/oozie/sla/service/SLAService.java
@@ -114,6 +114,11 @@ public class SLAService implements Service {
new SLAWorker(calcImpl).run();
}
+ @VisibleForTesting
+ public void startSLAWorker() {
+ new Thread(new SLAWorker(calcImpl)).start();
+ }
+
private class SLAWorker implements Runnable {
SLACalculator calc;
http://git-wip-us.apache.org/repos/asf/oozie/blob/f553c4e5/core/src/test/java/org/apache/oozie/service/TestHASLAService.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/service/TestHASLAService.java b/core/src/test/java/org/apache/oozie/service/TestHASLAService.java
index 3af263e..67fa56e 100644
--- a/core/src/test/java/org/apache/oozie/service/TestHASLAService.java
+++ b/core/src/test/java/org/apache/oozie/service/TestHASLAService.java
@@ -207,6 +207,7 @@ public class TestHASLAService extends ZKXTestCase {
SLAService slas = Services.get().get(SLAService.class);
SLACalculatorMemory slaCalcMem = (SLACalculatorMemory) slas.getSLACalculator();
slaCalcMem.init(Services.get().getConf());
+ slaCalcMem.updateAllSlaStatus();
List<String> slaMapKeys = new ArrayList<String>();
Iterator<String> itr = slaCalcMem.iterator();
while (itr.hasNext()) {
@@ -226,6 +227,7 @@ public class TestHASLAService extends ZKXTestCase {
dummySlaCalcMem.setEventHandlerService(dummyEhs);
dummyEhs.init(Services.get());
dummySlaCalcMem.init(Services.get().getConf());
+ dummySlaCalcMem.updateAllSlaStatus();
slaMapKeys = new ArrayList<String>();
itr = dummySlaCalcMem.iterator();
while (itr.hasNext()) {
@@ -402,6 +404,7 @@ public class TestHASLAService extends ZKXTestCase {
SLAService slas = Services.get().get(SLAService.class);
SLACalculatorMemory slaCalcMem1 = (SLACalculatorMemory) slas.getSLACalculator();
slaCalcMem1.init(Services.get().get(ConfigurationService.class).getConf());
+ slaCalcMem1.updateAllSlaStatus();
List<String> idList = new ArrayList<String>();
idList.add(id);
slaCalcMem1.disableAlert(idList);
http://git-wip-us.apache.org/repos/asf/oozie/blob/f553c4e5/core/src/test/java/org/apache/oozie/sla/TestSLACalculatorMemory.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/sla/TestSLACalculatorMemory.java b/core/src/test/java/org/apache/oozie/sla/TestSLACalculatorMemory.java
index 559e2b3..34011f6 100644
--- a/core/src/test/java/org/apache/oozie/sla/TestSLACalculatorMemory.java
+++ b/core/src/test/java/org/apache/oozie/sla/TestSLACalculatorMemory.java
@@ -144,6 +144,7 @@ public class TestSLACalculatorMemory extends XDataTestCase {
slaCalcMemory.addRegistration(jobId2, slaRegBean2);
slaCalcMemory.addRegistration(jobId3, slaRegBean3);
+ slaCalcMemory.updateAllSlaStatus();
SLACalcStatus calc1 = slaCalcMemory.get(jobId1);
SLACalcStatus calc2 = slaCalcMemory.get(jobId2);
SLACalcStatus calc3 = slaCalcMemory.get(jobId3);
@@ -190,7 +191,7 @@ public class TestSLACalculatorMemory extends XDataTestCase {
slaCalcMemory = new SLACalculatorMemory();
slaCalcMemory.init(Services.get().get(ConfigurationService.class).getConf());
-
+ slaCalcMemory.updateAllSlaStatus();
SLACalcStatus calc = new SLACalcStatus(SLASummaryQueryExecutor.getInstance().get(
SLASummaryQuery.GET_SLA_SUMMARY, jobId1), SLARegistrationQueryExecutor.getInstance().get(
@@ -250,7 +251,7 @@ public class TestSLACalculatorMemory extends XDataTestCase {
SLACalculatorMemory slaCalcMemory = new SLACalculatorMemory();
slaCalcMemory.init(Services.get().get(ConfigurationService.class).getConf());
SLARegistrationBean slaRegBean1 = _createSLARegistration("job-1-W", AppType.WORKFLOW_JOB);
- String jobId1 = slaRegBean1.getId();
+ final String jobId1 = slaRegBean1.getId();
slaRegBean1.setExpectedEnd(sdf.parse("2013-03-07"));
slaRegBean1.setExpectedStart(sdf.parse("2012-03-07"));
slaCalcMemory.addRegistration(jobId1, slaRegBean1);
@@ -276,6 +277,8 @@ public class TestSLACalculatorMemory extends XDataTestCase {
slaCalcMemory = new SLACalculatorMemory();
slaCalcMemory.init(Services.get().get(ConfigurationService.class).getConf());
+ slaCalcMemory.updateAllSlaStatus();
+
slaSummary = SLASummaryQueryExecutor.getInstance().get(SLASummaryQuery.GET_SLA_SUMMARY, jobId1);
assertEquals("job-1-W", slaSummary.getId());
assertEquals(8, slaSummary.getEventProcessed());
@@ -302,6 +305,7 @@ public class TestSLACalculatorMemory extends XDataTestCase {
slaCalcMemory = new SLACalculatorMemory();
slaCalcMemory.init(Services.get().get(ConfigurationService.class).getConf());
+ slaCalcMemory.updateAllSlaStatus();
slaSummary = SLASummaryQueryExecutor.getInstance().get(SLASummaryQuery.GET_SLA_SUMMARY, jobId1);
assertEquals("FAILED", slaSummary.getJobStatus());
@@ -323,8 +327,16 @@ public class TestSLACalculatorMemory extends XDataTestCase {
SLASummaryQueryExecutor.getInstance().executeUpdate(SLASummaryQuery.UPDATE_SLA_SUMMARY_ALL, slaSummaryBean);
- slaCalcMemory = new SLACalculatorMemory();
- slaCalcMemory.init(Services.get().get(ConfigurationService.class).getConf());
+ SLAService slaService = Services.get().get(SLAService.class);
+ slaService.startSLAWorker();
+ slaService.addStatusEvent(jobId1, "RUNNING", null, null, null);
+ waitFor(60 * 1000, new Predicate() {
+ public boolean evaluate() throws Exception {
+ return SLASummaryQueryExecutor.getInstance().get(SLASummaryQuery.GET_SLA_SUMMARY, jobId1)
+ .getEventProcessed() == 7;
+ }
+ });
+
slaSummary = SLASummaryQueryExecutor.getInstance().get(SLASummaryQuery.GET_SLA_SUMMARY, slaSummaryBean.getId());
//since job is already running and it's a old job
assertEquals(7, slaSummary.getEventProcessed());
@@ -484,6 +496,8 @@ public class TestSLACalculatorMemory extends XDataTestCase {
CoordActionQueryExecutor.getInstance().executeUpdate(CoordActionQuery.UPDATE_COORD_ACTION_FOR_START, coordAction2);
slaCalcMemory.init(Services.get().get(ConfigurationService.class).getConf());
+ slaCalcMemory.updateAllSlaStatus();
+
slaSummary = SLASummaryQueryExecutor.getInstance().get(SLASummaryQuery.GET_SLA_SUMMARY, jobId1);
slaSummary2 = SLASummaryQueryExecutor.getInstance().get(SLASummaryQuery.GET_SLA_SUMMARY, jobId2);
http://git-wip-us.apache.org/repos/asf/oozie/blob/f553c4e5/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 0649b46..2b36ec0 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.3.1 release
+OOZIE-2863 SLACalculatorMemory.loadOnRestart causing delay in server start (puru via satishsaley)
OOZIE-2862 Coord change command doesn't change job to running if job was killed without creating any actions (puru)
OOZIE-2811 amend Add support for filtering out properties from SparkConfigurationService
OOZIE-2807 Oozie gets RM delegation token even for checking job status (satishsaley)
[30/31] oozie git commit: OOZIE-2885 Running Spark actions should not
need Hive on the classpath (satishsaley)
Posted by sa...@apache.org.
OOZIE-2885 Running Spark actions should not need Hive on the classpath (satishsaley)
(cherry picked from commit 0034f7f471dd5bc9a61aac0ae957c50976f552c8)
sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/c2d4a14a
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/c2d4a14a
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/c2d4a14a
Branch: refs/heads/branch-4.3
Commit: c2d4a14a20ba5153d9a0752944c753048dd42530
Parents: d1c57dc
Author: satishsaley <sa...@apache.org>
Authored: Mon Sep 25 09:49:40 2017 -0700
Committer: satishsaley <sa...@apache.org>
Committed: Fri Dec 8 16:34:56 2017 -0800
----------------------------------------------------------------------
release-log.txt | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/c2d4a14a/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 5063bbe..729d27c 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.3.1 release
+OOZIE-2885 Running Spark actions should not need Hive on the classpath (satishsaley)
OOZIE-3058 nocleanup option is missing in oozie-coordinator-0.5.xsd (satishsaley)
OOZIE-3036 Spark 2.2.0 support: tell Spark not to get any delegation tokens (andras.piros via gezapeti)
OOZIE-3028 Oozie Pig Action fails with no python dependencies (dbist13 via rohini)
[27/31] oozie git commit: OOZIE-2802 Spark action failure on Spark
2.1.0 due to duplicate sharelibs (gezapeti via rkanter)
Posted by sa...@apache.org.
OOZIE-2802 Spark action failure on Spark 2.1.0 due to duplicate sharelibs (gezapeti via rkanter)
(cherry picked from commit e8bd9fc9271d07aaa074df3fd45e3b8d51974ce1)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/ca9f3374
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/ca9f3374
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/ca9f3374
Branch: refs/heads/branch-4.3
Commit: ca9f337406b869dda7e4d773e730878d91042470
Parents: 4363cca
Author: Robert Kanter <rk...@apache.org>
Authored: Thu Mar 2 16:57:50 2017 -0800
Committer: satishsaley <sa...@apache.org>
Committed: Fri Dec 8 16:34:55 2017 -0800
----------------------------------------------------------------------
release-log.txt | 1 +
.../apache/oozie/action/hadoop/SparkMain.java | 61 ++++++++++----
.../TestDuplicateFilteringInSparkMain.java | 87 ++++++++++++++++++++
3 files changed, 133 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/ca9f3374/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 222c48f..cf43d61 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.3.1 release
+OOZIE-2802 Spark action failure on Spark 2.1.0 due to duplicate sharelibs (gezapeti via rkanter)
OOZIE-2803 Mask passwords when printing out configs/args in MapReduceMain and SparkMain (pbacsko via rkanter)
OOZIE-2799 Setting log location for spark sql on hive (satishsaley)
OOZIE-2792 Hive2 action is not parsing Spark application ID from log file properly when Hive is on Spark (zhengxb2005 via rkanter)
http://git-wip-us.apache.org/repos/asf/oozie/blob/ca9f3374/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
----------------------------------------------------------------------
diff --git a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
index 88ac64e..c24d95c 100644
--- a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
+++ b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
@@ -27,8 +27,9 @@ import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
import java.util.Iterator;
-import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Properties;
@@ -79,6 +80,8 @@ public class SparkMain extends LauncherMain {
private static final String SPARK_YARN_JAR = "spark.yarn.jar";
private static final String SPARK_YARN_JARS = "spark.yarn.jars";
public static final String HIVE_SITE_CONF = "hive-site.xml";
+ public static final String FILES_OPTION = "--files";
+ public static final String ARCHIVES_OPTION = "--archives";
public static void main(String[] args) throws Exception {
run(SparkMain.class, args);
@@ -135,6 +138,7 @@ public class SparkMain extends LauncherMain {
boolean addedLog4jExecutorSettings = false;
StringBuilder driverClassPath = new StringBuilder();
StringBuilder executorClassPath = new StringBuilder();
+ String userFiles = null, userArchives = null;
String sparkOpts = actionConf.get(SparkActionExecutor.SPARK_OPTS);
if (StringUtils.isNotEmpty(sparkOpts)) {
List<String> sparkOptions = splitSparkOpts(sparkOpts);
@@ -177,6 +181,16 @@ public class SparkMain extends LauncherMain {
addedLog4jDriverSettings = true;
}
}
+ if(opt.startsWith(FILES_OPTION)) {
+ userFiles = sparkOptions.get(i + 1);
+ i++;
+ addToSparkArgs = false;
+ }
+ if(opt.startsWith(ARCHIVES_OPTION)) {
+ userArchives = sparkOptions.get(i + 1);
+ i++;
+ addToSparkArgs = false;
+ }
if(addToSparkArgs) {
sparkArgs.add(opt);
}
@@ -225,19 +239,24 @@ public class SparkMain extends LauncherMain {
}
if ((yarnClusterMode || yarnClientMode)) {
- LinkedList<URI> fixedUris = fixFsDefaultUris(DistributedCache.getCacheFiles(actionConf));
- JarFilter jarfilter = new JarFilter(fixedUris, jarPath);
+ Map<String, URI> fixedFileUrisMap = fixFsDefaultUrisAndFilterDuplicates(DistributedCache.getCacheFiles(actionConf));
+ fixedFileUrisMap.put(SPARK_LOG4J_PROPS, new Path(SPARK_LOG4J_PROPS).toUri());
+ fixedFileUrisMap.put(HIVE_SITE_CONF, new Path(HIVE_SITE_CONF).toUri());
+ addUserDefined(userFiles, fixedFileUrisMap);
+ Collection<URI> fixedFileUris = fixedFileUrisMap.values();
+ JarFilter jarfilter = new JarFilter(fixedFileUris, jarPath);
jarfilter.filter();
jarPath = jarfilter.getApplicationJar();
- fixedUris.add(new Path(SPARK_LOG4J_PROPS).toUri());
- fixedUris.add(new Path(HIVE_SITE_CONF).toUri());
- String cachedFiles = StringUtils.join(fixedUris, ",");
+
+ String cachedFiles = StringUtils.join(fixedFileUris, ",");
if (cachedFiles != null && !cachedFiles.isEmpty()) {
sparkArgs.add("--files");
sparkArgs.add(cachedFiles);
}
- fixedUris = fixFsDefaultUris(DistributedCache.getCacheArchives(actionConf));
- String cachedArchives = StringUtils.join(fixedUris, ",");
+ Map<String, URI> fixedArchiveUrisMap = fixFsDefaultUrisAndFilterDuplicates(DistributedCache.
+ getCacheArchives(actionConf));
+ addUserDefined(userArchives, fixedArchiveUrisMap);
+ String cachedArchives = StringUtils.join(fixedArchiveUrisMap.values(), ",");
if (cachedArchives != null && !cachedArchives.isEmpty()) {
sparkArgs.add("--archives");
sparkArgs.add(cachedArchives);
@@ -277,6 +296,15 @@ public class SparkMain extends LauncherMain {
}
}
+ private void addUserDefined(String userList, Map<String, URI> urisMap) {
+ if(userList != null) {
+ for (String file : userList.split(",")) {
+ Path p = new Path(file);
+ urisMap.put(p.getName(), p.toUri());
+ }
+ }
+ }
+
private void prepareHadoopConfig(Configuration actionConf) throws IOException {
// Copying oozie.action.conf.xml into hadoop configuration *-site files.
if (actionConf.getBoolean(CONF_OOZIE_SPARK_SETUP_HADOOP_CONF_DIR, false)) {
@@ -433,23 +461,24 @@ public class SparkMain extends LauncherMain {
/**
* Convert URIs into the default format which Spark expects
- *
+ * Also filters out duplicate entries
* @param files
* @return
* @throws IOException
* @throws URISyntaxException
*/
- private LinkedList<URI> fixFsDefaultUris(URI[] files) throws IOException, URISyntaxException {
+ static Map<String, URI> fixFsDefaultUrisAndFilterDuplicates(URI[] files) throws IOException, URISyntaxException {
+ Map<String, URI> map= new HashMap<>();
if (files == null) {
- return null;
+ return map;
}
- LinkedList<URI> listUris = new LinkedList<URI>();
FileSystem fs = FileSystem.get(new Configuration(true));
for (int i = 0; i < files.length; i++) {
URI fileUri = files[i];
- listUris.add(getFixedUri(fs, fileUri));
+ Path p = new Path(fileUri);
+ map.put(p.getName(), getFixedUri(fs, fileUri));
}
- return listUris;
+ return map;
}
/**
@@ -573,7 +602,7 @@ public class SparkMain extends LauncherMain {
private String sparkVersion = "1.X.X";
private String sparkYarnJar;
private String applicationJar;
- private LinkedList<URI> listUris = null;
+ private Collection<URI> listUris = null;
/**
* @param listUris List of URIs to be filtered
@@ -581,7 +610,7 @@ public class SparkMain extends LauncherMain {
* @throws IOException
* @throws URISyntaxException
*/
- public JarFilter(LinkedList<URI> listUris, String jarPath) throws URISyntaxException, IOException {
+ public JarFilter(Collection<URI> listUris, String jarPath) throws URISyntaxException, IOException {
this.listUris = listUris;
applicationJar = jarPath;
Path p = new Path(jarPath);
http://git-wip-us.apache.org/repos/asf/oozie/blob/ca9f3374/sharelib/spark/src/test/java/org/apache/oozie/action/hadoop/TestDuplicateFilteringInSparkMain.java
----------------------------------------------------------------------
diff --git a/sharelib/spark/src/test/java/org/apache/oozie/action/hadoop/TestDuplicateFilteringInSparkMain.java b/sharelib/spark/src/test/java/org/apache/oozie/action/hadoop/TestDuplicateFilteringInSparkMain.java
new file mode 100644
index 0000000..9a231b1
--- /dev/null
+++ b/sharelib/spark/src/test/java/org/apache/oozie/action/hadoop/TestDuplicateFilteringInSparkMain.java
@@ -0,0 +1,87 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oozie.action.hadoop;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import static org.junit.Assert.*;
+import static org.hamcrest.CoreMatchers.*;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+@RunWith(Parameterized.class)
+public class TestDuplicateFilteringInSparkMain {
+
+ private static final String PREFIX = "hdfs://namenode.address:8020/folder/";
+
+ static List<URI> getURIs(String... uriStrings) throws URISyntaxException {
+ URI[] uris = new URI[uriStrings.length];
+ for (int i = 0; i != uriStrings.length; ++i) {
+ uris[i] = new URI(PREFIX + uriStrings[i]);
+ }
+ return Arrays.asList(uris);
+ }
+
+ static Object[] testCase(List<URI> inputs, List<URI> expectedOutputs) {
+ return new Object[] {inputs, expectedOutputs};
+ }
+
+ @Parameterized.Parameters
+ public static List<Object[]> params() throws Exception {
+ return Arrays.asList(
+ testCase(getURIs("file.io"),
+ getURIs("file.io")),
+
+ testCase(getURIs("file.io", "file.io", "file.io"),
+ getURIs("file.io")),
+
+ testCase(getURIs("file.io", "file3.io", "file.io"),
+ getURIs("file.io", "file3.io")),
+
+ testCase(getURIs("file.io", "file3.io", "file2.io"),
+ getURIs("file.io", "file2.io", "file3.io"))
+ );
+ }
+
+ private List<URI> input;
+
+ private List<URI> expectedOutput;
+
+ public TestDuplicateFilteringInSparkMain(List<URI> input, List<URI> result) {
+ this.input = input;
+ this.expectedOutput = result;
+ }
+
+ @Test
+ public void test() throws Exception{
+ Map<String, URI> uriMap = SparkMain.fixFsDefaultUrisAndFilterDuplicates(input.toArray(new URI[input.size()]));
+ assertThat("Duplicate filtering failed for >>" + input + "<<", uriMap.size(), is(expectedOutput.size()));
+ List<URI> outputList = Arrays.asList(uriMap.values().toArray(new URI[0]));
+ Collections.sort(outputList);
+ assertThat("Files are different in result ", outputList, is(expectedOutput));
+ }
+
+}
[31/31] oozie git commit: OOZIE-3058 nocleanup option is missing in
oozie-coordinator-0.5.xsd (satishsaley)
Posted by sa...@apache.org.
OOZIE-3058 nocleanup option is missing in oozie-coordinator-0.5.xsd (satishsaley)
(cherry picked from commit 5e4d28843c64671a1308a4484f8d68c3e2afccbb)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/d1c57dcd
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/d1c57dcd
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/d1c57dcd
Branch: refs/heads/branch-4.3
Commit: d1c57dcd989ee964cbe859e60a58688a2e8a7069
Parents: 169a1a2
Author: satishsaley <sa...@apache.org>
Authored: Wed Sep 20 09:47:46 2017 -0700
Committer: satishsaley <sa...@apache.org>
Committed: Fri Dec 8 16:34:56 2017 -0800
----------------------------------------------------------------------
client/src/main/resources/oozie-coordinator-0.5.xsd | 1 +
docs/src/site/twiki/CoordinatorFunctionalSpec.twiki | 2 ++
release-log.txt | 1 +
3 files changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/d1c57dcd/client/src/main/resources/oozie-coordinator-0.5.xsd
----------------------------------------------------------------------
diff --git a/client/src/main/resources/oozie-coordinator-0.5.xsd b/client/src/main/resources/oozie-coordinator-0.5.xsd
index 2b63629..80909ad 100644
--- a/client/src/main/resources/oozie-coordinator-0.5.xsd
+++ b/client/src/main/resources/oozie-coordinator-0.5.xsd
@@ -163,6 +163,7 @@
</xs:sequence>
<xs:attribute name="name" type="coordinator:IDENTIFIER" use="required"/>
<xs:attribute name="dataset" type="xs:string" use="required"/>
+ <xs:attribute name="nocleanup" type="xs:boolean" use="optional"/>
</xs:complexType>
<xs:complexType name="ACTION">
<xs:sequence minOccurs="1" maxOccurs="1">
http://git-wip-us.apache.org/repos/asf/oozie/blob/d1c57dcd/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki b/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki
index e3ac514..b285020 100644
--- a/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki
+++ b/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki
@@ -3899,6 +3899,7 @@ the notification:
</xs:sequence>
<xs:attribute name="name" type="coordinator:IDENTIFIER" use="required"/>
<xs:attribute name="dataset" type="xs:string" use="required"/>
+ <xs:attribute name="nocleanup" type="xs:boolean" use="optional"/>
</xs:complexType>
<xs:complexType name="ACTION">
<xs:sequence minOccurs="1" maxOccurs="1">
@@ -4024,6 +4025,7 @@ the notification:
</xs:sequence>
<xs:attribute name="name" type="coordinator:IDENTIFIER" use="required"/>
<xs:attribute name="dataset" type="coordinator:IDENTIFIER" use="required"/>
+ <xs:attribute name="nocleanup" type="xs:boolean" use="optional"/>
</xs:complexType>
<xs:complexType name="ACTION">
<xs:sequence minOccurs="1" maxOccurs="1">
http://git-wip-us.apache.org/repos/asf/oozie/blob/d1c57dcd/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 16d3af8..5063bbe 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.3.1 release
+OOZIE-3058 nocleanup option is missing in oozie-coordinator-0.5.xsd (satishsaley)
OOZIE-3036 Spark 2.2.0 support: tell Spark not to get any delegation tokens (andras.piros via gezapeti)
OOZIE-3028 Oozie Pig Action fails with no python dependencies (dbist13 via rohini)
OOZIE-2984 Parse spark-defaults.conf values with spaces without needing the quotes (andras.piros via gezapeti)
[10/31] oozie git commit: OOZIE-2777 Config-default.xml longer than
64k results in java.io.UTFDataFormatException (gezapeti via harsh)
Posted by sa...@apache.org.
OOZIE-2777 Config-default.xml longer than 64k results in java.io.UTFDataFormatException (gezapeti via harsh)
(cherry picked from commit a914196882e845b9908f041c5b0d8712e4eada6b)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/f5b835b3
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/f5b835b3
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/f5b835b3
Branch: refs/heads/branch-4.3
Commit: f5b835b3d3931cc371d35b2c1a25a3920c971f6c
Parents: 6e5dccf
Author: Harsh J <ha...@cloudera.com>
Authored: Thu Feb 2 16:23:08 2017 +0530
Committer: satishsaley <sa...@apache.org>
Committed: Fri Dec 8 16:34:55 2017 -0800
----------------------------------------------------------------------
.../oozie/util/StringSerializationUtil.java | 72 +++++++++++++++++
.../oozie/workflow/lite/LiteWorkflowApp.java | 61 +++++----------
.../workflow/lite/LiteWorkflowInstance.java | 33 +-------
.../org/apache/oozie/workflow/lite/NodeDef.java | 21 +++--
.../oozie/command/wf/TestSubmitXCommand.java | 47 ++++++++++-
.../oozie/util/TestStringSerializationUtil.java | 53 +++++++++++++
.../workflow/lite/TestLiteWorkflowApp.java | 78 +++++++++++++++++++
.../test/resources/oldWorkFlowApp.serialized | Bin 0 -> 68945 bytes
release-log.txt | 1 +
9 files changed, 288 insertions(+), 78 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/f5b835b3/core/src/main/java/org/apache/oozie/util/StringSerializationUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/util/StringSerializationUtil.java b/core/src/main/java/org/apache/oozie/util/StringSerializationUtil.java
new file mode 100644
index 0000000..6f930c0
--- /dev/null
+++ b/core/src/main/java/org/apache/oozie/util/StringSerializationUtil.java
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oozie.util;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+
+/**
+ * This class provides a workaround for the 64k limit for string in DataOutput.
+ */
+public final class StringSerializationUtil {
+ // Using unique string to indicate version. This is to make sure that it
+ // doesn't match with user data.
+ private static final String DATA_VERSION = "V==1";
+ private static final int CONVERSION_TRESHOLD = 60000;
+
+ private StringSerializationUtil() {
+ }
+
+ /**
+ * Writes out value to dOut. Converts it to byte array if the length of the UTF-8 byte array representation of the
+ * string is longer than 60k bytes.
+ *
+ * @param dOut
+ * @param value
+ * @throws IOException
+ */
+ public static void writeString(DataOutput dOut, String value) throws IOException {
+ if (value == null) {
+ dOut.writeUTF(value);
+ return;
+ }
+
+ byte[] data = value.getBytes(StandardCharsets.UTF_8.name());
+ if (data.length > CONVERSION_TRESHOLD) {
+ dOut.writeUTF(DATA_VERSION);
+ dOut.writeInt(data.length);
+ dOut.write(data);
+ } else {
+ dOut.writeUTF(value);
+ }
+ }
+
+ public static String readString(DataInput dIn) throws IOException {
+ String value = dIn.readUTF();
+ if (DATA_VERSION.equals(value)) {
+ int length = dIn.readInt();
+ byte[] data = new byte[length];
+ dIn.readFully(data);
+ value = new String(data, StandardCharsets.UTF_8.name());
+ }
+ return value;
+ }
+}
http://git-wip-us.apache.org/repos/asf/oozie/blob/f5b835b3/core/src/main/java/org/apache/oozie/workflow/lite/LiteWorkflowApp.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/workflow/lite/LiteWorkflowApp.java b/core/src/main/java/org/apache/oozie/workflow/lite/LiteWorkflowApp.java
index a06595d..5e6dc7a 100644
--- a/core/src/main/java/org/apache/oozie/workflow/lite/LiteWorkflowApp.java
+++ b/core/src/main/java/org/apache/oozie/workflow/lite/LiteWorkflowApp.java
@@ -20,24 +20,26 @@ package org.apache.oozie.workflow.lite;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.oozie.util.StringSerializationUtil;
import org.apache.oozie.workflow.WorkflowApp;
import org.apache.oozie.workflow.WorkflowException;
import org.apache.oozie.util.ParamChecker;
-import org.apache.oozie.util.XLog;
import org.apache.oozie.ErrorCode;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
-import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.LinkedHashMap;
-import java.util.List;
import java.util.Map;
//TODO javadoc
public class LiteWorkflowApp implements Writable, WorkflowApp {
+ /**
+ * Serialization of strings longer than 65k changed. This flag marks which method to use during reading.
+ */
+ public static final int NEW_SERIALIZATION_METHOD_FLAG = -1;
private String name;
private String definition;
private Map<String, NodeDef> nodesMap = new LinkedHashMap<String, NodeDef>();
@@ -111,13 +113,9 @@ public class LiteWorkflowApp implements Writable, WorkflowApp {
@Override
public void write(DataOutput dataOutput) throws IOException {
dataOutput.writeUTF(name);
- //dataOutput.writeUTF(definition);
- //writeUTF() has limit 65535, so split long string to multiple short strings
- List<String> defList = divideStr(definition);
- dataOutput.writeInt(defList.size());
- for (String d : defList) {
- dataOutput.writeUTF(d);
- }
+ // write out -1 as a marker to use StringSerializationUtil. Previously it was split to 20k long bits in a list.
+ dataOutput.writeInt(NEW_SERIALIZATION_METHOD_FLAG);
+ StringSerializationUtil.writeString(dataOutput, definition);
dataOutput.writeInt(nodesMap.size());
for (NodeDef n : getNodeDefs()) {
dataOutput.writeUTF(n.getClass().getName());
@@ -125,43 +123,22 @@ public class LiteWorkflowApp implements Writable, WorkflowApp {
}
}
- /**
- * To split long string to a list of smaller strings.
- *
- * @param str
- * @return List
- */
- private List<String> divideStr(String str) {
- List<String> list = new ArrayList<String>();
- int len = 20000;
- int strlen = str.length();
- int start = 0;
- int end = len;
-
- while (end < strlen) {
- list.add(str.substring(start, end));
- start = end;
- end += len;
- }
-
- if (strlen <= end) {
- list.add(str.substring(start, strlen));
- }
- return list;
- }
-
@Override
public void readFields(DataInput dataInput) throws IOException {
name = dataInput.readUTF();
- //definition = dataInput.readUTF();
//read the full definition back
- int defListSize = dataInput.readInt();
- StringBuilder sb = new StringBuilder();
- for (int i = 0; i < defListSize; i++) {
- sb.append(dataInput.readUTF());
+ int definitionListFlag = dataInput.readInt();
+ if(definitionListFlag > NEW_SERIALIZATION_METHOD_FLAG) {
+ // negative number marking the usage of StringSerializationUtil
+ // positive number is the length of the array the String was broken into.
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < definitionListFlag; i++) {
+ sb.append(dataInput.readUTF());
+ }
+ definition = sb.toString();
+ } else {
+ definition = StringSerializationUtil.readString(dataInput);
}
- definition = sb.toString();
-
int numNodes = dataInput.readInt();
for (int x = 0; x < numNodes; x++) {
try {
http://git-wip-us.apache.org/repos/asf/oozie/blob/f5b835b3/core/src/main/java/org/apache/oozie/workflow/lite/LiteWorkflowInstance.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/workflow/lite/LiteWorkflowInstance.java b/core/src/main/java/org/apache/oozie/workflow/lite/LiteWorkflowInstance.java
index 2b13e67..560353c 100644
--- a/core/src/main/java/org/apache/oozie/workflow/lite/LiteWorkflowInstance.java
+++ b/core/src/main/java/org/apache/oozie/workflow/lite/LiteWorkflowInstance.java
@@ -25,6 +25,7 @@ import org.apache.oozie.ErrorCode;
import org.apache.oozie.client.OozieClient;
import org.apache.oozie.service.DagXLogInfoService;
import org.apache.oozie.service.XLogService;
+import org.apache.oozie.util.StringSerializationUtil;
import org.apache.oozie.util.ParamChecker;
import org.apache.oozie.util.XConfiguration;
import org.apache.oozie.util.XLog;
@@ -52,10 +53,6 @@ public class LiteWorkflowInstance implements Writable, WorkflowInstance {
private static String ROOT = PATH_SEPARATOR;
private static String TRANSITION_SEPARATOR = "#";
- // Using unique string to indicate version. This is to make sure that it
- // doesn't match with user data.
- private static final String DATA_VERSION = "V==1";
-
private static class NodeInstance {
String nodeName;
boolean started = false;
@@ -556,6 +553,7 @@ public class LiteWorkflowInstance implements Writable, WorkflowInstance {
@Override
public void write(DataOutput dOut) throws IOException {
+
dOut.writeUTF(instanceId);
//Hadoop Configuration has to get its act right
@@ -577,7 +575,7 @@ public class LiteWorkflowInstance implements Writable, WorkflowInstance {
dOut.writeInt(persistentVars.size());
for (Map.Entry<String, String> entry : persistentVars.entrySet()) {
dOut.writeUTF(entry.getKey());
- writeStringAsBytes(entry.getValue(), dOut);
+ StringSerializationUtil.writeString(dOut, entry.getValue());
}
}
@@ -607,34 +605,11 @@ public class LiteWorkflowInstance implements Writable, WorkflowInstance {
int numVars = dIn.readInt();
for (int x = 0; x < numVars; x++) {
String vName = dIn.readUTF();
- String vVal = readBytesAsString(dIn);
- persistentVars.put(vName, vVal);
+ persistentVars.put(vName, StringSerializationUtil.readString(dIn));
}
refreshLog();
}
- private void writeStringAsBytes(String value, DataOutput dOut) throws IOException {
- if (value == null) {
- dOut.writeUTF(null);
- return;
- }
- dOut.writeUTF(DATA_VERSION);
- byte[] data = value.getBytes("UTF-8");
- dOut.writeInt(data.length);
- dOut.write(data);
- }
-
- private String readBytesAsString(DataInput dIn) throws IOException {
- String value = dIn.readUTF();
- if (value != null && value.equals(DATA_VERSION)) {
- int length = dIn.readInt();
- byte[] data = new byte[length];
- dIn.readFully(data);
- value = new String(data, "UTF-8");
- }
- return value;
- }
-
@Override
public Configuration getConf() {
return conf;
http://git-wip-us.apache.org/repos/asf/oozie/blob/f5b835b3/core/src/main/java/org/apache/oozie/workflow/lite/NodeDef.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/workflow/lite/NodeDef.java b/core/src/main/java/org/apache/oozie/workflow/lite/NodeDef.java
index 496b008..9ecad01 100644
--- a/core/src/main/java/org/apache/oozie/workflow/lite/NodeDef.java
+++ b/core/src/main/java/org/apache/oozie/workflow/lite/NodeDef.java
@@ -20,6 +20,7 @@ package org.apache.oozie.workflow.lite;
import org.apache.hadoop.io.Writable;
import org.apache.oozie.service.LiteWorkflowStoreService;
+import org.apache.oozie.util.StringSerializationUtil;
import org.apache.oozie.util.ParamChecker;
import org.apache.oozie.workflow.WorkflowException;
@@ -151,7 +152,7 @@ public class NodeDef implements Writable {
throw new IOException(ex);
}
}
- conf = dataInput.readUTF();
+ conf = readString(dataInput);
if (conf.equals("null")) {
conf = null;
}
@@ -195,7 +196,7 @@ public class NodeDef implements Writable {
throw new IOException(ex);
}
}
- conf = dataInput.readUTF();
+ conf = readString(dataInput);
if (conf.equals("null")) {
conf = null;
}
@@ -237,7 +238,7 @@ public class NodeDef implements Writable {
}
dataOutput.writeUTF(handlerClass.getName());
if (conf != null) {
- dataOutput.writeUTF(conf);
+ writeString(dataOutput, conf);
}
else {
dataOutput.writeUTF("null");
@@ -248,6 +249,14 @@ public class NodeDef implements Writable {
}
}
+ private void writeString(DataOutput dataOutput, String value) throws IOException {
+ StringSerializationUtil.writeString(dataOutput, value);
+ }
+
+ private String readString(DataInput dataInput) throws IOException {
+ return StringSerializationUtil.readString(dataInput);
+ }
+
/**
* Write as version one format, this version was since 3.1.
*
@@ -281,14 +290,14 @@ public class NodeDef implements Writable {
dataOutput.writeUTF(nodeDefVersion);
dataOutput.writeUTF(name);
if (cred != null) {
- dataOutput.writeUTF(cred);
+ writeString(dataOutput, cred);
}
else {
dataOutput.writeUTF("null");
}
- dataOutput.writeUTF(handlerClass.getName());
+ writeString(dataOutput, handlerClass.getName());
if (conf != null) {
- dataOutput.writeUTF(conf);
+ writeString(dataOutput, conf);
}
else {
dataOutput.writeUTF("null");
http://git-wip-us.apache.org/repos/asf/oozie/blob/f5b835b3/core/src/test/java/org/apache/oozie/command/wf/TestSubmitXCommand.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/command/wf/TestSubmitXCommand.java b/core/src/test/java/org/apache/oozie/command/wf/TestSubmitXCommand.java
index 47ff8ca..e71a662 100644
--- a/core/src/test/java/org/apache/oozie/command/wf/TestSubmitXCommand.java
+++ b/core/src/test/java/org/apache/oozie/command/wf/TestSubmitXCommand.java
@@ -27,6 +27,7 @@ import java.io.PrintWriter;
import java.io.StringReader;
import java.net.URI;
+import org.apache.commons.lang.RandomStringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.oozie.ErrorCode;
import org.apache.oozie.WorkflowActionBean;
@@ -122,6 +123,50 @@ public class TestSubmitXCommand extends XDataTestCase {
}
}
+ public void testSubmitLongXml() throws Exception {
+ Configuration conf = new XConfiguration();
+ String workflowUri = getTestCaseFileUri("workflow.xml");
+ String actionXml = "<map-reduce>"
+ + "<job-tracker>${jobTracker}</job-tracker>"
+ + "<name-node>${nameNode}</name-node>"
+ + " <prepare>"
+ + " <delete path=\"${nameNode}/user/${wf:user()}/mr/${outputDir}\"/>"
+ + " </prepare>"
+ + " <configuration>"
+ + " <property><name>bb</name><value>BB</value></property>"
+ + " <property><name>cc</name><value>from_action</value></property>"
+ + " </configuration>"
+ + " </map-reduce>";
+ String appXml = "<workflow-app xmlns='uri:oozie:workflow:0.5' name='too-long-wf'> " +
+ "<global><configuration>"+generate64kData()+"</configuration></global><start to='mr-node' /> "
+ + " <action name=\"mr-node\">"
+ + actionXml
+ + " <ok to=\"end\"/>"
+ + " <error to=\"end\"/>"
+ + "</action>"
+ + "<end name='end' /> " + "</workflow-app>";
+
+ writeToFile(appXml, workflowUri);
+ conf.set(OozieClient.APP_PATH, workflowUri);
+ conf.set(OozieClient.USER_NAME, getTestUser());
+ addBunchOfProperties(conf);
+
+ SubmitXCommand sc = new SubmitXCommand(conf);
+ sc.call();
+ }
+
+ private void addBunchOfProperties(Configuration conf) {
+ int i=0;
+ while(conf.size() < 10000){
+ conf.set("ID"+i, i+"something");
+ i++;
+ }
+ }
+
+ private String generate64kData() {
+ return "<property><name>radnom</name><value>"+ RandomStringUtils.randomAlphanumeric(70000)+"</value></property>";
+ }
+
public void testAppPathIsFile1() throws Exception {
Configuration conf = new XConfiguration();
String workflowUri = getTestCaseFileUri("workflow.xml");
@@ -375,8 +420,8 @@ public class TestSubmitXCommand extends XDataTestCase {
assertEquals(getNameNodeUri()+"/default-output-dir", actionConf.get("mixed"));
}
+
private void writeToFile(String appXml, String appPath) throws IOException {
- // TODO Auto-generated method stub
File wf = new File(URI.create(appPath));
PrintWriter out = null;
try {
http://git-wip-us.apache.org/repos/asf/oozie/blob/f5b835b3/core/src/test/java/org/apache/oozie/util/TestStringSerializationUtil.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/util/TestStringSerializationUtil.java b/core/src/test/java/org/apache/oozie/util/TestStringSerializationUtil.java
new file mode 100644
index 0000000..1c580db
--- /dev/null
+++ b/core/src/test/java/org/apache/oozie/util/TestStringSerializationUtil.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oozie.util;
+
+import org.apache.commons.lang.RandomStringUtils;
+import org.junit.Test;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInput;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.DataOutputStream;
+import java.io.IOException;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+public class TestStringSerializationUtil {
+
+ @Test
+ public void testStrings() {
+ for (int i = 1; i < 150000; i += 10000) {
+ String value = RandomStringUtils.random(i);
+ try {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ DataOutput dataOutput = new DataOutputStream(baos);
+ StringSerializationUtil.writeString(dataOutput, value);
+ DataInput dataInput = new DataInputStream(new ByteArrayInputStream(baos.toByteArray()));
+ assertEquals("Error in serialization for size " + i,
+ value, StringSerializationUtil.readString(dataInput));
+ } catch (IOException e) {
+ e.printStackTrace();
+ fail("Error in serialization for size " + i + "\n" + value);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/oozie/blob/f5b835b3/core/src/test/java/org/apache/oozie/workflow/lite/TestLiteWorkflowApp.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/workflow/lite/TestLiteWorkflowApp.java b/core/src/test/java/org/apache/oozie/workflow/lite/TestLiteWorkflowApp.java
new file mode 100644
index 0000000..74c1618
--- /dev/null
+++ b/core/src/test/java/org/apache/oozie/workflow/lite/TestLiteWorkflowApp.java
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oozie.workflow.lite;
+
+import com.google.common.base.Strings;
+import org.apache.commons.lang.RandomStringUtils;
+import org.apache.oozie.service.LiteWorkflowStoreService;
+import org.apache.oozie.service.Services;
+import org.apache.oozie.test.XTestCase;
+import org.apache.oozie.util.IOUtils;
+import org.junit.Test;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+
+
+public class TestLiteWorkflowApp extends XTestCase {
+
+ @Override
+ protected void setUp() throws Exception {
+ super.setUp();
+ new Services().init();
+ }
+
+ @Override
+ protected void tearDown() throws Exception {
+ Services.get().destroy();
+ super.tearDown();
+ }
+
+ @Test
+ public void testReadWrite() throws Exception{
+ String definition = "test"+ RandomStringUtils.random(100 * 1024);
+ LiteWorkflowApp app = new LiteWorkflowApp("name", definition, new StartNodeDef(LiteWorkflowStoreService.LiteControlNodeHandler.class, "foo"));
+ app.addNode(new EndNodeDef("foo", LiteWorkflowStoreService.LiteControlNodeHandler.class));
+ ByteArrayOutputStream baos= new ByteArrayOutputStream();
+ DataOutputStream out = new DataOutputStream(baos);
+ app.write(out);
+ ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
+ DataInputStream in = new DataInputStream(bais);
+ LiteWorkflowApp app2 = new LiteWorkflowApp();
+ app2.readFields(in);
+ assertTrue(app.equals(app2));
+ }
+
+ /**
+ * Before OOZIE-2777 the "defintion" field of LiteWorkflowApp was split into 20k long strings and serialized out
+ * one after the other.
+ * @throws Exception
+ */
+ @Test
+ public void testOldFormatRead() throws Exception{
+ String definition = Strings.repeat("abcdefghijk", 6234);
+ LiteWorkflowApp app = new LiteWorkflowApp("name", definition, new StartNodeDef(LiteWorkflowStoreService.LiteControlNodeHandler.class, "foo"));
+ app.addNode(new EndNodeDef("foo", LiteWorkflowStoreService.LiteControlNodeHandler.class));
+ DataInputStream in = new DataInputStream(IOUtils.getResourceAsStream("oldWorkFlowApp.serialized", -1));
+ LiteWorkflowApp app2 = new LiteWorkflowApp();
+ app2.readFields(in);
+ assertTrue(app.equals(app2));
+ }
+}
http://git-wip-us.apache.org/repos/asf/oozie/blob/f5b835b3/core/src/test/resources/oldWorkFlowApp.serialized
----------------------------------------------------------------------
diff --git a/core/src/test/resources/oldWorkFlowApp.serialized b/core/src/test/resources/oldWorkFlowApp.serialized
new file mode 100644
index 0000000..be2e2cd
Binary files /dev/null and b/core/src/test/resources/oldWorkFlowApp.serialized differ
http://git-wip-us.apache.org/repos/asf/oozie/blob/f5b835b3/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 147149e..3ee166a 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.3.1 release
+OOZIE-2777 Config-default.xml longer than 64k results in java.io.UTFDataFormatException (gezapeti via harsh)
OOZIE-2771 Allow retrieving keystore and truststore passwords from Hadoop Credential Provider (asasvari via abhishekbafna)
OOZIE-2748 NPE in LauncherMapper.printArgs() (pbacsko via rkanter)
OOZIE-2654 Zookeeper dependent services should not depend on Connectionstate to be valid before cleaning up (venkatnrangan via abhishekbafna)
[22/31] oozie git commit: OOZIE-2984 Parse spark-defaults.conf values
with spaces without needing the quotes (andras.piros via gezapeti)
Posted by sa...@apache.org.
OOZIE-2984 Parse spark-defaults.conf values with spaces without needing the quotes (andras.piros via gezapeti)
(cherry picked from commit 1ae68908b26e7a36089c47ca64c3c683dc2f6099)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/0f293eaa
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/0f293eaa
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/0f293eaa
Branch: refs/heads/branch-4.3
Commit: 0f293eaa5d5efa9e317db8fc606c59b4ff6dd99f
Parents: 636e434
Author: Gezapeti Cseh <ge...@gmail.com>
Authored: Fri Jul 28 13:33:40 2017 +0200
Committer: satishsaley <sa...@apache.org>
Committed: Fri Dec 8 16:34:55 2017 -0800
----------------------------------------------------------------------
.../site/twiki/DG_SparkActionExtension.twiki | 11 +-
release-log.txt | 1 +
.../action/hadoop/SparkOptionsSplitter.java | 130 +++++++++++++++----
.../action/hadoop/TestSparkArgsExtractor.java | 37 ++++++
.../action/hadoop/TestSparkOptionsSplitter.java | 36 ++++-
5 files changed, 184 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/0f293eaa/docs/src/site/twiki/DG_SparkActionExtension.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/DG_SparkActionExtension.twiki b/docs/src/site/twiki/DG_SparkActionExtension.twiki
index 94175cd..a815869 100644
--- a/docs/src/site/twiki/DG_SparkActionExtension.twiki
+++ b/docs/src/site/twiki/DG_SparkActionExtension.twiki
@@ -97,9 +97,16 @@ The =class= element if present, indicates the spark's application main class.
The =jar= element indicates a comma separated list of jars or python files.
-The =spark-opts= element if present, contains a list of spark options that can be passed to spark driver. Spark configuration
+The =spark-opts= element, if present, contains a list of spark options that can be passed to spark driver. Spark configuration
options can be passed by specifying '--conf key=value' here, or from =oozie.service.SparkConfigurationService.spark.configurations=
-in oozie-site.xml. Values containing whitespaces should be enclosed by double quotes. The =spark-opts= configs have priority.
+in oozie-site.xml. Values containing whitespaces can be enclosed by double quotes. The =spark-opts= configs have priority.
+
+Some examples of the =spark-opts= element:
+ * '--conf key=value'
+ * '--conf key1=value1 value2'
+ * '--conf key1="value1 value2"'
+ * '--conf key1=value1 key2="value2 value3"'
+ * '--conf key=value --verbose'
The =arg= element if present, contains arguments that can be passed to spark application.
http://git-wip-us.apache.org/repos/asf/oozie/blob/0f293eaa/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 0995972..a0946fe 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.3.1 release
+OOZIE-2984 Parse spark-defaults.conf values with spaces without needing the quotes (andras.piros via gezapeti)
OOZIE-2825 Custom Authentication doc page is not well formatted (Jan Hentschel via rkanter)
OOZIE-2747 README.txt is out of date (Jan Hentschel via rkanter)
OOZIE-2923 Improve Spark options parsing (andras.piros via gezapeti)
http://git-wip-us.apache.org/repos/asf/oozie/blob/0f293eaa/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkOptionsSplitter.java
----------------------------------------------------------------------
diff --git a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkOptionsSplitter.java b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkOptionsSplitter.java
index 30def6f..b614322 100644
--- a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkOptionsSplitter.java
+++ b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkOptionsSplitter.java
@@ -18,47 +18,133 @@
package org.apache.oozie.action.hadoop;
+import org.apache.commons.lang.StringUtils;
+
import java.util.ArrayList;
import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
class SparkOptionsSplitter {
/**
- * Converts the options to be Spark-compatible.
+ * Matches an option key.
+ * <p/>
+ * Some examples:
+ * <ul>
+ * <li>{@code key1}</li>
+ * <li>{@code key2=}</li>
+ * <li>{@code key3a-key3b}</li>
+ * <li>{@code key4a-KEY4b=}</li>
+ * </ul>
+ */
+ private static final Pattern OPTION_KEY_PREFIX = Pattern.compile("\\s*--[a-zA-Z0-9.]+[\\-a-zA-Z0-9.]*[=]?");
+
+ /**
+ * Matches an option key / value pair, where the whole value part is quoted.
+ * <p/>
+ * Some examples:
+ * <ul>
+ * <li>{@code spark.executor.extraJavaOptions="-XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp"}</li>
+ * <li>{@code spark.executor.extraJavaOptions="-XX:HeapDumpPath=/tmp"}</li>
+ * </ul>
+ */
+ private static final String VALUE_HAS_QUOTES_AT_ENDS_REGEX = "[a-zA-Z0-9.]+=\".+\"";
+
+ /**
+ * Matches an option key / value pair, where the value part has quotes in between.
+ * <p/>
+ * Some examples:
* <ul>
- * <li>Parameters are separated by whitespace and can be groupped using double quotes</li>
- * <li>Quotes should be removed</li>
- * <li>Adjacent whitespace separators are treated as one</li>
+ * <li>{@code spark.executor.extraJavaOptions=-XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp
+ * -Dlog4j.configuration="/some path/spark-log4j.properties"}</li>
+ * <li>{@code spark.executor.extraJavaOptions=-XX:+HeapDumpOnOutOfMemoryError
+ * -Dlog4j.configuration="/some path/spark-log4j.properties" -XX:HeapDumpPath=/tmp}</li>
+ * <li>{@code spark.executor.extraJavaOptions=-Dlog4j.configuration="/some path/spark-log4j.properties"
+ * -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp}</li>
* </ul>
+ */
+ private static final String VALUE_HAS_QUOTES_IN_BETWEEN_REGEX =
+ "[a-zA-Z0-9.]+=.*(\\w\\s+\"\\w+[\\s+\\w]*\"|\"\\w+[\\s+\\w]*\"\\s+\\w)+.*";
+
+ /**
+ * Converts the options to be Spark-compatible.
+ * <p/>
+ * Following will be considered when converting a single option:
+ * <ul>
+ * <li>parameter keys should begin with {@link #OPTION_KEY_PREFIX}</li>
+ * <li>parameter values should be separated by whitespace(s) and can be grouped using double quotes</li>
+ * <li>double quotes should be removed when the entire value part of a parameter key / value pair is double quoted,
+ * and no partial value is quoted. {@see #VALUE_HAS_QUOTES_AT_ENDS_REGEX}
+ * and {@see #VALUE_HAS_QUOTES_IN_BETWEEN_REGEX}</li>
+ * <li>double quotes should be kept when only a value part of a parameter key / value pair is double quoted</li>
+ * <li>adjacent whitespace separators are treated as one</li>
+ * </ul>
+ * <p/>
+ * Please have a look at the scenarios within {@code TestSparkOptionsSplitter} for details.
*
* @param sparkOpts the options for Spark
- * @return the options parsed into a list
+ * @return the options parsed into a {@link List}
*/
static List<String> splitSparkOpts(final String sparkOpts) {
- final List<String> result = new ArrayList<String>();
- final StringBuilder currentWord = new StringBuilder();
-
- boolean insideQuote = false;
- for (int i = 0; i < sparkOpts.length(); i++) {
- final char c = sparkOpts.charAt(i);
- if (c == '"') {
- insideQuote = !insideQuote;
- }
- else if (Character.isWhitespace(c) && !insideQuote) {
- if (currentWord.length() > 0) {
- result.add(currentWord.toString());
- currentWord.setLength(0);
+ final List<String> result = new ArrayList<>();
+ final Matcher matcher = OPTION_KEY_PREFIX.matcher(sparkOpts);
+
+ int start = 0, end;
+ while (matcher.find()) {
+ end = matcher.start();
+
+ if (start > 0) {
+ final String maybeQuotedValue = sparkOpts.substring(start, end).trim();
+ if (StringUtils.isNotEmpty(maybeQuotedValue)) {
+ result.add(unquoteEntirelyQuotedValue(maybeQuotedValue));
}
}
- else {
- currentWord.append(c);
+
+ String sparkOpt = matchSparkOpt(sparkOpts, matcher);
+ if (sparkOpt.endsWith("=")) {
+ sparkOpt = sparkOpt.replaceAll("=", "");
}
+ result.add(sparkOpt);
+
+ start = matcher.end();
}
- if (currentWord.length() > 0) {
- result.add(currentWord.toString());
+ final String maybeEntirelyQuotedValue = sparkOpts.substring(start).trim();
+ if (StringUtils.isNotEmpty(maybeEntirelyQuotedValue)) {
+ result.add(unquoteEntirelyQuotedValue(maybeEntirelyQuotedValue));
}
return result;
}
+
+ private static String matchSparkOpt(final String sparkOpts, final Matcher matcher) {
+ return sparkOpts.substring(matcher.start(), matcher.end()).trim();
+ }
+
+ /**
+ * Unquote an option value if and only if it has quotes at both ends, and doesn't have any quotes in between.
+ * <p/>
+ * Some examples:
+ * <ul>
+ * <li>{@code key=value1 value2}: remains unquoted (isn't quoted at all)</li>
+ * <li>{@code key=value1 "value2"}: remains quoted (has quotes in between)</li>
+ * <li>{@code key="value1 value2" "value3 value4"}: remains quoted (has quotes both ends, but has also some in between)</li>
+ * <li>{@code key="value1 value2 value3 value4"}: gets unquoted (has quotes both ends, and no quotes in between)</li>
+ * </ul>
+ * @param maybeEntirelyQuotedValue a {@code String} that is a parameter value but not necessarily quoted
+ * @return an unquoted version of the input {@String}, when {@code maybeEntirelyQuotedValue} had quotes at both ends, and didn't
+ * have any quotes in between. Else {@code maybeEntirelyQuotedValue}
+ */
+ private static String unquoteEntirelyQuotedValue(final String maybeEntirelyQuotedValue) {
+ final boolean hasQuotesAtEnds = maybeEntirelyQuotedValue.matches(VALUE_HAS_QUOTES_AT_ENDS_REGEX);
+ final boolean hasQuotesInBetween = maybeEntirelyQuotedValue.matches(VALUE_HAS_QUOTES_IN_BETWEEN_REGEX);
+ final boolean isEntirelyQuoted = hasQuotesAtEnds && !hasQuotesInBetween;
+
+ if (isEntirelyQuoted) {
+ return maybeEntirelyQuotedValue.replaceAll("\"", "");
+ }
+
+ return maybeEntirelyQuotedValue;
+ }
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/0f293eaa/sharelib/spark/src/test/java/org/apache/oozie/action/hadoop/TestSparkArgsExtractor.java
----------------------------------------------------------------------
diff --git a/sharelib/spark/src/test/java/org/apache/oozie/action/hadoop/TestSparkArgsExtractor.java b/sharelib/spark/src/test/java/org/apache/oozie/action/hadoop/TestSparkArgsExtractor.java
index 7db26a6..d0541ca 100644
--- a/sharelib/spark/src/test/java/org/apache/oozie/action/hadoop/TestSparkArgsExtractor.java
+++ b/sharelib/spark/src/test/java/org/apache/oozie/action/hadoop/TestSparkArgsExtractor.java
@@ -194,4 +194,41 @@ public class TestSparkArgsExtractor {
"arg1"),
sparkArgs);
}
+
+ @Test
+ public void testQuotedDriverAndExecutorExtraJavaOptionsParsing() throws Exception {
+ final Configuration actionConf = new Configuration();
+ actionConf.set(SparkActionExecutor.SPARK_MASTER, "yarn");
+ actionConf.set(SparkActionExecutor.SPARK_MODE, "client");
+ actionConf.set(SparkActionExecutor.SPARK_CLASS, "org.apache.oozie.example.SparkFileCopy");
+ actionConf.set(SparkActionExecutor.SPARK_JOB_NAME, "Spark Copy File");
+ actionConf.set(SparkActionExecutor.SPARK_OPTS,
+ "--conf spark.executor.extraJavaOptions=\"-XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp\"" +
+ "--conf spark.driver.extraJavaOptions=\"-Xmn2703m -XX:SurvivorRatio=2 -XX:ParallelGCThreads=20\"");
+ actionConf.set(SparkActionExecutor.SPARK_JAR, "/lib/test.jar");
+
+ final String[] mainArgs = {"arg0", "arg1"};
+ final List<String> sparkArgs = new SparkArgsExtractor(actionConf).extract(mainArgs);
+
+ assertEquals("Spark args mismatch",
+ Lists.newArrayList("--master", "yarn",
+ "--deploy-mode", "client",
+ "--name", "Spark Copy File",
+ "--class", "org.apache.oozie.example.SparkFileCopy",
+ "--conf", "spark.executor.extraJavaOptions=-XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp " +
+ "-Dlog4j.configuration=spark-log4j.properties",
+ "--conf", "spark.driver.extraJavaOptions=-Xmn2703m -XX:SurvivorRatio=2 -XX:ParallelGCThreads=20 " +
+ "-Dlog4j.configuration=spark-log4j.properties",
+ "--conf", "spark.executor.extraClassPath=$PWD/*",
+ "--conf", "spark.driver.extraClassPath=$PWD/*",
+ "--conf", "spark.yarn.security.tokens.hive.enabled=false",
+ "--conf", "spark.yarn.security.tokens.hbase.enabled=false",
+ "--files", "spark-log4j.properties,hive-site.xml",
+ "--conf", "spark.yarn.jar=null",
+ "--verbose",
+ "/lib/test.jar",
+ "arg0",
+ "arg1"),
+ sparkArgs);
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/oozie/blob/0f293eaa/sharelib/spark/src/test/java/org/apache/oozie/action/hadoop/TestSparkOptionsSplitter.java
----------------------------------------------------------------------
diff --git a/sharelib/spark/src/test/java/org/apache/oozie/action/hadoop/TestSparkOptionsSplitter.java b/sharelib/spark/src/test/java/org/apache/oozie/action/hadoop/TestSparkOptionsSplitter.java
index 02786a2..4be9777 100644
--- a/sharelib/spark/src/test/java/org/apache/oozie/action/hadoop/TestSparkOptionsSplitter.java
+++ b/sharelib/spark/src/test/java/org/apache/oozie/action/hadoop/TestSparkOptionsSplitter.java
@@ -34,13 +34,35 @@ public class TestSparkOptionsSplitter {
@Parameterized.Parameters
public static List<Object[]> params() {
return Arrays.asList(new Object[][]{
- {"--option1 value1", Arrays.asList("--option1", "value1")},
- {"--option1 value1", Arrays.asList("--option1", "value1")},
- {" --option1 value1 ", Arrays.asList("--option1", "value1")},
- {"--conf special=value1", Arrays.asList("--conf", "special=value1")},
- {"--conf special=\"value1\"", Arrays.asList("--conf", "special=value1")},
- {"--conf special=\"value1 value2\"", Arrays.asList("--conf", "special=value1 value2")},
- {" --conf special=\"value1 value2\" ", Arrays.asList("--conf", "special=value1 value2")},
+ {"--option1 value1",
+ Arrays.asList("--option1", "value1")},
+ {"--option1 value1",
+ Arrays.asList("--option1", "value1")},
+ {" --option1 value1 ",
+ Arrays.asList("--option1", "value1")},
+ {"--conf special=value1",
+ Arrays.asList("--conf", "special=value1")},
+ {"--conf special=\"value1\"",
+ Arrays.asList("--conf", "special=value1")},
+ {"--conf special=\"value1 value2\"",
+ Arrays.asList("--conf", "special=value1 value2")},
+ {" --conf special=\"value1 value2\" ",
+ Arrays.asList("--conf", "special=value1 value2")},
+ {"--conf special=value1 value2 --conf value3",
+ Arrays.asList("--conf", "special=value1 value2", "--conf", "value3")},
+ {"--conf special1=value1 special2=\"value2 value3\" --conf value4",
+ Arrays.asList("--conf", "special1=value1 special2=\"value2 value3\"", "--conf", "value4")},
+ {"--conf special1=value1 value2 special2=value3 value4",
+ Arrays.asList("--conf", "special1=value1 value2 special2=value3 value4")},
+ {"--option1 value1 --option2=value2",
+ Arrays.asList("--option1", "value1", "--option2", "value2")},
+ {"--conf spark.executor.extraJavaOptions=\"-XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp\"",
+ Arrays.asList("--conf",
+ "spark.executor.extraJavaOptions=-XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp")},
+ {"--option1 value1 --verbose",
+ Arrays.asList("--option1", "value1", "--verbose")},
+ {"--verbose --option1 value1",
+ Arrays.asList("--verbose", "--option1", "value1")}
});
}
[16/31] oozie git commit: OOZIE-2799 Setting log location for spark
sql on hive (satishsaley)
Posted by sa...@apache.org.
OOZIE-2799 Setting log location for spark sql on hive (satishsaley)
(cherry picked from commit 77817e73588f665cbe43b0971cd93465d40711fb)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/4ed56b6d
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/4ed56b6d
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/4ed56b6d
Branch: refs/heads/branch-4.3
Commit: 4ed56b6d40f9f0f2686f63afbf3ed97a0ae40453
Parents: 22aae95
Author: Satish Subhashrao Saley <sa...@yahoo-inc.com>
Authored: Wed Feb 15 09:18:47 2017 -0800
Committer: satishsaley <sa...@apache.org>
Committed: Fri Dec 8 16:34:55 2017 -0800
----------------------------------------------------------------------
.../action/hadoop/SparkActionExecutor.java | 6 +++
release-log.txt | 1 +
.../apache/oozie/action/hadoop/SparkMain.java | 47 ++++++++++++++++++++
3 files changed, 54 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/4ed56b6d/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java
index 8e8d7d3..e2e023e 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java
@@ -160,4 +160,10 @@ public class SparkActionExecutor extends JavaActionExecutor {
protected String getLauncherMain(Configuration launcherConf, Element actionXml) {
return launcherConf.get(LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS, SPARK_MAIN_CLASS_NAME);
}
+
+ @Override
+ public String[] getShareLibFilesForActionConf() {
+ return new String[] { "hive-site.xml" };
+ }
+
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/4ed56b6d/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index b816b95..db89107 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.3.1 release
+OOZIE-2799 Setting log location for spark sql on hive (satishsaley)
OOZIE-2792 Hive2 action is not parsing Spark application ID from log file properly when Hive is on Spark (zhengxb2005 via rkanter)
OOZIE-2786 Pass Oozie workflow ID and settings to Spark application configuration (zhengxb2005 via rkanter)
OOZIE-2790 log4j configuration is not passed to spark executors (satishsaley)
http://git-wip-us.apache.org/repos/asf/oozie/blob/4ed56b6d/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
----------------------------------------------------------------------
diff --git a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
index 3cc6ccb..db1e197 100644
--- a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
+++ b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
@@ -22,6 +22,7 @@ import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
+import java.lang.reflect.Field;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
@@ -41,6 +42,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.filecache.DistributedCache;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.log4j.PropertyConfigurator;
import org.apache.spark.deploy.SparkSubmit;
@@ -76,6 +78,8 @@ public class SparkMain extends LauncherMain {
private static final Pattern SPARK_VERSION_1 = Pattern.compile("^1.*");
private static final String SPARK_YARN_JAR = "spark.yarn.jar";
private static final String SPARK_YARN_JARS = "spark.yarn.jars";
+ public static final String HIVE_SITE_CONF = "hive-site.xml";
+
public static void main(String[] args) throws Exception {
run(SparkMain.class, args);
}
@@ -89,6 +93,7 @@ public class SparkMain extends LauncherMain {
setYarnTag(actionConf);
LauncherMainHadoopUtils.killChildYarnJobs(actionConf);
String logFile = setUpSparkLog4J(actionConf);
+ setHiveSite(actionConf);
List<String> sparkArgs = new ArrayList<String>();
sparkArgs.add(MASTER_OPTION);
@@ -225,6 +230,7 @@ public class SparkMain extends LauncherMain {
jarfilter.filter();
jarPath = jarfilter.getApplicationJar();
fixedUris.add(new Path(SPARK_LOG4J_PROPS).toUri());
+ fixedUris.add(new Path(HIVE_SITE_CONF).toUri());
String cachedFiles = StringUtils.join(fixedUris, ",");
if (cachedFiles != null && !cachedFiles.isEmpty()) {
sparkArgs.add("--files");
@@ -519,6 +525,47 @@ public class SparkMain extends LauncherMain {
}
/**
+ * Sets up hive-site.xml
+ *
+ * @param hiveConf
+ * @throws IOException
+ */
+ private void setHiveSite(Configuration hiveConf) throws IOException {
+ // See https://issues.apache.org/jira/browse/HIVE-1411
+ hiveConf.set("datanucleus.plugin.pluginRegistryBundleCheck", "LOG");
+
+ // To ensure that the logs go into container attempt tmp directory
+ // When unset, default is
+ // System.getProperty("java.io.tmpdir") + File.separator +
+ // System.getProperty("user.name")
+ hiveConf.unset("hive.querylog.location");
+ hiveConf.unset("hive.exec.local.scratchdir");
+
+ // Write the action configuration out to hive-site.xml
+ OutputStream os = null;
+ try {
+ os = new FileOutputStream(HIVE_SITE_CONF);
+ hiveConf.writeXml(os);
+ }
+ finally {
+ if (os != null) {
+ os.close();
+ }
+ }
+ // Reset the hiveSiteURL static variable as we just created
+ // hive-site.xml.
+ // If prepare block had a drop partition it would have been initialized
+ // to null.
+ try {
+ Field declaredField = HiveConf.class.getDeclaredField("hiveSiteURL");
+ declaredField.setAccessible(true);
+ declaredField.set(null, HiveConf.class.getClassLoader().getResource("hive-site.xml"));
+ }
+ catch (Throwable ignore) {
+ }
+ }
+
+ /**
* This class is used for filtering out unwanted jars.
*/
static class JarFilter {
[28/31] oozie git commit: OOZIE-2923 Improve Spark options parsing
(andras.piros via gezapeti)
Posted by sa...@apache.org.
OOZIE-2923 Improve Spark options parsing (andras.piros via gezapeti)
(cherry picked from commit f45b18791a92201fb4959857cafbda7543942022)
sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/e284e63c
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/e284e63c
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/e284e63c
Branch: refs/heads/branch-4.3
Commit: e284e63cd260b07778369292f0d052f1b73cb521
Parents: bbac99c
Author: Gezapeti Cseh <ge...@gmail.com>
Authored: Thu Jun 8 13:59:06 2017 +0200
Committer: satishsaley <sa...@apache.org>
Committed: Fri Dec 8 16:34:55 2017 -0800
----------------------------------------------------------------------
release-log.txt | 1 +
.../oozie/action/hadoop/HadoopUriFinder.java | 66 +++
.../apache/oozie/action/hadoop/JarFilter.java | 122 +++++
.../oozie/action/hadoop/SparkArgsExtractor.java | 333 ++++++++++++
.../apache/oozie/action/hadoop/SparkMain.java | 520 +++----------------
.../action/hadoop/SparkOptionsSplitter.java | 64 +++
.../oozie/action/hadoop/TestJarFilter.java | 1 -
.../action/hadoop/TestSparkArgsExtractor.java | 197 +++++++
.../oozie/action/hadoop/TestSparkMain.java | 13 -
.../action/hadoop/TestSparkOptionsSplitter.java | 16 +-
10 files changed, 850 insertions(+), 483 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/e284e63c/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 9834dfe..8af2ee2 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.3.1 release
+OOZIE-2923 Improve Spark options parsing (andras.piros via gezapeti)
OOZIE-2886 Ensure consistent versioning of hadoop jars in sharelibs (dbist13 via rkanter)
OOZIE-2872 Address backward compatibility issue introduced by OOZIE-2748 (pbacsko)
OOZIE-2863 SLACalculatorMemory.loadOnRestart causing delay in server start (puru via satishsaley)
http://git-wip-us.apache.org/repos/asf/oozie/blob/e284e63c/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/HadoopUriFinder.java
----------------------------------------------------------------------
diff --git a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/HadoopUriFinder.java b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/HadoopUriFinder.java
new file mode 100644
index 0000000..3f9e982
--- /dev/null
+++ b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/HadoopUriFinder.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oozie.action.hadoop;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.jar.JarFile;
+import java.util.jar.Manifest;
+
+class HadoopUriFinder {
+
+ static String getJarVersion(final File jarFile) throws IOException {
+ try (final JarFile openedJarFile = new JarFile(jarFile)) {
+ final Manifest manifest = openedJarFile.getManifest();
+ return manifest.getMainAttributes().getValue("Specification-Version");
+ }
+ }
+
+ static URI getFixedUri(final URI fileUri) throws URISyntaxException, IOException {
+ final FileSystem fs = FileSystem.get(new Configuration(true));
+ return getFixedUri(fs, fileUri);
+ }
+
+ /**
+ * Spark compares URIs based on scheme, host and port. Here we convert URIs
+ * into the default format so that Spark won't think those belong to
+ * different file system. This will avoid an extra copy of files which
+ * already exists on same hdfs.
+ *
+ * @param fs
+ * @param fileUri
+ * @return fixed uri
+ * @throws URISyntaxException
+ */
+ static URI getFixedUri(final FileSystem fs, final URI fileUri) throws URISyntaxException {
+ if (fs.getUri().getScheme().equals(fileUri.getScheme())
+ && (fs.getUri().getHost().equals(fileUri.getHost()) || fileUri.getHost() == null)
+ && (fs.getUri().getPort() == -1 || fileUri.getPort() == -1
+ || fs.getUri().getPort() == fileUri.getPort())) {
+ return new URI(fs.getUri().getScheme(), fileUri.getUserInfo(), fs.getUri().getHost(), fs.getUri().getPort(),
+ fileUri.getPath(), fileUri.getQuery(), fileUri.getFragment());
+ }
+ return fileUri;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/oozie/blob/e284e63c/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/JarFilter.java
----------------------------------------------------------------------
diff --git a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/JarFilter.java b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/JarFilter.java
new file mode 100644
index 0000000..d0b4b5e
--- /dev/null
+++ b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/JarFilter.java
@@ -0,0 +1,122 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oozie.action.hadoop;
+
+import org.apache.hadoop.fs.Path;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.Collection;
+import java.util.Iterator;
+
+/**
+ * This class is used for filtering out unwanted jars.
+ */
+class JarFilter {
+ private String sparkVersion = "1.X.X";
+ private String sparkYarnJar;
+ private String applicationJar;
+ private Collection<URI> listUris = null;
+
+ /**
+ * @param listUris List of URIs to be filtered
+ * @param jarPath Application jar
+ * @throws IOException
+ * @throws URISyntaxException
+ */
+ JarFilter(final Collection<URI> listUris, final String jarPath) throws URISyntaxException, IOException {
+ this.listUris = listUris;
+ applicationJar = jarPath;
+ final Path p = new Path(jarPath);
+ if (p.isAbsolute()) {
+ applicationJar = HadoopUriFinder.getFixedUri(p.toUri()).toString();
+ }
+ }
+
+ /**
+ * Filters out the Spark yarn jar and application jar. Also records
+ * spark yarn jar's version.
+ *
+ * @throws OozieActionConfiguratorException
+ */
+ void filter() throws OozieActionConfiguratorException {
+ final Iterator<URI> iterator = listUris.iterator();
+ File matchedFile = null;
+ final Path applJarPath = new Path(applicationJar);
+ while (iterator.hasNext()) {
+ final URI uri = iterator.next();
+ final Path p = new Path(uri);
+ if (SparkMain.SPARK_YARN_JAR_PATTERN.matcher(p.getName()).find()) {
+ matchedFile = SparkMain.getMatchingFile(SparkMain.SPARK_YARN_JAR_PATTERN);
+ }
+ else if (SparkMain.SPARK_ASSEMBLY_JAR_PATTERN.matcher(p.getName()).find()) {
+ matchedFile = SparkMain.getMatchingFile(SparkMain.SPARK_ASSEMBLY_JAR_PATTERN);
+ }
+ if (matchedFile != null) {
+ sparkYarnJar = uri.toString();
+ try {
+ sparkVersion = HadoopUriFinder.getJarVersion(matchedFile);
+ System.out.println("Spark Version " + sparkVersion);
+ }
+ catch (final IOException io) {
+ System.out.println(
+ "Unable to open " + matchedFile.getPath() + ". Default Spark Version " + sparkVersion);
+ }
+ iterator.remove();
+ matchedFile = null;
+ }
+ // Here we skip the application jar, because
+ // (if uris are same,) it will get distributed multiple times
+ // - one time with --files and another time as application jar.
+ if (isApplicationJar(p.getName(), uri, applJarPath)) {
+ final String fragment = uri.getFragment();
+ applicationJar = fragment != null && fragment.length() > 0 ? fragment : uri.toString();
+ iterator.remove();
+ }
+ }
+ }
+
+ /**
+ * Checks if a file is application jar
+ *
+ * @param fileName fileName name of the file
+ * @param fileUri fileUri URI of the file
+ * @param applJarPath Path of application jar
+ * @return true if fileName or fileUri is the application jar
+ */
+ private boolean isApplicationJar(final String fileName, final URI fileUri, final Path applJarPath) {
+ return (fileName.equals(applicationJar) || fileUri.toString().equals(applicationJar)
+ || applJarPath.getName().equals(fileName)
+ || applicationJar.equals(fileUri.getFragment()));
+ }
+
+ String getApplicationJar() {
+ return applicationJar;
+ }
+
+ String getSparkYarnJar() {
+ return sparkYarnJar;
+ }
+
+ String getSparkVersion() {
+ return sparkVersion;
+ }
+}
http://git-wip-us.apache.org/repos/asf/oozie/blob/e284e63c/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkArgsExtractor.java
----------------------------------------------------------------------
diff --git a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkArgsExtractor.java b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkArgsExtractor.java
new file mode 100644
index 0000000..3270eea
--- /dev/null
+++ b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkArgsExtractor.java
@@ -0,0 +1,333 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oozie.action.hadoop;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.filecache.DistributedCache;
+import org.apache.hadoop.fs.Path;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Pattern;
+
+class SparkArgsExtractor {
+ private static final Pattern SPARK_DEFAULTS_FILE_PATTERN = Pattern.compile("spark-defaults.conf");
+ private static final String FILES_OPTION = "--files";
+ private static final String ARCHIVES_OPTION = "--archives";
+ private static final String LOG4J_CONFIGURATION_JAVA_OPTION = "-Dlog4j.configuration=";
+ private static final String HIVE_SECURITY_TOKEN = "spark.yarn.security.tokens.hive.enabled";
+ private static final String HBASE_SECURITY_TOKEN = "spark.yarn.security.tokens.hbase.enabled";
+ private static final String PWD = "$PWD" + File.separator + "*";
+ private static final String MASTER_OPTION = "--master";
+ private static final String MODE_OPTION = "--deploy-mode";
+ private static final String JOB_NAME_OPTION = "--name";
+ private static final String CLASS_NAME_OPTION = "--class";
+ private static final String VERBOSE_OPTION = "--verbose";
+ private static final String DRIVER_CLASSPATH_OPTION = "--driver-class-path";
+ private static final String EXECUTOR_CLASSPATH = "spark.executor.extraClassPath=";
+ private static final String DRIVER_CLASSPATH = "spark.driver.extraClassPath=";
+ private static final String EXECUTOR_EXTRA_JAVA_OPTIONS = "spark.executor.extraJavaOptions=";
+ private static final String DRIVER_EXTRA_JAVA_OPTIONS = "spark.driver.extraJavaOptions=";
+ private static final Pattern SPARK_VERSION_1 = Pattern.compile("^1.*");
+ private static final String SPARK_YARN_JAR = "spark.yarn.jar";
+ private static final String SPARK_YARN_JARS = "spark.yarn.jars";
+ private static final String OPT_SEPARATOR = "=";
+ private static final String OPT_VALUE_SEPARATOR = ",";
+ private static final String CONF_OPTION = "--conf";
+ private static final String MASTER_OPTION_YARN_CLUSTER = "yarn-cluster";
+ private static final String MASTER_OPTION_YARN_CLIENT = "yarn-client";
+ private static final String MASTER_OPTION_YARN = "yarn";
+ private static final String DEPLOY_MODE_CLUSTER = "cluster";
+ private static final String DEPLOY_MODE_CLIENT = "client";
+ private static final String SPARK_YARN_TAGS = "spark.yarn.tags";
+ private static final String OPT_PROPERTIES_FILE = "--properties-file";
+
+ private boolean pySpark = false;
+ private final Configuration actionConf;
+
+ SparkArgsExtractor(final Configuration actionConf) {
+ this.actionConf = actionConf;
+ }
+
+ boolean isPySpark() {
+ return pySpark;
+ }
+
+ List<String> extract(final String[] mainArgs) throws OozieActionConfiguratorException, IOException, URISyntaxException {
+ final List<String> sparkArgs = new ArrayList<>();
+
+ sparkArgs.add(MASTER_OPTION);
+ final String master = actionConf.get(SparkActionExecutor.SPARK_MASTER);
+ sparkArgs.add(master);
+
+ // In local mode, everything runs here in the Launcher Job.
+ // In yarn-client mode, the driver runs here in the Launcher Job and the
+ // executor in Yarn.
+ // In yarn-cluster mode, the driver and executor run in Yarn.
+ final String sparkDeployMode = actionConf.get(SparkActionExecutor.SPARK_MODE);
+ if (sparkDeployMode != null) {
+ sparkArgs.add(MODE_OPTION);
+ sparkArgs.add(sparkDeployMode);
+ }
+ final boolean yarnClusterMode = master.equals(MASTER_OPTION_YARN_CLUSTER)
+ || (master.equals(MASTER_OPTION_YARN) && sparkDeployMode != null && sparkDeployMode.equals(DEPLOY_MODE_CLUSTER));
+ final boolean yarnClientMode = master.equals(MASTER_OPTION_YARN_CLIENT)
+ || (master.equals(MASTER_OPTION_YARN) && sparkDeployMode != null && sparkDeployMode.equals(DEPLOY_MODE_CLIENT));
+
+ sparkArgs.add(JOB_NAME_OPTION);
+ sparkArgs.add(actionConf.get(SparkActionExecutor.SPARK_JOB_NAME));
+
+ final String className = actionConf.get(SparkActionExecutor.SPARK_CLASS);
+ if (className != null) {
+ sparkArgs.add(CLASS_NAME_OPTION);
+ sparkArgs.add(className);
+ }
+
+ appendOoziePropertiesToSparkConf(sparkArgs);
+
+ String jarPath = actionConf.get(SparkActionExecutor.SPARK_JAR);
+ if (jarPath != null && jarPath.endsWith(".py")) {
+ pySpark = true;
+ }
+ boolean addedHiveSecurityToken = false;
+ boolean addedHBaseSecurityToken = false;
+ boolean addedLog4jDriverSettings = false;
+ boolean addedLog4jExecutorSettings = false;
+ final StringBuilder driverClassPath = new StringBuilder();
+ final StringBuilder executorClassPath = new StringBuilder();
+ final StringBuilder userFiles = new StringBuilder();
+ final StringBuilder userArchives = new StringBuilder();
+ final String sparkOpts = actionConf.get(SparkActionExecutor.SPARK_OPTS);
+ if (StringUtils.isNotEmpty(sparkOpts)) {
+ final List<String> sparkOptions = SparkOptionsSplitter.splitSparkOpts(sparkOpts);
+ for (int i = 0; i < sparkOptions.size(); i++) {
+ String opt = sparkOptions.get(i);
+ boolean addToSparkArgs = true;
+ if (yarnClusterMode || yarnClientMode) {
+ if (opt.startsWith(EXECUTOR_CLASSPATH)) {
+ appendWithPathSeparator(opt.substring(EXECUTOR_CLASSPATH.length()), executorClassPath);
+ addToSparkArgs = false;
+ }
+ if (opt.startsWith(DRIVER_CLASSPATH)) {
+ appendWithPathSeparator(opt.substring(DRIVER_CLASSPATH.length()), driverClassPath);
+ addToSparkArgs = false;
+ }
+ if (opt.equals(DRIVER_CLASSPATH_OPTION)) {
+ // we need the next element after this option
+ appendWithPathSeparator(sparkOptions.get(i + 1), driverClassPath);
+ // increase i to skip the next element.
+ i++;
+ addToSparkArgs = false;
+ }
+ }
+ if (opt.startsWith(HIVE_SECURITY_TOKEN)) {
+ addedHiveSecurityToken = true;
+ }
+ if (opt.startsWith(HBASE_SECURITY_TOKEN)) {
+ addedHBaseSecurityToken = true;
+ }
+ if (opt.startsWith(EXECUTOR_EXTRA_JAVA_OPTIONS) || opt.startsWith(DRIVER_EXTRA_JAVA_OPTIONS)) {
+ if (!opt.contains(LOG4J_CONFIGURATION_JAVA_OPTION)) {
+ opt += " " + LOG4J_CONFIGURATION_JAVA_OPTION + SparkMain.SPARK_LOG4J_PROPS;
+ } else {
+ System.out.println("Warning: Spark Log4J settings are overwritten." +
+ " Child job IDs may not be available");
+ }
+ if (opt.startsWith(EXECUTOR_EXTRA_JAVA_OPTIONS)) {
+ addedLog4jExecutorSettings = true;
+ } else {
+ addedLog4jDriverSettings = true;
+ }
+ }
+ if (opt.startsWith(FILES_OPTION)) {
+ final String userFile;
+ if (opt.contains(OPT_SEPARATOR)) {
+ userFile = opt.substring(opt.indexOf(OPT_SEPARATOR) + OPT_SEPARATOR.length());
+ }
+ else {
+ userFile = sparkOptions.get(i + 1);
+ i++;
+ }
+ if (userFiles.length() > 0) {
+ userFiles.append(OPT_VALUE_SEPARATOR);
+ }
+ userFiles.append(userFile);
+ addToSparkArgs = false;
+ }
+ if (opt.startsWith(ARCHIVES_OPTION)) {
+ final String userArchive;
+ if (opt.contains(OPT_SEPARATOR)) {
+ userArchive = opt.substring(opt.indexOf(OPT_SEPARATOR) + OPT_SEPARATOR.length());
+ }
+ else {
+ userArchive = sparkOptions.get(i + 1);
+ i++;
+ }
+ if (userArchives.length() > 0) {
+ userArchives.append(OPT_VALUE_SEPARATOR);
+ }
+ userArchives.append(userArchive);
+ addToSparkArgs = false;
+ }
+ if (addToSparkArgs) {
+ sparkArgs.add(opt);
+ }
+ else if (sparkArgs.get(sparkArgs.size() - 1).equals(CONF_OPTION)) {
+ sparkArgs.remove(sparkArgs.size() - 1);
+ }
+ }
+ }
+
+ if ((yarnClusterMode || yarnClientMode)) {
+ // Include the current working directory (of executor container)
+ // in executor classpath, because it will contain localized
+ // files
+ appendWithPathSeparator(PWD, executorClassPath);
+ appendWithPathSeparator(PWD, driverClassPath);
+
+ sparkArgs.add(CONF_OPTION);
+ sparkArgs.add(EXECUTOR_CLASSPATH + executorClassPath.toString());
+
+ sparkArgs.add(CONF_OPTION);
+ sparkArgs.add(DRIVER_CLASSPATH + driverClassPath.toString());
+ }
+
+ if (actionConf.get(LauncherMain.MAPREDUCE_JOB_TAGS) != null) {
+ sparkArgs.add(CONF_OPTION);
+ sparkArgs.add(SPARK_YARN_TAGS + OPT_SEPARATOR + actionConf.get(LauncherMain.MAPREDUCE_JOB_TAGS));
+ }
+
+ if (!addedHiveSecurityToken) {
+ sparkArgs.add(CONF_OPTION);
+ sparkArgs.add(HIVE_SECURITY_TOKEN + OPT_SEPARATOR + Boolean.toString(false));
+ }
+ if (!addedHBaseSecurityToken) {
+ sparkArgs.add(CONF_OPTION);
+ sparkArgs.add(HBASE_SECURITY_TOKEN + OPT_SEPARATOR + Boolean.toString(false));
+ }
+ if (!addedLog4jExecutorSettings) {
+ sparkArgs.add(CONF_OPTION);
+ sparkArgs.add(EXECUTOR_EXTRA_JAVA_OPTIONS + LOG4J_CONFIGURATION_JAVA_OPTION + SparkMain.SPARK_LOG4J_PROPS);
+ }
+ if (!addedLog4jDriverSettings) {
+ sparkArgs.add(CONF_OPTION);
+ sparkArgs.add(DRIVER_EXTRA_JAVA_OPTIONS + LOG4J_CONFIGURATION_JAVA_OPTION + SparkMain.SPARK_LOG4J_PROPS);
+ }
+ final File defaultConfFile = SparkMain.getMatchingFile(SPARK_DEFAULTS_FILE_PATTERN);
+ if (defaultConfFile != null) {
+ sparkArgs.add(OPT_PROPERTIES_FILE);
+ sparkArgs.add(SPARK_DEFAULTS_FILE_PATTERN.toString());
+ }
+
+ if ((yarnClusterMode || yarnClientMode)) {
+ final Map<String, URI> fixedFileUrisMap =
+ SparkMain.fixFsDefaultUrisAndFilterDuplicates(DistributedCache.getCacheFiles(actionConf));
+ fixedFileUrisMap.put(SparkMain.SPARK_LOG4J_PROPS, new Path(SparkMain.SPARK_LOG4J_PROPS).toUri());
+ fixedFileUrisMap.put(SparkMain.HIVE_SITE_CONF, new Path(SparkMain.HIVE_SITE_CONF).toUri());
+ addUserDefined(userFiles.toString(), fixedFileUrisMap);
+ final Collection<URI> fixedFileUris = fixedFileUrisMap.values();
+ final JarFilter jarFilter = new JarFilter(fixedFileUris, jarPath);
+ jarFilter.filter();
+ jarPath = jarFilter.getApplicationJar();
+
+ final String cachedFiles = StringUtils.join(fixedFileUris, OPT_VALUE_SEPARATOR);
+ if (cachedFiles != null && !cachedFiles.isEmpty()) {
+ sparkArgs.add(FILES_OPTION);
+ sparkArgs.add(cachedFiles);
+ }
+ final Map<String, URI> fixedArchiveUrisMap = SparkMain.fixFsDefaultUrisAndFilterDuplicates(DistributedCache.
+ getCacheArchives(actionConf));
+ addUserDefined(userArchives.toString(), fixedArchiveUrisMap);
+ final String cachedArchives = StringUtils.join(fixedArchiveUrisMap.values(), OPT_VALUE_SEPARATOR);
+ if (cachedArchives != null && !cachedArchives.isEmpty()) {
+ sparkArgs.add(ARCHIVES_OPTION);
+ sparkArgs.add(cachedArchives);
+ }
+ setSparkYarnJarsConf(sparkArgs, jarFilter.getSparkYarnJar(), jarFilter.getSparkVersion());
+ }
+
+ if (!sparkArgs.contains(VERBOSE_OPTION)) {
+ sparkArgs.add(VERBOSE_OPTION);
+ }
+
+ sparkArgs.add(jarPath);
+ sparkArgs.addAll(Arrays.asList(mainArgs));
+
+ return sparkArgs;
+ }
+
+ private void appendWithPathSeparator(final String what, final StringBuilder to) {
+ if (to.length() > 0) {
+ to.append(File.pathSeparator);
+ }
+ to.append(what);
+ }
+
+ private void addUserDefined(final String userList, final Map<String, URI> urisMap) {
+ if (userList != null) {
+ for (final String file : userList.split(OPT_VALUE_SEPARATOR)) {
+ if (!StringUtils.isEmpty(file)) {
+ final Path p = new Path(file);
+ urisMap.put(p.getName(), p.toUri());
+ }
+ }
+ }
+ }
+
+ /*
+ * Get properties that needs to be passed to Spark as Spark configuration from actionConf.
+ */
+ @VisibleForTesting
+ void appendOoziePropertiesToSparkConf(final List<String> sparkArgs) {
+ for (final Map.Entry<String, String> oozieConfig : actionConf
+ .getValByRegex("^oozie\\.(?!launcher|spark).+").entrySet()) {
+ sparkArgs.add(CONF_OPTION);
+ sparkArgs.add(String.format("spark.%s=%s", oozieConfig.getKey(), oozieConfig.getValue()));
+ }
+ }
+
+ /**
+ * Sets spark.yarn.jars for Spark 2.X. Sets spark.yarn.jar for Spark 1.X.
+ *
+ * @param sparkArgs
+ * @param sparkYarnJar
+ * @param sparkVersion
+ */
+ private void setSparkYarnJarsConf(final List<String> sparkArgs, final String sparkYarnJar, final String sparkVersion) {
+ if (SPARK_VERSION_1.matcher(sparkVersion).find()) {
+ // In Spark 1.X.X, set spark.yarn.jar to avoid
+ // multiple distribution
+ sparkArgs.add(CONF_OPTION);
+ sparkArgs.add(SPARK_YARN_JAR + OPT_SEPARATOR + sparkYarnJar);
+ } else {
+ // In Spark 2.X.X, set spark.yarn.jars
+ sparkArgs.add(CONF_OPTION);
+ sparkArgs.add(SPARK_YARN_JARS + OPT_SEPARATOR + sparkYarnJar);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/oozie/blob/e284e63c/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
----------------------------------------------------------------------
diff --git a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
index c24d95c..3c373e3 100644
--- a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
+++ b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
@@ -26,21 +26,14 @@ import java.lang.reflect.Field;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Properties;
-import java.util.jar.JarFile;
-import java.util.jar.Manifest;
+import java.util.LinkedHashMap;
import java.util.regex.Pattern;
import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.filecache.DistributedCache;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
@@ -50,243 +43,56 @@ import org.apache.spark.deploy.SparkSubmit;
import com.google.common.annotations.VisibleForTesting;
public class SparkMain extends LauncherMain {
- private static final String MASTER_OPTION = "--master";
- private static final String MODE_OPTION = "--deploy-mode";
- private static final String JOB_NAME_OPTION = "--name";
- private static final String CLASS_NAME_OPTION = "--class";
- private static final String VERBOSE_OPTION = "--verbose";
- private static final String DRIVER_CLASSPATH_OPTION = "--driver-class-path";
- private static final String EXECUTOR_CLASSPATH = "spark.executor.extraClassPath=";
- private static final String DRIVER_CLASSPATH = "spark.driver.extraClassPath=";
- private static final String EXECUTOR_EXTRA_JAVA_OPTIONS = "spark.executor.extraJavaOptions=";
- private static final String DRIVER_EXTRA_JAVA_OPTIONS = "spark.driver.extraJavaOptions=";
- private static final String LOG4J_CONFIGURATION_JAVA_OPTION = "-Dlog4j.configuration=";
- private static final String HIVE_SECURITY_TOKEN = "spark.yarn.security.tokens.hive.enabled";
- private static final String HBASE_SECURITY_TOKEN = "spark.yarn.security.tokens.hbase.enabled";
- private static final String CONF_OOZIE_SPARK_SETUP_HADOOP_CONF_DIR = "oozie.action.spark.setup.hadoop.conf.dir";
- private static final String PWD = "$PWD" + File.separator + "*";
- private static final Pattern[] PYSPARK_DEP_FILE_PATTERN = { Pattern.compile("py4\\S*src.zip"),
- Pattern.compile("pyspark.zip") };
- private static final Pattern SPARK_DEFAULTS_FILE_PATTERN = Pattern.compile("spark-defaults.conf");
- private static final String SPARK_LOG4J_PROPS = "spark-log4j.properties";
+
@VisibleForTesting
static final Pattern[] SPARK_JOB_IDS_PATTERNS = {
Pattern.compile("Submitted application (application[0-9_]*)") };
- public static final Pattern SPARK_ASSEMBLY_JAR_PATTERN = Pattern
+ @VisibleForTesting
+ static final Pattern SPARK_ASSEMBLY_JAR_PATTERN = Pattern
.compile("^spark-assembly((?:(-|_|(\\d+\\.))\\d+(?:\\.\\d+)*))*\\.jar$");
- public static final Pattern SPARK_YARN_JAR_PATTERN = Pattern
+ @VisibleForTesting
+ static final Pattern SPARK_YARN_JAR_PATTERN = Pattern
.compile("^spark-yarn((?:(-|_|(\\d+\\.))\\d+(?:\\.\\d+)*))*\\.jar$");
- private static final Pattern SPARK_VERSION_1 = Pattern.compile("^1.*");
- private static final String SPARK_YARN_JAR = "spark.yarn.jar";
- private static final String SPARK_YARN_JARS = "spark.yarn.jars";
- public static final String HIVE_SITE_CONF = "hive-site.xml";
- public static final String FILES_OPTION = "--files";
- public static final String ARCHIVES_OPTION = "--archives";
-
- public static void main(String[] args) throws Exception {
+ static final String HIVE_SITE_CONF = "hive-site.xml";
+ static final String SPARK_LOG4J_PROPS = "spark-log4j.properties";
+
+ private static final String CONF_OOZIE_SPARK_SETUP_HADOOP_CONF_DIR = "oozie.action.spark.setup.hadoop.conf.dir";
+ private static final Pattern[] PYSPARK_DEP_FILE_PATTERN = { Pattern.compile("py4\\S*src.zip"),
+ Pattern.compile("pyspark.zip") };
+
+ public static void main(final String[] args) throws Exception {
run(SparkMain.class, args);
}
@Override
- protected void run(String[] args) throws Exception {
- boolean isPyspark = false;
- Configuration actionConf = loadActionConf();
+ protected void run(final String[] args) throws Exception {
+ final Configuration actionConf = loadActionConf();
prepareHadoopConfig(actionConf);
setYarnTag(actionConf);
LauncherMainHadoopUtils.killChildYarnJobs(actionConf);
- String logFile = setUpSparkLog4J(actionConf);
+ final String logFile = setUpSparkLog4J(actionConf);
setHiveSite(actionConf);
- List<String> sparkArgs = new ArrayList<String>();
-
- sparkArgs.add(MASTER_OPTION);
- String master = actionConf.get(SparkActionExecutor.SPARK_MASTER);
- sparkArgs.add(master);
-
- // In local mode, everything runs here in the Launcher Job.
- // In yarn-client mode, the driver runs here in the Launcher Job and the
- // executor in Yarn.
- // In yarn-cluster mode, the driver and executor run in Yarn.
- String sparkDeployMode = actionConf.get(SparkActionExecutor.SPARK_MODE);
- if (sparkDeployMode != null) {
- sparkArgs.add(MODE_OPTION);
- sparkArgs.add(sparkDeployMode);
- }
- boolean yarnClusterMode = master.equals("yarn-cluster")
- || (master.equals("yarn") && sparkDeployMode != null && sparkDeployMode.equals("cluster"));
- boolean yarnClientMode = master.equals("yarn-client")
- || (master.equals("yarn") && sparkDeployMode != null && sparkDeployMode.equals("client"));
-
- sparkArgs.add(JOB_NAME_OPTION);
- sparkArgs.add(actionConf.get(SparkActionExecutor.SPARK_JOB_NAME));
-
- String className = actionConf.get(SparkActionExecutor.SPARK_CLASS);
- if (className != null) {
- sparkArgs.add(CLASS_NAME_OPTION);
- sparkArgs.add(className);
- }
-
- appendOoziePropertiesToSparkConf(sparkArgs, actionConf);
-
- String jarPath = actionConf.get(SparkActionExecutor.SPARK_JAR);
- if(jarPath!=null && jarPath.endsWith(".py")){
- isPyspark = true;
- }
- boolean addedHiveSecurityToken = false;
- boolean addedHBaseSecurityToken = false;
- boolean addedLog4jDriverSettings = false;
- boolean addedLog4jExecutorSettings = false;
- StringBuilder driverClassPath = new StringBuilder();
- StringBuilder executorClassPath = new StringBuilder();
- String userFiles = null, userArchives = null;
- String sparkOpts = actionConf.get(SparkActionExecutor.SPARK_OPTS);
- if (StringUtils.isNotEmpty(sparkOpts)) {
- List<String> sparkOptions = splitSparkOpts(sparkOpts);
- for (int i = 0; i < sparkOptions.size(); i++) {
- String opt = sparkOptions.get(i);
- boolean addToSparkArgs = true;
- if (yarnClusterMode || yarnClientMode) {
- if (opt.startsWith(EXECUTOR_CLASSPATH)) {
- appendWithPathSeparator(opt.substring(EXECUTOR_CLASSPATH.length()), executorClassPath);
- addToSparkArgs = false;
- }
- if (opt.startsWith(DRIVER_CLASSPATH)) {
- appendWithPathSeparator(opt.substring(DRIVER_CLASSPATH.length()), driverClassPath);
- addToSparkArgs = false;
- }
- if (opt.equals(DRIVER_CLASSPATH_OPTION)) {
- // we need the next element after this option
- appendWithPathSeparator(sparkOptions.get(i + 1), driverClassPath);
- // increase i to skip the next element.
- i++;
- addToSparkArgs = false;
- }
- }
- if (opt.startsWith(HIVE_SECURITY_TOKEN)) {
- addedHiveSecurityToken = true;
- }
- if (opt.startsWith(HBASE_SECURITY_TOKEN)) {
- addedHBaseSecurityToken = true;
- }
- if (opt.startsWith(EXECUTOR_EXTRA_JAVA_OPTIONS) || opt.startsWith(DRIVER_EXTRA_JAVA_OPTIONS)) {
- if(!opt.contains(LOG4J_CONFIGURATION_JAVA_OPTION)) {
- opt += " " + LOG4J_CONFIGURATION_JAVA_OPTION + SPARK_LOG4J_PROPS;
- }else{
- System.out.println("Warning: Spark Log4J settings are overwritten." +
- " Child job IDs may not be available");
- }
- if(opt.startsWith(EXECUTOR_EXTRA_JAVA_OPTIONS)) {
- addedLog4jExecutorSettings = true;
- }else{
- addedLog4jDriverSettings = true;
- }
- }
- if(opt.startsWith(FILES_OPTION)) {
- userFiles = sparkOptions.get(i + 1);
- i++;
- addToSparkArgs = false;
- }
- if(opt.startsWith(ARCHIVES_OPTION)) {
- userArchives = sparkOptions.get(i + 1);
- i++;
- addToSparkArgs = false;
- }
- if(addToSparkArgs) {
- sparkArgs.add(opt);
- }
- }
- }
-
- if ((yarnClusterMode || yarnClientMode)) {
- // Include the current working directory (of executor container)
- // in executor classpath, because it will contain localized
- // files
- appendWithPathSeparator(PWD, executorClassPath);
- appendWithPathSeparator(PWD, driverClassPath);
-
- sparkArgs.add("--conf");
- sparkArgs.add(EXECUTOR_CLASSPATH + executorClassPath.toString());
-
- sparkArgs.add("--conf");
- sparkArgs.add(DRIVER_CLASSPATH + driverClassPath.toString());
- }
-
- if (actionConf.get(MAPREDUCE_JOB_TAGS) != null) {
- sparkArgs.add("--conf");
- sparkArgs.add("spark.yarn.tags=" + actionConf.get(MAPREDUCE_JOB_TAGS));
- }
-
- if (!addedHiveSecurityToken) {
- sparkArgs.add("--conf");
- sparkArgs.add(HIVE_SECURITY_TOKEN + "=false");
- }
- if (!addedHBaseSecurityToken) {
- sparkArgs.add("--conf");
- sparkArgs.add(HBASE_SECURITY_TOKEN + "=false");
- }
- if(!addedLog4jExecutorSettings) {
- sparkArgs.add("--conf");
- sparkArgs.add(EXECUTOR_EXTRA_JAVA_OPTIONS + LOG4J_CONFIGURATION_JAVA_OPTION + SPARK_LOG4J_PROPS);
- }
- if(!addedLog4jDriverSettings) {
- sparkArgs.add("--conf");
- sparkArgs.add(DRIVER_EXTRA_JAVA_OPTIONS + LOG4J_CONFIGURATION_JAVA_OPTION + SPARK_LOG4J_PROPS);
- }
- File defaultConfFile = getMatchingFile(SPARK_DEFAULTS_FILE_PATTERN);
- if (defaultConfFile != null) {
- sparkArgs.add("--properties-file");
- sparkArgs.add(SPARK_DEFAULTS_FILE_PATTERN.toString());
- }
- if ((yarnClusterMode || yarnClientMode)) {
- Map<String, URI> fixedFileUrisMap = fixFsDefaultUrisAndFilterDuplicates(DistributedCache.getCacheFiles(actionConf));
- fixedFileUrisMap.put(SPARK_LOG4J_PROPS, new Path(SPARK_LOG4J_PROPS).toUri());
- fixedFileUrisMap.put(HIVE_SITE_CONF, new Path(HIVE_SITE_CONF).toUri());
- addUserDefined(userFiles, fixedFileUrisMap);
- Collection<URI> fixedFileUris = fixedFileUrisMap.values();
- JarFilter jarfilter = new JarFilter(fixedFileUris, jarPath);
- jarfilter.filter();
- jarPath = jarfilter.getApplicationJar();
-
- String cachedFiles = StringUtils.join(fixedFileUris, ",");
- if (cachedFiles != null && !cachedFiles.isEmpty()) {
- sparkArgs.add("--files");
- sparkArgs.add(cachedFiles);
- }
- Map<String, URI> fixedArchiveUrisMap = fixFsDefaultUrisAndFilterDuplicates(DistributedCache.
- getCacheArchives(actionConf));
- addUserDefined(userArchives, fixedArchiveUrisMap);
- String cachedArchives = StringUtils.join(fixedArchiveUrisMap.values(), ",");
- if (cachedArchives != null && !cachedArchives.isEmpty()) {
- sparkArgs.add("--archives");
- sparkArgs.add(cachedArchives);
- }
- setSparkYarnJarsConf(sparkArgs, jarfilter.getSparkYarnJar(), jarfilter.getSparkVersion());
- }
-
- if (!sparkArgs.contains(VERBOSE_OPTION)) {
- sparkArgs.add(VERBOSE_OPTION);
- }
+ final SparkArgsExtractor sparkArgsExtractor = new SparkArgsExtractor(actionConf);
+ final List<String> sparkArgs = sparkArgsExtractor.extract(args);
- sparkArgs.add(jarPath);
- for (String arg : args) {
- sparkArgs.add(arg);
- }
- if (isPyspark){
+ if (sparkArgsExtractor.isPySpark()){
createPySparkLibFolder();
}
-
System.out.println("Spark Action Main class : " + SparkSubmit.class.getName());
System.out.println();
System.out.println("Oozie Spark action configuration");
System.out.println("=================================================================");
System.out.println();
- PasswordMasker passwordMasker = new PasswordMasker();
- for (String arg : sparkArgs) {
+
+ final PasswordMasker passwordMasker = new PasswordMasker();
+ for (final String arg : sparkArgs) {
System.out.println(" " + passwordMasker.maskPasswordsIfNecessary(arg));
}
System.out.println();
+
try {
runSpark(sparkArgs.toArray(new String[sparkArgs.size()]));
}
@@ -296,21 +102,12 @@ public class SparkMain extends LauncherMain {
}
}
- private void addUserDefined(String userList, Map<String, URI> urisMap) {
- if(userList != null) {
- for (String file : userList.split(",")) {
- Path p = new Path(file);
- urisMap.put(p.getName(), p.toUri());
- }
- }
- }
-
- private void prepareHadoopConfig(Configuration actionConf) throws IOException {
+ private void prepareHadoopConfig(final Configuration actionConf) throws IOException {
// Copying oozie.action.conf.xml into hadoop configuration *-site files.
if (actionConf.getBoolean(CONF_OOZIE_SPARK_SETUP_HADOOP_CONF_DIR, false)) {
- String actionXml = System.getProperty("oozie.action.conf.xml");
+ final String actionXml = System.getProperty("oozie.action.conf.xml");
if (actionXml != null) {
- File currentDir = new File(actionXml).getParentFile();
+ final File currentDir = new File(actionXml).getParentFile();
writeHadoopConfig(actionXml, currentDir);
}
}
@@ -324,15 +121,15 @@ public class SparkMain extends LauncherMain {
* @throws IOException if there is an error during file copy
*/
private void createPySparkLibFolder() throws OozieActionConfiguratorException, IOException {
- File pythonLibDir = new File("python/lib");
+ final File pythonLibDir = new File("python/lib");
if(!pythonLibDir.exists()){
pythonLibDir.mkdirs();
System.out.println("PySpark lib folder " + pythonLibDir.getAbsolutePath() + " folder created.");
}
- for(Pattern fileNamePattern : PYSPARK_DEP_FILE_PATTERN) {
- File file = getMatchingPyFile(fileNamePattern);
- File destination = new File(pythonLibDir, file.getName());
+ for(final Pattern fileNamePattern : PYSPARK_DEP_FILE_PATTERN) {
+ final File file = getMatchingPyFile(fileNamePattern);
+ final File destination = new File(pythonLibDir, file.getName());
FileUtils.copyFile(file, destination);
System.out.println("Copied " + file + " to " + destination.getAbsolutePath());
}
@@ -345,8 +142,8 @@ public class SparkMain extends LauncherMain {
* @return the file if there is one
* @throws OozieActionConfiguratorException if there is are no files matching the pattern
*/
- private File getMatchingPyFile(Pattern fileNamePattern) throws OozieActionConfiguratorException {
- File f = getMatchingFile(fileNamePattern);
+ private File getMatchingPyFile(final Pattern fileNamePattern) throws OozieActionConfiguratorException {
+ final File f = getMatchingFile(fileNamePattern);
if (f != null) {
return f;
}
@@ -362,17 +159,23 @@ public class SparkMain extends LauncherMain {
* @param fileNamePattern the pattern to look for
* @return the file if there is one else it returns null
*/
- private static File getMatchingFile(Pattern fileNamePattern) throws OozieActionConfiguratorException {
- File localDir = new File(".");
- for(String fileName : localDir.list()){
- if(fileNamePattern.matcher(fileName).find()){
+ static File getMatchingFile(final Pattern fileNamePattern) {
+ final File localDir = new File(".");
+
+ final String[] localFileNames = localDir.list();
+ if (localFileNames == null) {
+ return null;
+ }
+
+ for (final String fileName : localFileNames){
+ if (fileNamePattern.matcher(fileName).find()){
return new File(fileName);
}
}
return null;
}
- private void runSpark(String[] args) throws Exception {
+ private void runSpark(final String[] args) throws Exception {
System.out.println("=================================================================");
System.out.println();
System.out.println(">>> Invoking Spark class now >>>");
@@ -381,46 +184,13 @@ public class SparkMain extends LauncherMain {
SparkSubmit.main(args);
}
- /**
- * Converts the options to be Spark-compatible.
- * <ul>
- * <li>Parameters are separated by whitespace and can be groupped using double quotes</li>
- * <li>Quotes should be removed</li>
- * <li>Adjacent whitespace separators are treated as one</li>
- * </ul>
- * @param sparkOpts the options for Spark
- * @return the options parsed into a list
- */
- static List<String> splitSparkOpts(String sparkOpts){
- List<String> result = new ArrayList<String>();
- StringBuilder currentWord = new StringBuilder();
- boolean insideQuote = false;
- for (int i = 0; i < sparkOpts.length(); i++) {
- char c = sparkOpts.charAt(i);
- if (c == '"') {
- insideQuote = !insideQuote;
- } else if (Character.isWhitespace(c) && !insideQuote) {
- if (currentWord.length() > 0) {
- result.add(currentWord.toString());
- currentWord.setLength(0);
- }
- } else {
- currentWord.append(c);
- }
- }
- if(currentWord.length()>0) {
- result.add(currentWord.toString());
- }
- return result;
- }
-
- public static String setUpSparkLog4J(Configuration distcpConf) throws IOException {
+ private String setUpSparkLog4J(final Configuration actionConf) throws IOException {
// Logfile to capture job IDs
- String hadoopJobId = System.getProperty("oozie.launcher.job.id");
+ final String hadoopJobId = System.getProperty("oozie.launcher.job.id");
if (hadoopJobId == null) {
throw new RuntimeException("Launcher Hadoop Job ID system,property not set");
}
- String logFile = new File("spark-oozie-" + hadoopJobId + ".log").getAbsolutePath();
+ final String logFile = new File("spark-oozie-" + hadoopJobId + ".log").getAbsolutePath();
Properties hadoopProps = new Properties();
// Preparing log4j configuration
@@ -430,8 +200,8 @@ public class SparkMain extends LauncherMain {
hadoopProps.load(log4jFile.openStream());
}
- String logLevel = distcpConf.get("oozie.spark.log.level", "INFO");
- String rootLogLevel = distcpConf.get("oozie.action." + LauncherMapper.ROOT_LOGGER_LEVEL, "INFO");
+ final String logLevel = actionConf.get("oozie.spark.log.level", "INFO");
+ final String rootLogLevel = actionConf.get("oozie.action." + LauncherMapper.ROOT_LOGGER_LEVEL, "INFO");
hadoopProps.setProperty("log4j.rootLogger", rootLogLevel + ", A");
hadoopProps.setProperty("log4j.logger.org.apache.spark", logLevel + ", A, jobid");
@@ -447,14 +217,10 @@ public class SparkMain extends LauncherMain {
hadoopProps.setProperty("log4j.logger.org.apache.hadoop.mapreduce.Job", "INFO, jobid");
hadoopProps.setProperty("log4j.logger.org.apache.hadoop.yarn.client.api.impl.YarnClientImpl", "INFO, jobid");
- String localProps = new File(SPARK_LOG4J_PROPS).getAbsolutePath();
- OutputStream os1 = new FileOutputStream(localProps);
- try {
+ final String localProps = new File(SPARK_LOG4J_PROPS).getAbsolutePath();
+ try (OutputStream os1 = new FileOutputStream(localProps)) {
hadoopProps.store(os1, "");
}
- finally {
- os1.close();
- }
PropertyConfigurator.configure(SPARK_LOG4J_PROPS);
return logFile;
}
@@ -467,100 +233,27 @@ public class SparkMain extends LauncherMain {
* @throws IOException
* @throws URISyntaxException
*/
- static Map<String, URI> fixFsDefaultUrisAndFilterDuplicates(URI[] files) throws IOException, URISyntaxException {
- Map<String, URI> map= new HashMap<>();
+ static Map<String, URI> fixFsDefaultUrisAndFilterDuplicates(final URI[] files) throws IOException, URISyntaxException {
+ final Map<String, URI> map= new LinkedHashMap<>();
if (files == null) {
return map;
}
- FileSystem fs = FileSystem.get(new Configuration(true));
+ final FileSystem fs = FileSystem.get(new Configuration(true));
for (int i = 0; i < files.length; i++) {
- URI fileUri = files[i];
- Path p = new Path(fileUri);
- map.put(p.getName(), getFixedUri(fs, fileUri));
+ final URI fileUri = files[i];
+ final Path p = new Path(fileUri);
+ map.put(p.getName(), HadoopUriFinder.getFixedUri(fs, fileUri));
}
return map;
}
/**
- * Sets spark.yarn.jars for Spark 2.X. Sets spark.yarn.jar for Spark 1.X.
- *
- * @param sparkArgs
- * @param sparkYarnJar
- * @param sparkVersion
- */
- private void setSparkYarnJarsConf(List<String> sparkArgs, String sparkYarnJar, String sparkVersion) {
- if (SPARK_VERSION_1.matcher(sparkVersion).find()) {
- // In Spark 1.X.X, set spark.yarn.jar to avoid
- // multiple distribution
- sparkArgs.add("--conf");
- sparkArgs.add(SPARK_YARN_JAR + "=" + sparkYarnJar);
- }
- else {
- // In Spark 2.X.X, set spark.yarn.jars
- sparkArgs.add("--conf");
- sparkArgs.add(SPARK_YARN_JARS + "=" + sparkYarnJar);
- }
- }
-
- private static String getJarVersion(File jarFile) throws IOException {
- @SuppressWarnings("resource")
- Manifest manifest = new JarFile(jarFile).getManifest();
- return manifest.getMainAttributes().getValue("Specification-Version");
- }
-
- /*
- * Get properties that needs to be passed to Spark as Spark configuration from actionConf.
- */
- @VisibleForTesting
- protected void appendOoziePropertiesToSparkConf(List<String> sparkArgs, Configuration actionConf) {
- for (Map.Entry<String, String> oozieConfig : actionConf
- .getValByRegex("^oozie\\.(?!launcher|spark).+").entrySet()) {
- sparkArgs.add("--conf");
- sparkArgs.add(String.format("spark.%s=%s", oozieConfig.getKey(), oozieConfig.getValue()));
- }
- }
-
- private void appendWithPathSeparator(String what, StringBuilder to){
- if(to.length() > 0){
- to.append(File.pathSeparator);
- }
- to.append(what);
- }
-
- private static URI getFixedUri(URI fileUri) throws URISyntaxException, IOException {
- FileSystem fs = FileSystem.get(new Configuration(true));
- return getFixedUri(fs, fileUri);
- }
-
- /**
- * Spark compares URIs based on scheme, host and port. Here we convert URIs
- * into the default format so that Spark won't think those belong to
- * different file system. This will avoid an extra copy of files which
- * already exists on same hdfs.
- *
- * @param fs
- * @param fileUri
- * @return fixed uri
- * @throws URISyntaxException
- */
- private static URI getFixedUri(FileSystem fs, URI fileUri) throws URISyntaxException {
- if (fs.getUri().getScheme().equals(fileUri.getScheme())
- && (fs.getUri().getHost().equals(fileUri.getHost()) || fileUri.getHost() == null)
- && (fs.getUri().getPort() == -1 || fileUri.getPort() == -1
- || fs.getUri().getPort() == fileUri.getPort())) {
- return new URI(fs.getUri().getScheme(), fileUri.getUserInfo(), fs.getUri().getHost(), fs.getUri().getPort(),
- fileUri.getPath(), fileUri.getQuery(), fileUri.getFragment());
- }
- return fileUri;
- }
-
- /**
* Sets up hive-site.xml
*
* @param hiveConf
* @throws IOException
*/
- private void setHiveSite(Configuration hiveConf) throws IOException {
+ private void setHiveSite(final Configuration hiveConf) throws IOException {
// See https://issues.apache.org/jira/browse/HIVE-1411
hiveConf.set("datanucleus.plugin.pluginRegistryBundleCheck", "LOG");
@@ -594,99 +287,4 @@ public class SparkMain extends LauncherMain {
catch (Throwable ignore) {
}
}
-
- /**
- * This class is used for filtering out unwanted jars.
- */
- static class JarFilter {
- private String sparkVersion = "1.X.X";
- private String sparkYarnJar;
- private String applicationJar;
- private Collection<URI> listUris = null;
-
- /**
- * @param listUris List of URIs to be filtered
- * @param jarPath Application jar
- * @throws IOException
- * @throws URISyntaxException
- */
- public JarFilter(Collection<URI> listUris, String jarPath) throws URISyntaxException, IOException {
- this.listUris = listUris;
- applicationJar = jarPath;
- Path p = new Path(jarPath);
- if (p.isAbsolute()) {
- applicationJar = getFixedUri(p.toUri()).toString();
- }
- }
-
- /**
- * Filters out the Spark yarn jar and application jar. Also records
- * spark yarn jar's version.
- *
- * @throws OozieActionConfiguratorException
- */
- public void filter() throws OozieActionConfiguratorException {
- Iterator<URI> iterator = listUris.iterator();
- File matchedFile = null;
- Path applJarPath = new Path(applicationJar);
- while (iterator.hasNext()) {
- URI uri = iterator.next();
- Path p = new Path(uri);
- if (SPARK_YARN_JAR_PATTERN.matcher(p.getName()).find()) {
- matchedFile = getMatchingFile(SPARK_YARN_JAR_PATTERN);
- }
- else if (SPARK_ASSEMBLY_JAR_PATTERN.matcher(p.getName()).find()) {
- matchedFile = getMatchingFile(SPARK_ASSEMBLY_JAR_PATTERN);
- }
- if (matchedFile != null) {
- sparkYarnJar = uri.toString();
- try {
- sparkVersion = getJarVersion(matchedFile);
- System.out.println("Spark Version " + sparkVersion);
- }
- catch (IOException io) {
- System.out.println(
- "Unable to open " + matchedFile.getPath() + ". Default Spark Version " + sparkVersion);
- }
- iterator.remove();
- matchedFile = null;
- }
- // Here we skip the application jar, because
- // (if uris are same,) it will get distributed multiple times
- // - one time with --files and another time as application jar.
- if (isApplicationJar(p.getName(), uri, applJarPath)) {
- String fragment = uri.getFragment();
- applicationJar = fragment != null && fragment.length() > 0 ? fragment : uri.toString();
- iterator.remove();
- }
- }
- }
-
- /**
- * Checks if a file is application jar
- *
- * @param fileName fileName name of the file
- * @param fileUri fileUri URI of the file
- * @param applJarPath Path of application jar
- * @return true if fileName or fileUri is the application jar
- */
- private boolean isApplicationJar(String fileName, URI fileUri, Path applJarPath) {
- return (fileName.equals(applicationJar) || fileUri.toString().equals(applicationJar)
- || applJarPath.getName().equals(fileName)
- || applicationJar.equals(fileUri.getFragment()));
- }
-
- public String getApplicationJar() {
- return applicationJar;
- }
-
- public String getSparkYarnJar() {
- return sparkYarnJar;
- }
-
- public String getSparkVersion() {
- return sparkVersion;
- }
-
- }
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/oozie/blob/e284e63c/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkOptionsSplitter.java
----------------------------------------------------------------------
diff --git a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkOptionsSplitter.java b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkOptionsSplitter.java
new file mode 100644
index 0000000..30def6f
--- /dev/null
+++ b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkOptionsSplitter.java
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oozie.action.hadoop;
+
+import java.util.ArrayList;
+import java.util.List;
+
+class SparkOptionsSplitter {
+
+ /**
+ * Converts the options to be Spark-compatible.
+ * <ul>
+ * <li>Parameters are separated by whitespace and can be groupped using double quotes</li>
+ * <li>Quotes should be removed</li>
+ * <li>Adjacent whitespace separators are treated as one</li>
+ * </ul>
+ *
+ * @param sparkOpts the options for Spark
+ * @return the options parsed into a list
+ */
+ static List<String> splitSparkOpts(final String sparkOpts) {
+ final List<String> result = new ArrayList<String>();
+ final StringBuilder currentWord = new StringBuilder();
+
+ boolean insideQuote = false;
+ for (int i = 0; i < sparkOpts.length(); i++) {
+ final char c = sparkOpts.charAt(i);
+ if (c == '"') {
+ insideQuote = !insideQuote;
+ }
+ else if (Character.isWhitespace(c) && !insideQuote) {
+ if (currentWord.length() > 0) {
+ result.add(currentWord.toString());
+ currentWord.setLength(0);
+ }
+ }
+ else {
+ currentWord.append(c);
+ }
+ }
+
+ if (currentWord.length() > 0) {
+ result.add(currentWord.toString());
+ }
+
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/oozie/blob/e284e63c/sharelib/spark/src/test/java/org/apache/oozie/action/hadoop/TestJarFilter.java
----------------------------------------------------------------------
diff --git a/sharelib/spark/src/test/java/org/apache/oozie/action/hadoop/TestJarFilter.java b/sharelib/spark/src/test/java/org/apache/oozie/action/hadoop/TestJarFilter.java
index 2d4c83c..ff1b3ce 100644
--- a/sharelib/spark/src/test/java/org/apache/oozie/action/hadoop/TestJarFilter.java
+++ b/sharelib/spark/src/test/java/org/apache/oozie/action/hadoop/TestJarFilter.java
@@ -31,7 +31,6 @@ import java.util.jar.Attributes;
import java.util.jar.JarOutputStream;
import java.util.jar.Manifest;
-import org.apache.oozie.action.hadoop.SparkMain.JarFilter;
import org.junit.Test;
public class TestJarFilter {
http://git-wip-us.apache.org/repos/asf/oozie/blob/e284e63c/sharelib/spark/src/test/java/org/apache/oozie/action/hadoop/TestSparkArgsExtractor.java
----------------------------------------------------------------------
diff --git a/sharelib/spark/src/test/java/org/apache/oozie/action/hadoop/TestSparkArgsExtractor.java b/sharelib/spark/src/test/java/org/apache/oozie/action/hadoop/TestSparkArgsExtractor.java
new file mode 100644
index 0000000..7db26a6
--- /dev/null
+++ b/sharelib/spark/src/test/java/org/apache/oozie/action/hadoop/TestSparkArgsExtractor.java
@@ -0,0 +1,197 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oozie.action.hadoop;
+
+import com.google.common.collect.Lists;
+import org.apache.hadoop.conf.Configuration;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+
+public class TestSparkArgsExtractor {
+
+ @Test
+ public void testAppendOoziePropertiesToSparkConf() throws Exception {
+ final List<String> sparkArgs = new ArrayList<>();
+ final Configuration actionConf = new Configuration();
+ actionConf.set("foo", "foo-not-to-include");
+ actionConf.set("oozie.launcher", "launcher-not-to-include");
+ actionConf.set("oozie.spark", "spark-not-to-include");
+ actionConf.set("oozie.bar", "bar");
+
+ new SparkArgsExtractor(actionConf).appendOoziePropertiesToSparkConf(sparkArgs);
+
+ assertEquals(Lists.newArrayList("--conf", "spark.oozie.bar=bar"), sparkArgs);
+ }
+
+ @Test
+ public void testLocalClientArgsParsing() throws Exception {
+ final Configuration actionConf = new Configuration();
+ actionConf.set(SparkActionExecutor.SPARK_MASTER, "local[*]");
+ actionConf.set(SparkActionExecutor.SPARK_MODE, "client");
+ actionConf.set(SparkActionExecutor.SPARK_CLASS, "org.apache.oozie.example.SparkFileCopy");
+ actionConf.set(SparkActionExecutor.SPARK_JOB_NAME, "Spark Copy File");
+ actionConf.set(SparkActionExecutor.SPARK_OPTS, "--driver-memory 1042M " +
+ "--conf spark.executor.extraClassPath=aaa " +
+ "--conf user.property.after.spark.executor.extraClassPath=bbb " +
+ "--conf spark.driver.extraClassPath=ccc " +
+ "--conf user.property.after.spark.driver.extraClassPath=ddd " +
+ "--conf spark.executor.extraJavaOptions=\"-XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp\"");
+ actionConf.set(SparkActionExecutor.SPARK_JAR, "/lib/test.jar");
+
+ final String[] mainArgs = {"arg0", "arg1"};
+ final List<String> sparkArgs = new SparkArgsExtractor(actionConf).extract(mainArgs);
+
+ assertEquals("Spark args mismatch",
+ Lists.newArrayList("--master", "local[*]",
+ "--deploy-mode", "client",
+ "--name", "Spark Copy File",
+ "--class", "org.apache.oozie.example.SparkFileCopy",
+ "--driver-memory", "1042M",
+ "--conf", "spark.executor.extraClassPath=aaa",
+ "--conf", "user.property.after.spark.executor.extraClassPath=bbb",
+ "--conf", "spark.driver.extraClassPath=ccc",
+ "--conf", "user.property.after.spark.driver.extraClassPath=ddd",
+ "--conf", "spark.executor.extraJavaOptions=-XX:+HeapDumpOnOutOfMemoryError " +
+ "-XX:HeapDumpPath=/tmp -Dlog4j.configuration=spark-log4j.properties",
+ "--conf", "spark.yarn.security.tokens.hive.enabled=false",
+ "--conf", "spark.yarn.security.tokens.hbase.enabled=false",
+ "--conf", "spark.driver.extraJavaOptions=-Dlog4j.configuration=spark-log4j.properties",
+ "--verbose",
+ "/lib/test.jar",
+ "arg0",
+ "arg1"),
+ sparkArgs);
+ }
+
+ @Test
+ public void testYarnClientExecutorAndDriverExtraClasspathsArgsParsing() throws Exception {
+ final Configuration actionConf = new Configuration();
+ actionConf.set(SparkActionExecutor.SPARK_MASTER, "yarn");
+ actionConf.set(SparkActionExecutor.SPARK_MODE, "client");
+ actionConf.set(SparkActionExecutor.SPARK_CLASS, "org.apache.oozie.example.SparkFileCopy");
+ actionConf.set(SparkActionExecutor.SPARK_JOB_NAME, "Spark Copy File");
+ actionConf.set(SparkActionExecutor.SPARK_OPTS, "--driver-memory 1042M " +
+ "--conf spark.executor.extraClassPath=aaa " +
+ "--conf user.property.after.spark.executor.extraClassPath=bbb " +
+ "--conf spark.driver.extraClassPath=ccc " +
+ "--conf user.property.after.spark.driver.extraClassPath=ddd " +
+ "--conf spark.executor.extraJavaOptions=\"-XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp\"");
+ actionConf.set(SparkActionExecutor.SPARK_JAR, "/lib/test.jar");
+
+ final String[] mainArgs = {"arg0", "arg1"};
+ final List<String> sparkArgs = new SparkArgsExtractor(actionConf).extract(mainArgs);
+
+ assertEquals("Spark args mismatch",
+ Lists.newArrayList("--master", "yarn",
+ "--deploy-mode", "client",
+ "--name", "Spark Copy File",
+ "--class", "org.apache.oozie.example.SparkFileCopy",
+ "--driver-memory", "1042M",
+ "--conf", "user.property.after.spark.executor.extraClassPath=bbb",
+ "--conf", "user.property.after.spark.driver.extraClassPath=ddd",
+ "--conf", "spark.executor.extraJavaOptions=-XX:+HeapDumpOnOutOfMemoryError " +
+ "-XX:HeapDumpPath=/tmp -Dlog4j.configuration=spark-log4j.properties",
+ "--conf", "spark.executor.extraClassPath=aaa:$PWD/*",
+ "--conf", "spark.driver.extraClassPath=ccc:$PWD/*",
+ "--conf", "spark.yarn.security.tokens.hive.enabled=false",
+ "--conf", "spark.yarn.security.tokens.hbase.enabled=false",
+ "--conf", "spark.driver.extraJavaOptions=-Dlog4j.configuration=spark-log4j.properties",
+ "--files", "spark-log4j.properties,hive-site.xml",
+ "--conf", "spark.yarn.jar=null",
+ "--verbose",
+ "/lib/test.jar",
+ "arg0",
+ "arg1"),
+ sparkArgs);
+ }
+
+ @Test
+ public void testYarnClientFilesAndArchivesArgsParsing() throws Exception {
+ final Configuration actionConf = new Configuration();
+ actionConf.set(SparkActionExecutor.SPARK_MASTER, "yarn");
+ actionConf.set(SparkActionExecutor.SPARK_MODE, "client");
+ actionConf.set(SparkActionExecutor.SPARK_CLASS, "org.apache.oozie.example.SparkFileCopy");
+ actionConf.set(SparkActionExecutor.SPARK_JOB_NAME, "Spark Copy File");
+ actionConf.set(SparkActionExecutor.SPARK_OPTS, "--files aaa " +
+ "--archives bbb " +
+ "--files=ccc " +
+ "--archives=ddd");
+ actionConf.set(SparkActionExecutor.SPARK_JAR, "/lib/test.jar");
+
+ final String[] mainArgs = {"arg0", "arg1"};
+ final List<String> sparkArgs = new SparkArgsExtractor(actionConf).extract(mainArgs);
+
+ assertEquals("Spark args mismatch",
+ Lists.newArrayList("--master", "yarn",
+ "--deploy-mode", "client",
+ "--name", "Spark Copy File",
+ "--class", "org.apache.oozie.example.SparkFileCopy",
+ "--conf", "spark.executor.extraClassPath=$PWD/*",
+ "--conf", "spark.driver.extraClassPath=$PWD/*",
+ "--conf", "spark.yarn.security.tokens.hive.enabled=false",
+ "--conf", "spark.yarn.security.tokens.hbase.enabled=false",
+ "--conf", "spark.executor.extraJavaOptions=-Dlog4j.configuration=spark-log4j.properties",
+ "--conf", "spark.driver.extraJavaOptions=-Dlog4j.configuration=spark-log4j.properties",
+ "--files", "spark-log4j.properties,hive-site.xml,aaa,ccc",
+ "--archives", "bbb,ddd",
+ "--conf", "spark.yarn.jar=null",
+ "--verbose",
+ "/lib/test.jar",
+ "arg0",
+ "arg1"),
+ sparkArgs);
+ }
+
+ @Test
+ public void testDriverClassPathArgsParsing() throws Exception {
+ final Configuration actionConf = new Configuration();
+ actionConf.set(SparkActionExecutor.SPARK_MASTER, "yarn");
+ actionConf.set(SparkActionExecutor.SPARK_MODE, "client");
+ actionConf.set(SparkActionExecutor.SPARK_CLASS, "org.apache.oozie.example.SparkFileCopy");
+ actionConf.set(SparkActionExecutor.SPARK_JOB_NAME, "Spark Copy File");
+ actionConf.set(SparkActionExecutor.SPARK_OPTS, "--driver-class-path aaa");
+ actionConf.set(SparkActionExecutor.SPARK_JAR, "/lib/test.jar");
+
+ final String[] mainArgs = {"arg0", "arg1"};
+ final List<String> sparkArgs = new SparkArgsExtractor(actionConf).extract(mainArgs);
+
+ assertEquals("Spark args mismatch",
+ Lists.newArrayList("--master", "yarn",
+ "--deploy-mode", "client",
+ "--name", "Spark Copy File",
+ "--class", "org.apache.oozie.example.SparkFileCopy",
+ "--conf", "spark.executor.extraClassPath=$PWD/*",
+ "--conf", "spark.driver.extraClassPath=aaa:$PWD/*",
+ "--conf", "spark.yarn.security.tokens.hive.enabled=false",
+ "--conf", "spark.yarn.security.tokens.hbase.enabled=false",
+ "--conf", "spark.executor.extraJavaOptions=-Dlog4j.configuration=spark-log4j.properties",
+ "--conf", "spark.driver.extraJavaOptions=-Dlog4j.configuration=spark-log4j.properties",
+ "--files", "spark-log4j.properties,hive-site.xml",
+ "--conf", "spark.yarn.jar=null",
+ "--verbose",
+ "/lib/test.jar",
+ "arg0",
+ "arg1"),
+ sparkArgs);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/oozie/blob/e284e63c/sharelib/spark/src/test/java/org/apache/oozie/action/hadoop/TestSparkMain.java
----------------------------------------------------------------------
diff --git a/sharelib/spark/src/test/java/org/apache/oozie/action/hadoop/TestSparkMain.java b/sharelib/spark/src/test/java/org/apache/oozie/action/hadoop/TestSparkMain.java
index 5f6a0cd..b9f37c8 100644
--- a/sharelib/spark/src/test/java/org/apache/oozie/action/hadoop/TestSparkMain.java
+++ b/sharelib/spark/src/test/java/org/apache/oozie/action/hadoop/TestSparkMain.java
@@ -127,19 +127,6 @@ public class TestSparkMain extends MainTestCase {
}
}
- public void testAppendOoziePropertiesToSparkConf() throws Exception {
- SparkMain instance = SparkMain.class.newInstance();
- List<String> sparkArgs = new ArrayList<String>();
- Configuration actionConf = new Configuration();
- actionConf.set("foo", "foo-not-to-include");
- actionConf.set("oozie.launcher", "launcher-not-to-include");
- actionConf.set("oozie.spark", "spark-not-to-include");
- actionConf.set("oozie.bar", "bar");
-
- instance.appendOoziePropertiesToSparkConf(sparkArgs, actionConf);
- assertEquals(Lists.newArrayList("--conf", "spark.oozie.bar=bar"), sparkArgs);
- }
-
public void testJobIDPattern() {
List<String> lines = new ArrayList<String>();
lines.add("Submitted application application_001");
http://git-wip-us.apache.org/repos/asf/oozie/blob/e284e63c/sharelib/spark/src/test/java/org/apache/oozie/action/hadoop/TestSparkOptionsSplitter.java
----------------------------------------------------------------------
diff --git a/sharelib/spark/src/test/java/org/apache/oozie/action/hadoop/TestSparkOptionsSplitter.java b/sharelib/spark/src/test/java/org/apache/oozie/action/hadoop/TestSparkOptionsSplitter.java
index 31f53ac..02786a2 100644
--- a/sharelib/spark/src/test/java/org/apache/oozie/action/hadoop/TestSparkOptionsSplitter.java
+++ b/sharelib/spark/src/test/java/org/apache/oozie/action/hadoop/TestSparkOptionsSplitter.java
@@ -34,13 +34,13 @@ public class TestSparkOptionsSplitter {
@Parameterized.Parameters
public static List<Object[]> params() {
return Arrays.asList(new Object[][]{
- {"--option1 value1", Arrays.asList(new String[]{"--option1", "value1"})},
- {"--option1 value1", Arrays.asList(new String[]{"--option1", "value1"})},
- {" --option1 value1 ", Arrays.asList(new String[]{"--option1", "value1"})},
- {"--conf special=value1", Arrays.asList(new String[]{"--conf", "special=value1"})},
- {"--conf special=\"value1\"", Arrays.asList(new String[]{"--conf", "special=value1"})},
- {"--conf special=\"value1 value2\"", Arrays.asList(new String[]{"--conf", "special=value1 value2"})},
- {" --conf special=\"value1 value2\" ", Arrays.asList(new String[]{"--conf", "special=value1 value2"})},
+ {"--option1 value1", Arrays.asList("--option1", "value1")},
+ {"--option1 value1", Arrays.asList("--option1", "value1")},
+ {" --option1 value1 ", Arrays.asList("--option1", "value1")},
+ {"--conf special=value1", Arrays.asList("--conf", "special=value1")},
+ {"--conf special=\"value1\"", Arrays.asList("--conf", "special=value1")},
+ {"--conf special=\"value1 value2\"", Arrays.asList("--conf", "special=value1 value2")},
+ {" --conf special=\"value1 value2\" ", Arrays.asList("--conf", "special=value1 value2")},
});
}
@@ -55,6 +55,6 @@ public class TestSparkOptionsSplitter {
@Test
public void test() {
- assertThat("Error for input >>" + input + "<<", SparkMain.splitSparkOpts(input), is(output));
+ assertThat("Error for input >>" + input + "<<", SparkOptionsSplitter.splitSparkOpts(input), is(output));
}
}
[29/31] oozie git commit: OOZIE-3036 Spark 2.2.0 support: tell Spark
not to get any delegation tokens (andras.piros via gezapeti)
Posted by sa...@apache.org.
OOZIE-3036 Spark 2.2.0 support: tell Spark not to get any delegation tokens (andras.piros via gezapeti)
(cherry picked from commit 04a6d0541e83ab457db7c79e86986bb69a8eb432)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/169a1a2a
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/169a1a2a
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/169a1a2a
Branch: refs/heads/branch-4.3
Commit: 169a1a2a18fea0f16cc278eef67b1e5e36cc2487
Parents: d64a4c3
Author: Gezapeti Cseh <ge...@gmail.com>
Authored: Tue Aug 15 11:45:39 2017 +0200
Committer: satishsaley <sa...@apache.org>
Committed: Fri Dec 8 16:34:56 2017 -0800
----------------------------------------------------------------------
release-log.txt | 1 +
.../oozie/action/hadoop/SparkArgsExtractor.java | 68 ++++++++++++++++----
.../action/hadoop/TestSparkArgsExtractor.java | 20 ++++++
3 files changed, 77 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/169a1a2a/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 557f657..16d3af8 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.3.1 release
+OOZIE-3036 Spark 2.2.0 support: tell Spark not to get any delegation tokens (andras.piros via gezapeti)
OOZIE-3028 Oozie Pig Action fails with no python dependencies (dbist13 via rohini)
OOZIE-2984 Parse spark-defaults.conf values with spaces without needing the quotes (andras.piros via gezapeti)
OOZIE-2825 Custom Authentication doc page is not well formatted (Jan Hentschel via rkanter)
http://git-wip-us.apache.org/repos/asf/oozie/blob/169a1a2a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkArgsExtractor.java
----------------------------------------------------------------------
diff --git a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkArgsExtractor.java b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkArgsExtractor.java
index 3270eea..a465d6d 100644
--- a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkArgsExtractor.java
+++ b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkArgsExtractor.java
@@ -40,8 +40,12 @@ class SparkArgsExtractor {
private static final String FILES_OPTION = "--files";
private static final String ARCHIVES_OPTION = "--archives";
private static final String LOG4J_CONFIGURATION_JAVA_OPTION = "-Dlog4j.configuration=";
- private static final String HIVE_SECURITY_TOKEN = "spark.yarn.security.tokens.hive.enabled";
- private static final String HBASE_SECURITY_TOKEN = "spark.yarn.security.tokens.hbase.enabled";
+ private static final String SECURITY_TOKENS_HADOOPFS = "spark.yarn.security.tokens.hadoopfs.enabled";
+ private static final String SECURITY_TOKENS_HIVE = "spark.yarn.security.tokens.hive.enabled";
+ private static final String SECURITY_TOKENS_HBASE = "spark.yarn.security.tokens.hbase.enabled";
+ private static final String SECURITY_CREDENTIALS_HADOOPFS = "spark.yarn.security.credentials.hadoopfs.enabled";
+ private static final String SECURITY_CREDENTIALS_HIVE = "spark.yarn.security.credentials.hive.enabled";
+ private static final String SECURITY_CREDENTIALS_HBASE = "spark.yarn.security.credentials.hbase.enabled";
private static final String PWD = "$PWD" + File.separator + "*";
private static final String MASTER_OPTION = "--master";
private static final String MODE_OPTION = "--deploy-mode";
@@ -114,8 +118,15 @@ class SparkArgsExtractor {
if (jarPath != null && jarPath.endsWith(".py")) {
pySpark = true;
}
- boolean addedHiveSecurityToken = false;
- boolean addedHBaseSecurityToken = false;
+
+ boolean addedSecurityTokensHadoopFS = false;
+ boolean addedSecurityTokensHive = false;
+ boolean addedSecurityTokensHBase = false;
+
+ boolean addedSecurityCredentialsHadoopFS = false;
+ boolean addedSecurityCredentialsHive = false;
+ boolean addedSecurityCredentialsHBase = false;
+
boolean addedLog4jDriverSettings = false;
boolean addedLog4jExecutorSettings = false;
final StringBuilder driverClassPath = new StringBuilder();
@@ -145,12 +156,27 @@ class SparkArgsExtractor {
addToSparkArgs = false;
}
}
- if (opt.startsWith(HIVE_SECURITY_TOKEN)) {
- addedHiveSecurityToken = true;
+
+ if (opt.startsWith(SECURITY_TOKENS_HADOOPFS)) {
+ addedSecurityTokensHadoopFS = true;
+ }
+ if (opt.startsWith(SECURITY_TOKENS_HIVE)) {
+ addedSecurityTokensHive = true;
}
- if (opt.startsWith(HBASE_SECURITY_TOKEN)) {
- addedHBaseSecurityToken = true;
+ if (opt.startsWith(SECURITY_TOKENS_HBASE)) {
+ addedSecurityTokensHBase = true;
}
+
+ if (opt.startsWith(SECURITY_CREDENTIALS_HADOOPFS)) {
+ addedSecurityCredentialsHadoopFS = true;
+ }
+ if (opt.startsWith(SECURITY_CREDENTIALS_HIVE)) {
+ addedSecurityCredentialsHive = true;
+ }
+ if (opt.startsWith(SECURITY_CREDENTIALS_HBASE)) {
+ addedSecurityCredentialsHBase = true;
+ }
+
if (opt.startsWith(EXECUTOR_EXTRA_JAVA_OPTIONS) || opt.startsWith(DRIVER_EXTRA_JAVA_OPTIONS)) {
if (!opt.contains(LOG4J_CONFIGURATION_JAVA_OPTION)) {
opt += " " + LOG4J_CONFIGURATION_JAVA_OPTION + SparkMain.SPARK_LOG4J_PROPS;
@@ -222,14 +248,32 @@ class SparkArgsExtractor {
sparkArgs.add(SPARK_YARN_TAGS + OPT_SEPARATOR + actionConf.get(LauncherMain.MAPREDUCE_JOB_TAGS));
}
- if (!addedHiveSecurityToken) {
+ if (!addedSecurityTokensHadoopFS) {
sparkArgs.add(CONF_OPTION);
- sparkArgs.add(HIVE_SECURITY_TOKEN + OPT_SEPARATOR + Boolean.toString(false));
+ sparkArgs.add(SECURITY_TOKENS_HADOOPFS + OPT_SEPARATOR + Boolean.toString(false));
}
- if (!addedHBaseSecurityToken) {
+ if (!addedSecurityTokensHive) {
sparkArgs.add(CONF_OPTION);
- sparkArgs.add(HBASE_SECURITY_TOKEN + OPT_SEPARATOR + Boolean.toString(false));
+ sparkArgs.add(SECURITY_TOKENS_HIVE + OPT_SEPARATOR + Boolean.toString(false));
}
+ if (!addedSecurityTokensHBase) {
+ sparkArgs.add(CONF_OPTION);
+ sparkArgs.add(SECURITY_TOKENS_HBASE + OPT_SEPARATOR + Boolean.toString(false));
+ }
+
+ if (!addedSecurityCredentialsHadoopFS) {
+ sparkArgs.add(CONF_OPTION);
+ sparkArgs.add(SECURITY_CREDENTIALS_HADOOPFS + OPT_SEPARATOR + Boolean.toString(false));
+ }
+ if (!addedSecurityCredentialsHive) {
+ sparkArgs.add(CONF_OPTION);
+ sparkArgs.add(SECURITY_CREDENTIALS_HIVE + OPT_SEPARATOR + Boolean.toString(false));
+ }
+ if (!addedSecurityCredentialsHBase) {
+ sparkArgs.add(CONF_OPTION);
+ sparkArgs.add(SECURITY_CREDENTIALS_HBASE + OPT_SEPARATOR + Boolean.toString(false));
+ }
+
if (!addedLog4jExecutorSettings) {
sparkArgs.add(CONF_OPTION);
sparkArgs.add(EXECUTOR_EXTRA_JAVA_OPTIONS + LOG4J_CONFIGURATION_JAVA_OPTION + SparkMain.SPARK_LOG4J_PROPS);
http://git-wip-us.apache.org/repos/asf/oozie/blob/169a1a2a/sharelib/spark/src/test/java/org/apache/oozie/action/hadoop/TestSparkArgsExtractor.java
----------------------------------------------------------------------
diff --git a/sharelib/spark/src/test/java/org/apache/oozie/action/hadoop/TestSparkArgsExtractor.java b/sharelib/spark/src/test/java/org/apache/oozie/action/hadoop/TestSparkArgsExtractor.java
index d0541ca..574bf24 100644
--- a/sharelib/spark/src/test/java/org/apache/oozie/action/hadoop/TestSparkArgsExtractor.java
+++ b/sharelib/spark/src/test/java/org/apache/oozie/action/hadoop/TestSparkArgsExtractor.java
@@ -73,8 +73,12 @@ public class TestSparkArgsExtractor {
"--conf", "user.property.after.spark.driver.extraClassPath=ddd",
"--conf", "spark.executor.extraJavaOptions=-XX:+HeapDumpOnOutOfMemoryError " +
"-XX:HeapDumpPath=/tmp -Dlog4j.configuration=spark-log4j.properties",
+ "--conf", "spark.yarn.security.tokens.hadoopfs.enabled=false",
"--conf", "spark.yarn.security.tokens.hive.enabled=false",
"--conf", "spark.yarn.security.tokens.hbase.enabled=false",
+ "--conf", "spark.yarn.security.credentials.hadoopfs.enabled=false",
+ "--conf", "spark.yarn.security.credentials.hive.enabled=false",
+ "--conf", "spark.yarn.security.credentials.hbase.enabled=false",
"--conf", "spark.driver.extraJavaOptions=-Dlog4j.configuration=spark-log4j.properties",
"--verbose",
"/lib/test.jar",
@@ -113,8 +117,12 @@ public class TestSparkArgsExtractor {
"-XX:HeapDumpPath=/tmp -Dlog4j.configuration=spark-log4j.properties",
"--conf", "spark.executor.extraClassPath=aaa:$PWD/*",
"--conf", "spark.driver.extraClassPath=ccc:$PWD/*",
+ "--conf", "spark.yarn.security.tokens.hadoopfs.enabled=false",
"--conf", "spark.yarn.security.tokens.hive.enabled=false",
"--conf", "spark.yarn.security.tokens.hbase.enabled=false",
+ "--conf", "spark.yarn.security.credentials.hadoopfs.enabled=false",
+ "--conf", "spark.yarn.security.credentials.hive.enabled=false",
+ "--conf", "spark.yarn.security.credentials.hbase.enabled=false",
"--conf", "spark.driver.extraJavaOptions=-Dlog4j.configuration=spark-log4j.properties",
"--files", "spark-log4j.properties,hive-site.xml",
"--conf", "spark.yarn.jar=null",
@@ -148,8 +156,12 @@ public class TestSparkArgsExtractor {
"--class", "org.apache.oozie.example.SparkFileCopy",
"--conf", "spark.executor.extraClassPath=$PWD/*",
"--conf", "spark.driver.extraClassPath=$PWD/*",
+ "--conf", "spark.yarn.security.tokens.hadoopfs.enabled=false",
"--conf", "spark.yarn.security.tokens.hive.enabled=false",
"--conf", "spark.yarn.security.tokens.hbase.enabled=false",
+ "--conf", "spark.yarn.security.credentials.hadoopfs.enabled=false",
+ "--conf", "spark.yarn.security.credentials.hive.enabled=false",
+ "--conf", "spark.yarn.security.credentials.hbase.enabled=false",
"--conf", "spark.executor.extraJavaOptions=-Dlog4j.configuration=spark-log4j.properties",
"--conf", "spark.driver.extraJavaOptions=-Dlog4j.configuration=spark-log4j.properties",
"--files", "spark-log4j.properties,hive-site.xml,aaa,ccc",
@@ -182,8 +194,12 @@ public class TestSparkArgsExtractor {
"--class", "org.apache.oozie.example.SparkFileCopy",
"--conf", "spark.executor.extraClassPath=$PWD/*",
"--conf", "spark.driver.extraClassPath=aaa:$PWD/*",
+ "--conf", "spark.yarn.security.tokens.hadoopfs.enabled=false",
"--conf", "spark.yarn.security.tokens.hive.enabled=false",
"--conf", "spark.yarn.security.tokens.hbase.enabled=false",
+ "--conf", "spark.yarn.security.credentials.hadoopfs.enabled=false",
+ "--conf", "spark.yarn.security.credentials.hive.enabled=false",
+ "--conf", "spark.yarn.security.credentials.hbase.enabled=false",
"--conf", "spark.executor.extraJavaOptions=-Dlog4j.configuration=spark-log4j.properties",
"--conf", "spark.driver.extraJavaOptions=-Dlog4j.configuration=spark-log4j.properties",
"--files", "spark-log4j.properties,hive-site.xml",
@@ -221,8 +237,12 @@ public class TestSparkArgsExtractor {
"-Dlog4j.configuration=spark-log4j.properties",
"--conf", "spark.executor.extraClassPath=$PWD/*",
"--conf", "spark.driver.extraClassPath=$PWD/*",
+ "--conf", "spark.yarn.security.tokens.hadoopfs.enabled=false",
"--conf", "spark.yarn.security.tokens.hive.enabled=false",
"--conf", "spark.yarn.security.tokens.hbase.enabled=false",
+ "--conf", "spark.yarn.security.credentials.hadoopfs.enabled=false",
+ "--conf", "spark.yarn.security.credentials.hive.enabled=false",
+ "--conf", "spark.yarn.security.credentials.hbase.enabled=false",
"--files", "spark-log4j.properties,hive-site.xml",
"--conf", "spark.yarn.jar=null",
"--verbose",
[09/31] oozie git commit: OOZIE-2771 Allow retrieving keystore and
truststore passwords from Hadoop Credential Provider (asasvari via
abhishekbafna)
Posted by sa...@apache.org.
OOZIE-2771 Allow retrieving keystore and truststore passwords from Hadoop Credential Provider (asasvari via abhishekbafna)
(cherry picked from commit f0c4f2daf61ca7c8b059d55be0465025b79f123e)
server/src/main/java/org/apache/oozie/server/SSLServerConnectorFactory.java
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/6e5dccf7
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/6e5dccf7
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/6e5dccf7
Branch: refs/heads/branch-4.3
Commit: 6e5dccf75c8898320850e8386f45a169dacb9ad7
Parents: 3b24d9d
Author: abhisek bafna <ab...@hortonworks.com>
Authored: Wed Jan 25 12:24:38 2017 +0530
Committer: satishsaley <sa...@apache.org>
Committed: Fri Dec 8 16:34:55 2017 -0800
----------------------------------------------------------------------
release-log.txt | 1 +
.../oozie/server/SSLServerConnectorFactory.java | 167 +++++++++++++++++++
2 files changed, 168 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/6e5dccf7/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index e1026d6..147149e 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.3.1 release
+OOZIE-2771 Allow retrieving keystore and truststore passwords from Hadoop Credential Provider (asasvari via abhishekbafna)
OOZIE-2748 NPE in LauncherMapper.printArgs() (pbacsko via rkanter)
OOZIE-2654 Zookeeper dependent services should not depend on Connectionstate to be valid before cleaning up (venkatnrangan via abhishekbafna)
OOZIE-2690 OOZIE NPE while executing kill() (abhishekbafna via jaydeepvishwakarma)
http://git-wip-us.apache.org/repos/asf/oozie/blob/6e5dccf7/server/src/main/java/org/apache/oozie/server/SSLServerConnectorFactory.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/oozie/server/SSLServerConnectorFactory.java b/server/src/main/java/org/apache/oozie/server/SSLServerConnectorFactory.java
new file mode 100644
index 0000000..a7253d7
--- /dev/null
+++ b/server/src/main/java/org/apache/oozie/server/SSLServerConnectorFactory.java
@@ -0,0 +1,167 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oozie.server;
+
+
+import com.google.common.base.Preconditions;
+import com.google.inject.Inject;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.oozie.service.ConfigurationService;
+import org.eclipse.jetty.http.HttpVersion;
+import org.eclipse.jetty.server.HttpConfiguration;
+import org.eclipse.jetty.server.HttpConnectionFactory;
+import org.eclipse.jetty.server.SecureRequestCustomizer;
+import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.server.ServerConnector;
+import org.eclipse.jetty.server.SslConnectionFactory;
+import org.eclipse.jetty.util.ssl.SslContextFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Arrays;
+
+/**
+ * Factory that is used to configure SSL settings for the Oozie server.
+ */
+class SSLServerConnectorFactory {
+ private static final Logger LOG = LoggerFactory.getLogger(SSLServerConnectorFactory.class);
+ public static final String OOZIE_HTTPS_TRUSTSTORE_FILE = "oozie.https.truststore.file";
+ public static final String OOZIE_HTTPS_TRUSTSTORE_PASS = "oozie.https.truststore.pass";
+ public static final String OOZIE_HTTPS_KEYSTORE_PASS = "oozie.https.keystore.pass";
+ public static final String OOZIE_HTTPS_KEYSTORE_FILE = "oozie.https.keystore.file";
+ public static final String OOZIE_HTTPS_EXCLUDE_PROTOCOLS = "oozie.https.exclude.protocols";
+ public static final String OOZIE_HTTPS_INCLUDE_PROTOCOLS = "oozie.https.include.protocols";
+ public static final String OOZIE_HTTPS_INCLUDE_CIPHER_SUITES = "oozie.https.include.cipher.suites";
+ public static final String OOZIE_HTTPS_EXCLUDE_CIPHER_SUITES = "oozie.https.exclude.cipher.suites";
+
+ private SslContextFactory sslContextFactory;
+ private Configuration conf;
+
+ @Inject
+ public SSLServerConnectorFactory(final SslContextFactory sslContextFactory) {
+ this.sslContextFactory = Preconditions.checkNotNull(sslContextFactory, "sslContextFactory is null");
+ }
+
+ /**
+ * Construct a ServerConnector object with SSL settings
+ *
+ * @param oozieHttpsPort Oozie HTTPS port
+ * @param conf Oozie configuration
+ * @param server jetty Server which the connector is attached to
+ *
+ * @return ServerConnector
+ */
+ public ServerConnector createSecureServerConnector(int oozieHttpsPort, Configuration conf, Server server) {
+ this.conf = Preconditions.checkNotNull(conf, "conf is null");
+ Preconditions.checkNotNull(server, "server is null");
+ Preconditions.checkState(oozieHttpsPort >= 1 && oozieHttpsPort <= 65535,
+ String.format("Invalid port number specified: \'%d\'. It should be between 1 and 65535.", oozieHttpsPort));
+
+ setIncludeProtocols();
+ setExcludeProtocols();
+
+ setIncludeCipherSuites();
+ setExludeCipherSuites();
+
+ setTrustStorePath();
+ setTrustStorePass();
+
+ setKeyStoreFile();
+ setKeystorePass();
+
+ HttpConfiguration httpsConfiguration = getHttpsConfiguration();
+ ServerConnector secureServerConnector = new ServerConnector(server,
+ new SslConnectionFactory(sslContextFactory, HttpVersion.HTTP_1_1.asString()),
+ new HttpConnectionFactory(httpsConfiguration));
+
+ secureServerConnector.setPort(oozieHttpsPort);
+
+ LOG.info(String.format("Secure server connector created, listenning on port %d", oozieHttpsPort));
+ return secureServerConnector;
+ }
+
+ private void setExludeCipherSuites() {
+ String excludeCipherList = conf.get(OOZIE_HTTPS_EXCLUDE_CIPHER_SUITES);
+ String[] excludeCipherSuites = excludeCipherList.split(",");
+ sslContextFactory.setExcludeCipherSuites(excludeCipherSuites);
+
+ LOG.info(String.format("SSL context - excluding cipher suites: %s", Arrays.toString(excludeCipherSuites)));
+ }
+
+ private void setIncludeCipherSuites() {
+ String includeCipherList = conf.get(OOZIE_HTTPS_INCLUDE_CIPHER_SUITES);
+ if (includeCipherList == null || includeCipherList.isEmpty()) {
+ return;
+ }
+
+ String[] includeCipherSuites = includeCipherList.split(",");
+ sslContextFactory.setIncludeCipherSuites(includeCipherSuites);
+
+ LOG.info(String.format("SSL context - including cipher suites: %s", Arrays.toString(includeCipherSuites)));
+ }
+
+ private void setIncludeProtocols() {
+ String enabledProtocolsList = conf.get(OOZIE_HTTPS_INCLUDE_PROTOCOLS);
+ String[] enabledProtocols = enabledProtocolsList.split(",");
+ sslContextFactory.setIncludeProtocols(enabledProtocols);
+
+ LOG.info(String.format("SSL context - including protocols: %s", Arrays.toString(enabledProtocols)));
+ }
+
+ private void setExcludeProtocols() {
+ String excludedProtocolsList = conf.get(OOZIE_HTTPS_EXCLUDE_PROTOCOLS);
+ if (excludedProtocolsList == null || excludedProtocolsList.isEmpty()) {
+ return;
+ }
+ String[] excludedProtocols = excludedProtocolsList.split(",");
+ sslContextFactory.setExcludeProtocols(excludedProtocols);
+ LOG.info(String.format("SSL context - excluding protocols: %s", Arrays.toString(excludedProtocols)));
+ }
+
+ private void setTrustStorePath() {
+ String trustStorePath = conf.get(OOZIE_HTTPS_TRUSTSTORE_FILE);
+ Preconditions.checkNotNull(trustStorePath, "trustStorePath is null");
+ sslContextFactory.setTrustStorePath(trustStorePath);
+ }
+
+ private void setTrustStorePass() {
+ String trustStorePass = ConfigurationService.getPassword(conf, OOZIE_HTTPS_TRUSTSTORE_PASS).trim();
+ Preconditions.checkNotNull(trustStorePass, "setTrustStorePass is null");
+ sslContextFactory.setTrustStorePassword(trustStorePass);
+ }
+
+ private void setKeystorePass() {
+ String keystorePass = ConfigurationService.getPassword(conf, OOZIE_HTTPS_KEYSTORE_PASS).trim();
+ Preconditions.checkNotNull(keystorePass, "keystorePass is null");
+ sslContextFactory.setKeyManagerPassword(keystorePass);
+ }
+
+ private void setKeyStoreFile() {
+ String keystoreFile = conf.get(OOZIE_HTTPS_KEYSTORE_FILE);
+ Preconditions.checkNotNull(keystoreFile, "keystoreFile is null");
+ sslContextFactory.setKeyStorePath(keystoreFile);
+ }
+
+ private HttpConfiguration getHttpsConfiguration() {
+ HttpConfiguration https = new HttpConfigurationWrapper(conf).getDefaultHttpConfiguration();
+ https.setSecureScheme("https");
+ https.addCustomizer(new SecureRequestCustomizer());
+ return https;
+ }
+}
[07/31] oozie git commit: OOZIE-2654 Zookeeper dependent services
should not depend on Connectionstate to be valid before cleaning up
(venkatnrangan via abhishekbafna)
Posted by sa...@apache.org.
OOZIE-2654 Zookeeper dependent services should not depend on Connectionstate to be valid before cleaning up (venkatnrangan via abhishekbafna)
(cherry picked from commit 1431ca4eca747df44073bc076339e13185305f30)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/a0c0ed99
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/a0c0ed99
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/a0c0ed99
Branch: refs/heads/branch-4.3
Commit: a0c0ed99cca79974300cfe191db948eb5d681376
Parents: 51eb9a8
Author: abhisek bafna <ab...@hortonworks.com>
Authored: Fri Jan 6 13:07:03 2017 +0530
Committer: satishsaley <sa...@apache.org>
Committed: Fri Dec 8 16:34:55 2017 -0800
----------------------------------------------------------------------
.../java/org/apache/oozie/service/ZKJobsConcurrencyService.java | 2 +-
core/src/main/java/org/apache/oozie/service/ZKLocksService.java | 2 +-
core/src/main/java/org/apache/oozie/util/ZKUtils.java | 4 +---
release-log.txt | 1 +
4 files changed, 4 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/a0c0ed99/core/src/main/java/org/apache/oozie/service/ZKJobsConcurrencyService.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/service/ZKJobsConcurrencyService.java b/core/src/main/java/org/apache/oozie/service/ZKJobsConcurrencyService.java
index a64f613..93f5bf8 100644
--- a/core/src/main/java/org/apache/oozie/service/ZKJobsConcurrencyService.java
+++ b/core/src/main/java/org/apache/oozie/service/ZKJobsConcurrencyService.java
@@ -83,7 +83,7 @@ public class ZKJobsConcurrencyService extends JobsConcurrencyService implements
*/
@Override
public void destroy() {
- if (leaderLatch != null && ZKConnectionListener.getZKConnectionState() != ConnectionState.LOST) {
+ if (leaderLatch != null) {
IOUtils.closeSafely(leaderLatch);
}
if (zk != null) {
http://git-wip-us.apache.org/repos/asf/oozie/blob/a0c0ed99/core/src/main/java/org/apache/oozie/service/ZKLocksService.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/service/ZKLocksService.java b/core/src/main/java/org/apache/oozie/service/ZKLocksService.java
index 8acbad9..83790cf 100644
--- a/core/src/main/java/org/apache/oozie/service/ZKLocksService.java
+++ b/core/src/main/java/org/apache/oozie/service/ZKLocksService.java
@@ -84,7 +84,7 @@ public class ZKLocksService extends MemoryLocksService implements Service, Instr
*/
@Override
public void destroy() {
- if (reaper != null && ZKConnectionListener.getZKConnectionState() != ConnectionState.LOST) {
+ if (reaper != null) {
try {
reaper.close();
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/a0c0ed99/core/src/main/java/org/apache/oozie/util/ZKUtils.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/util/ZKUtils.java b/core/src/main/java/org/apache/oozie/util/ZKUtils.java
index 8069641..5835fb2 100644
--- a/core/src/main/java/org/apache/oozie/util/ZKUtils.java
+++ b/core/src/main/java/org/apache/oozie/util/ZKUtils.java
@@ -177,9 +177,7 @@ public class ZKUtils {
// If there are no more classes using ZooKeeper, we should teardown everything.
users.remove(user);
if (users.isEmpty() && zk != null) {
- if (ZKConnectionListener.getZKConnectionState() != ConnectionState.LOST) {
- zk.teardown();
- }
+ zk.teardown();
zk = null;
}
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/a0c0ed99/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 648e52b..0f70852 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.3.1 release
+OOZIE-2654 Zookeeper dependent services should not depend on Connectionstate to be valid before cleaning up (venkatnrangan via abhishekbafna)
OOZIE-2690 OOZIE NPE while executing kill() (abhishekbafna via jaydeepvishwakarma)
-- Oozie 4.3.0 release
[24/31] oozie git commit: OOZIE-2792 Hive2 action is not parsing
Spark application ID from log file properly when Hive is on Spark
(zhengxb2005 via rkanter)
Posted by sa...@apache.org.
OOZIE-2792 Hive2 action is not parsing Spark application ID from log file properly when Hive is on Spark (zhengxb2005 via rkanter)
(cherry picked from commit 4483f6dd94daa0aee1bdaf19cbb4b4a68cbefa8c)
sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/22aae95c
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/22aae95c
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/22aae95c
Branch: refs/heads/branch-4.3
Commit: 22aae95c23d16abf3ee647abde561355b6bc9fb9
Parents: fb44f42
Author: Robert Kanter <rk...@apache.org>
Authored: Tue Feb 14 16:40:46 2017 -0800
Committer: satishsaley <sa...@apache.org>
Committed: Fri Dec 8 16:34:55 2017 -0800
----------------------------------------------------------------------
.../oozie/action/hadoop/TestDistcpMain.java | 26 +++++++++-
release-log.txt | 1 +
.../apache/oozie/action/hadoop/DistcpMain.java | 6 ++-
.../apache/oozie/action/hadoop/HiveMain.java | 5 +-
.../oozie/action/hadoop/TestHiveMain.java | 35 +++++++++++--
.../apache/oozie/action/hadoop/Hive2Main.java | 8 ++-
.../oozie/action/hadoop/TestHive2Main.java | 52 ++++++++++++++++++++
.../oozie/action/hadoop/LauncherMain.java | 32 +++++++-----
.../org/apache/oozie/action/hadoop/PigMain.java | 6 ++-
.../apache/oozie/action/hadoop/TestPigMain.java | 23 +++++++++
.../apache/oozie/action/hadoop/SparkMain.java | 3 +-
.../oozie/action/hadoop/TestSparkMain.java | 18 +++++++
.../apache/oozie/action/hadoop/SqoopMain.java | 6 ++-
.../oozie/action/hadoop/TestSqoopMain.java | 49 ++++++++++++++++++
14 files changed, 247 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/22aae95c/core/src/test/java/org/apache/oozie/action/hadoop/TestDistcpMain.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestDistcpMain.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestDistcpMain.java
index bf08ccb..199962e 100644
--- a/core/src/test/java/org/apache/oozie/action/hadoop/TestDistcpMain.java
+++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestDistcpMain.java
@@ -23,10 +23,13 @@ import org.apache.oozie.util.XConfiguration;
import java.io.File;
import java.io.FileOutputStream;
-import java.io.IOException;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Writer;
+import java.util.ArrayList;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
import org.apache.oozie.action.hadoop.DistcpMain;
@@ -95,4 +98,25 @@ public class TestDistcpMain extends MainTestCase {
new File(getTestCaseDir(), "action.xml").delete();
return null;
}
+
+ public void testJobIDPattern() {
+ List<String> lines = new ArrayList<String>();
+ lines.add("Job complete: job_001");
+ lines.add("Job job_002 completed successfully");
+ lines.add("Submitted application application_003");
+ // Non-matching ones
+ lines.add("Job complete: job004");
+ lines.add("Job complete: (job_005");
+ lines.add("Job abc job_006 completed successfully");
+ lines.add("Submitted application. application_007");
+ Set<String> jobIds = new LinkedHashSet<String>();
+ for (String line : lines) {
+ LauncherMain.extractJobIDs(line, DistcpMain.DISTCP_JOB_IDS_PATTERNS, jobIds);
+ }
+ Set<String> expected = new LinkedHashSet<String>();
+ expected.add("job_001");
+ expected.add("job_002");
+ expected.add("job_003");
+ assertEquals(expected, jobIds);
+ }
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/22aae95c/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 76bab5f..b816b95 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.3.1 release
+OOZIE-2792 Hive2 action is not parsing Spark application ID from log file properly when Hive is on Spark (zhengxb2005 via rkanter)
OOZIE-2786 Pass Oozie workflow ID and settings to Spark application configuration (zhengxb2005 via rkanter)
OOZIE-2790 log4j configuration is not passed to spark executors (satishsaley)
OOZIE-2787 Oozie distributes application jar twice making the spark job fail (satishsaley)
http://git-wip-us.apache.org/repos/asf/oozie/blob/22aae95c/sharelib/distcp/src/main/java/org/apache/oozie/action/hadoop/DistcpMain.java
----------------------------------------------------------------------
diff --git a/sharelib/distcp/src/main/java/org/apache/oozie/action/hadoop/DistcpMain.java b/sharelib/distcp/src/main/java/org/apache/oozie/action/hadoop/DistcpMain.java
index 2b84f92..fcaadef 100644
--- a/sharelib/distcp/src/main/java/org/apache/oozie/action/hadoop/DistcpMain.java
+++ b/sharelib/distcp/src/main/java/org/apache/oozie/action/hadoop/DistcpMain.java
@@ -33,12 +33,16 @@ import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
import org.apache.log4j.PropertyConfigurator;
+import com.google.common.annotations.VisibleForTesting;
+
public class DistcpMain extends JavaMain {
private Constructor<?> construct;
private Object[] constArgs;
private static final String DISTCP_LOG4J_PROPS = "distcp-log4j.properties";
- private static final Pattern[] DISTCP_JOB_IDS_PATTERNS = {
+
+ @VisibleForTesting
+ static final Pattern[] DISTCP_JOB_IDS_PATTERNS = {
Pattern.compile("Job complete: (job_\\S*)"),
Pattern.compile("Job (job_\\S*) completed successfully"),
Pattern.compile("Submitted application (application[0-9_]*)")
http://git-wip-us.apache.org/repos/asf/oozie/blob/22aae95c/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java
----------------------------------------------------------------------
diff --git a/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java b/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java
index bbcaefa..1e47989 100644
--- a/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java
+++ b/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java
@@ -40,8 +40,11 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.cli.CliDriver;
import org.apache.hadoop.hive.conf.HiveConf;
+import com.google.common.annotations.VisibleForTesting;
+
public class HiveMain extends LauncherMain {
- private static final Pattern[] HIVE_JOB_IDS_PATTERNS = {
+ @VisibleForTesting
+ static final Pattern[] HIVE_JOB_IDS_PATTERNS = {
Pattern.compile("Ended Job = (job_\\S*)"),
Pattern.compile("Submitted application (application[0-9_]*)")
};
http://git-wip-us.apache.org/repos/asf/oozie/blob/22aae95c/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveMain.java
----------------------------------------------------------------------
diff --git a/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveMain.java b/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveMain.java
index 3026311..2ba0da7 100644
--- a/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveMain.java
+++ b/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveMain.java
@@ -18,10 +18,6 @@
package org.apache.oozie.action.hadoop;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.oozie.test.MiniHCatServer;
-import org.apache.oozie.util.XConfiguration;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FileWriter;
@@ -29,16 +25,27 @@ import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.net.URL;
+import java.util.ArrayList;
+import java.util.LinkedHashSet;
+import java.util.List;
import java.util.Properties;
+import java.util.Set;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.oozie.test.MiniHCatServer;
+import org.apache.oozie.util.XConfiguration;
public class TestHiveMain extends MainTestCase {
private SecurityManager SECURITY_MANAGER;
+ @Override
protected void setUp() throws Exception {
super.setUp();
SECURITY_MANAGER = System.getSecurityManager();
}
+ @Override
protected void tearDown() throws Exception {
System.setSecurityManager(SECURITY_MANAGER);
super.tearDown();
@@ -61,6 +68,7 @@ public class TestHiveMain extends MainTestCase {
return buffer.toString();
}
+ @Override
public Void call() throws Exception {
if (System.getenv("HADOOP_HOME") == null) {
System.out.println("WARNING: 'HADOOP_HOME' env var not defined, TestHiveMain test is not running");
@@ -165,4 +173,23 @@ public class TestHiveMain extends MainTestCase {
}
return null;
}
+
+ public void testJobIDPattern() {
+ List<String> lines = new ArrayList<String>();
+ lines.add("Ended Job = job_001");
+ lines.add("Submitted application application_002");
+ // Non-matching ones
+ lines.add("Ended Job = . job_003");
+ lines.add("Ended Job = abc004");
+ lines.add("Submitted application = job_005");
+ lines.add("Submitted application. job_006");
+ Set<String> jobIds = new LinkedHashSet<String>();
+ for (String line : lines) {
+ LauncherMain.extractJobIDs(line, HiveMain.HIVE_JOB_IDS_PATTERNS, jobIds);
+ }
+ Set<String> expected = new LinkedHashSet<String>();
+ expected.add("job_001");
+ expected.add("job_002");
+ assertEquals(expected, jobIds);
+ }
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/22aae95c/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java
----------------------------------------------------------------------
diff --git a/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java b/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java
index 54f7039..8e214fa 100644
--- a/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java
+++ b/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java
@@ -37,10 +37,14 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hive.beeline.BeeLine;
+import com.google.common.annotations.VisibleForTesting;
+
public class Hive2Main extends LauncherMain {
- private static final Pattern[] HIVE2_JOB_IDS_PATTERNS = {
+ @VisibleForTesting
+ static final Pattern[] HIVE2_JOB_IDS_PATTERNS = {
Pattern.compile("Ended Job = (job_\\S*)"),
- Pattern.compile("Submitted application (application[0-9_]*)")
+ Pattern.compile("Submitted application (application[0-9_]*)"),
+ Pattern.compile("Running with YARN Application = (application_[0-9]+)")
};
private static final Set<String> DISALLOWED_BEELINE_OPTIONS = new HashSet<String>();
http://git-wip-us.apache.org/repos/asf/oozie/blob/22aae95c/sharelib/hive2/src/test/java/org/apache/oozie/action/hadoop/TestHive2Main.java
----------------------------------------------------------------------
diff --git a/sharelib/hive2/src/test/java/org/apache/oozie/action/hadoop/TestHive2Main.java b/sharelib/hive2/src/test/java/org/apache/oozie/action/hadoop/TestHive2Main.java
new file mode 100644
index 0000000..b94b7c3
--- /dev/null
+++ b/sharelib/hive2/src/test/java/org/apache/oozie/action/hadoop/TestHive2Main.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oozie.action.hadoop;
+
+import java.util.ArrayList;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.oozie.test.XTestCase;
+
+public class TestHive2Main extends XTestCase {
+
+ public void testJobIDPattern() {
+ List<String> lines = new ArrayList<String>();
+ lines.add("Ended Job = job_001");
+ lines.add("Running with YARN Application = application_002");
+ lines.add("Submitted application application_003");
+ // Non-matching ones
+ lines.add("Ended Job = ");
+ lines.add("Ended Job = abc");
+ lines.add("Running with YARN Application = job_004");
+ lines.add("Running with YARN Application = application005");
+ lines.add("Running with YARN Application = application_abc");
+ lines.add("Submitted application job_006");
+ Set<String> jobIds = new LinkedHashSet<String>();
+ for (String line : lines) {
+ LauncherMain.extractJobIDs(line, Hive2Main.HIVE2_JOB_IDS_PATTERNS,
+ jobIds);
+ }
+ Set<String> expected = new LinkedHashSet<String>();
+ expected.add("job_001");
+ expected.add("job_002");
+ expected.add("job_003");
+ assertEquals(expected, jobIds);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/oozie/blob/22aae95c/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java
----------------------------------------------------------------------
diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java
index 785ca5e..8087178 100644
--- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java
+++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java
@@ -38,6 +38,10 @@ import java.util.regex.Pattern;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.util.Shell;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.mapred.JobConf;
@@ -69,17 +73,7 @@ public abstract class LauncherMain {
try (BufferedReader br = new BufferedReader(new FileReader(logFile))) {
String line = br.readLine();
while (line != null) {
- for (Pattern pattern : patterns) {
- Matcher matcher = pattern.matcher(line);
- if (matcher.find()) {
- String jobId = matcher.group(1);
- if (StringUtils.isEmpty(jobId) || jobId.equalsIgnoreCase("NULL")) {
- continue;
- }
- jobId = jobId.replaceAll("application", "job");
- jobIds.add(jobId);
- }
- }
+ extractJobIDs(line, patterns, jobIds);
line = br.readLine();
}
} catch (IOException e) {
@@ -90,6 +84,22 @@ public abstract class LauncherMain {
return jobIds.isEmpty() ? null : StringUtils.join(jobIds, ",");
}
+ @VisibleForTesting
+ protected static void extractJobIDs(String line, Pattern[] patterns, Set<String> jobIds) {
+ Preconditions.checkNotNull(line);
+ for (Pattern pattern : patterns) {
+ Matcher matcher = pattern.matcher(line);
+ if (matcher.find()) {
+ String jobId = matcher.group(1);
+ if (StringUtils.isEmpty(jobId) || jobId.equalsIgnoreCase("NULL")) {
+ continue;
+ }
+ jobId = jobId.replaceAll("application", "job");
+ jobIds.add(jobId);
+ }
+ }
+ }
+
protected static void writeExternalChildIDs(String logFile, Pattern[] patterns, String name) {
// Harvesting and recording Hadoop Job IDs
String jobIds = getHadoopJobIds(logFile, patterns);
http://git-wip-us.apache.org/repos/asf/oozie/blob/22aae95c/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java
----------------------------------------------------------------------
diff --git a/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java b/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java
index 87c4cc4..db194af 100644
--- a/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java
+++ b/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java
@@ -22,6 +22,9 @@ import org.apache.pig.Main;
import org.apache.pig.PigRunner;
import org.apache.pig.tools.pigstats.JobStats;
import org.apache.pig.tools.pigstats.PigStats;
+
+import com.google.common.annotations.VisibleForTesting;
+
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
@@ -49,7 +52,8 @@ public class PigMain extends LauncherMain {
private static final Set<String> DISALLOWED_PIG_OPTIONS = new HashSet<String>();
public static final int STRING_BUFFER_SIZE = 100;
- private static final Pattern[] PIG_JOB_IDS_PATTERNS = {
+ @VisibleForTesting
+ static final Pattern[] PIG_JOB_IDS_PATTERNS = {
Pattern.compile("HadoopJobId: (job_\\S*)"),
Pattern.compile("Submitted application (application[0-9_]*)")
};
http://git-wip-us.apache.org/repos/asf/oozie/blob/22aae95c/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigMain.java
----------------------------------------------------------------------
diff --git a/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigMain.java b/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigMain.java
index 081d319..22e7952 100644
--- a/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigMain.java
+++ b/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigMain.java
@@ -31,8 +31,12 @@ import java.io.OutputStreamWriter;
import java.io.Writer;
import java.io.FileWriter;
import java.io.FileReader;
+import java.util.ArrayList;
+import java.util.LinkedHashSet;
+import java.util.List;
import java.util.Map;
import java.util.Properties;
+import java.util.Set;
import java.net.URL;
public class TestPigMain extends PigTestCase {
@@ -157,4 +161,23 @@ public class TestPigMain extends PigTestCase {
return null;
}
+ public void testJobIDPattern() {
+ List<String> lines = new ArrayList<String>();
+ lines.add("HadoopJobId: job_001");
+ lines.add("Submitted application application_002");
+ // Non-matching ones
+ lines.add("HadoopJobId is set. job_003");
+ lines.add("HadoopJobId: abc004");
+ lines.add("Submitted application = job_005");
+ lines.add("Submitted application. job_006");
+ Set<String> jobIds = new LinkedHashSet<String>();
+ for (String line : lines) {
+ LauncherMain.extractJobIDs(line, PigMain.PIG_JOB_IDS_PATTERNS, jobIds);
+ }
+ Set<String> expected = new LinkedHashSet<String>();
+ expected.add("job_001");
+ expected.add("job_002");
+ assertEquals(expected, jobIds);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/22aae95c/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
----------------------------------------------------------------------
diff --git a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
index 12eb61a..3cc6ccb 100644
--- a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
+++ b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
@@ -66,7 +66,8 @@ public class SparkMain extends LauncherMain {
Pattern.compile("pyspark.zip") };
private static final Pattern SPARK_DEFAULTS_FILE_PATTERN = Pattern.compile("spark-defaults.conf");
private static final String SPARK_LOG4J_PROPS = "spark-log4j.properties";
- private static final Pattern[] SPARK_JOB_IDS_PATTERNS = {
+ @VisibleForTesting
+ static final Pattern[] SPARK_JOB_IDS_PATTERNS = {
Pattern.compile("Submitted application (application[0-9_]*)") };
public static final Pattern SPARK_ASSEMBLY_JAR_PATTERN = Pattern
.compile("^spark-assembly((?:(-|_|(\\d+\\.))\\d+(?:\\.\\d+)*))*\\.jar$");
http://git-wip-us.apache.org/repos/asf/oozie/blob/22aae95c/sharelib/spark/src/test/java/org/apache/oozie/action/hadoop/TestSparkMain.java
----------------------------------------------------------------------
diff --git a/sharelib/spark/src/test/java/org/apache/oozie/action/hadoop/TestSparkMain.java b/sharelib/spark/src/test/java/org/apache/oozie/action/hadoop/TestSparkMain.java
index bf4adf9..5f6a0cd 100644
--- a/sharelib/spark/src/test/java/org/apache/oozie/action/hadoop/TestSparkMain.java
+++ b/sharelib/spark/src/test/java/org/apache/oozie/action/hadoop/TestSparkMain.java
@@ -26,7 +26,9 @@ import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.util.ArrayList;
+import java.util.LinkedHashSet;
import java.util.List;
+import java.util.Set;
import java.util.regex.Pattern;
import org.apache.hadoop.conf.Configuration;
@@ -137,4 +139,20 @@ public class TestSparkMain extends MainTestCase {
instance.appendOoziePropertiesToSparkConf(sparkArgs, actionConf);
assertEquals(Lists.newArrayList("--conf", "spark.oozie.bar=bar"), sparkArgs);
}
+
+ public void testJobIDPattern() {
+ List<String> lines = new ArrayList<String>();
+ lines.add("Submitted application application_001");
+ // Non-matching ones
+ lines.add("Submitted application job_002");
+ lines.add("HadoopJobId: application_003");
+ lines.add("Submitted application = application_004");
+ Set<String> jobIds = new LinkedHashSet<String>();
+ for (String line : lines) {
+ LauncherMain.extractJobIDs(line, SparkMain.SPARK_JOB_IDS_PATTERNS, jobIds);
+ }
+ Set<String> expected = new LinkedHashSet<String>();
+ expected.add("job_001");
+ assertEquals(expected, jobIds);
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/oozie/blob/22aae95c/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java
----------------------------------------------------------------------
diff --git a/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java b/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java
index cb65eb8..cb8f56e 100644
--- a/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java
+++ b/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java
@@ -32,11 +32,14 @@ import org.apache.hadoop.fs.Path;
import org.apache.log4j.PropertyConfigurator;
import org.apache.sqoop.Sqoop;
+import com.google.common.annotations.VisibleForTesting;
+
public class SqoopMain extends LauncherMain {
public static final String SQOOP_SITE_CONF = "sqoop-site.xml";
- private static final Pattern[] SQOOP_JOB_IDS_PATTERNS = {
+ @VisibleForTesting
+ static final Pattern[] SQOOP_JOB_IDS_PATTERNS = {
Pattern.compile("Job complete: (job_\\S*)"),
Pattern.compile("Job (job_\\S*) has completed successfully"),
Pattern.compile("Submitted application (application[0-9_]*)")
@@ -153,6 +156,7 @@ public class SqoopMain extends LauncherMain {
return logFile;
}
+ @Override
protected void run(String[] args) throws Exception {
System.out.println();
System.out.println("Oozie Sqoop action configuration");
http://git-wip-us.apache.org/repos/asf/oozie/blob/22aae95c/sharelib/sqoop/src/test/java/org/apache/oozie/action/hadoop/TestSqoopMain.java
----------------------------------------------------------------------
diff --git a/sharelib/sqoop/src/test/java/org/apache/oozie/action/hadoop/TestSqoopMain.java b/sharelib/sqoop/src/test/java/org/apache/oozie/action/hadoop/TestSqoopMain.java
new file mode 100644
index 0000000..d6f96d5
--- /dev/null
+++ b/sharelib/sqoop/src/test/java/org/apache/oozie/action/hadoop/TestSqoopMain.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oozie.action.hadoop;
+
+import java.util.ArrayList;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.oozie.test.XTestCase;
+
+public class TestSqoopMain extends XTestCase {
+
+ public void testJobIDPattern() {
+ List<String> lines = new ArrayList<String>();
+ lines.add("Job complete: job_001");
+ lines.add("Job job_002 has completed successfully");
+ lines.add("Submitted application application_003");
+ // Non-matching ones
+ lines.add("Job complete: application_004");
+ lines.add("Job application_005 has completed successfully");
+ lines.add("Submitted application job_006");
+ Set<String> jobIds = new LinkedHashSet<String>();
+ for (String line : lines) {
+ LauncherMain.extractJobIDs(line, SqoopMain.SQOOP_JOB_IDS_PATTERNS,
+ jobIds);
+ }
+ Set<String> expected = new LinkedHashSet<String>();
+ expected.add("job_001");
+ expected.add("job_002");
+ expected.add("job_003");
+ assertEquals(expected, jobIds);
+ }
+}
\ No newline at end of file
[20/31] oozie git commit: OOZIE-2811 amend Add support for filtering
out properties from SparkConfigurationService
Posted by sa...@apache.org.
OOZIE-2811 amend Add support for filtering out properties from SparkConfigurationService
(cherry picked from commit 669cea997b2d12310e86c0af57e5d3f98aef66dd)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/3f751ca2
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/3f751ca2
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/3f751ca2
Branch: refs/heads/branch-4.3
Commit: 3f751ca2b2e43c9137292f1274d083ae6c757d10
Parents: eeebb28
Author: Gezapeti Cseh <ge...@gmail.com>
Authored: Wed Mar 22 20:44:09 2017 +0100
Committer: satishsaley <sa...@apache.org>
Committed: Fri Dec 8 16:34:55 2017 -0800
----------------------------------------------------------------------
.../apache/oozie/service/TestSparkConfigurationService.java | 6 +++---
release-log.txt | 1 +
2 files changed, 4 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/3f751ca2/core/src/test/java/org/apache/oozie/service/TestSparkConfigurationService.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/service/TestSparkConfigurationService.java b/core/src/test/java/org/apache/oozie/service/TestSparkConfigurationService.java
index 0e00a45..1278efa 100644
--- a/core/src/test/java/org/apache/oozie/service/TestSparkConfigurationService.java
+++ b/core/src/test/java/org/apache/oozie/service/TestSparkConfigurationService.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
http://git-wip-us.apache.org/repos/asf/oozie/blob/3f751ca2/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index b073ba9..5427bad 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.3.1 release
+OOZIE-2811 amend Add support for filtering out properties from SparkConfigurationService
OOZIE-2807 Oozie gets RM delegation token even for checking job status (satishsaley)
OOZIE-2811 Add support for filtering out properties from SparkConfigurationService (gezapeti via rkanter)
OOZIE-2802 Spark action failure on Spark 2.1.0 due to duplicate sharelibs (gezapeti via rkanter)
[15/31] oozie git commit: amend OOZIE-2787 Oozie distributes
application jar twice making the spark job fail (satishsaley)
Posted by sa...@apache.org.
amend OOZIE-2787 Oozie distributes application jar twice making the spark job fail (satishsaley)
(cherry picked from commit 0da31f47ea975387c5d05763c632a5b8915bd9cc)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/fb44f427
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/fb44f427
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/fb44f427
Branch: refs/heads/branch-4.3
Commit: fb44f427ed6ac9ebd1b771c1fe2dec06c7370b4b
Parents: 3709ea2
Author: Satish Subhashrao Saley <sa...@yahoo-inc.com>
Authored: Thu Feb 9 23:00:35 2017 -0800
Committer: satishsaley <sa...@apache.org>
Committed: Fri Dec 8 16:34:55 2017 -0800
----------------------------------------------------------------------
.../site/twiki/DG_SparkActionExtension.twiki | 36 ++++++
.../apache/oozie/action/hadoop/SparkMain.java | 24 +++-
.../oozie/action/hadoop/TestJarFilter.java | 109 +++++++++++++++++++
3 files changed, 165 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/fb44f427/docs/src/site/twiki/DG_SparkActionExtension.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/DG_SparkActionExtension.twiki b/docs/src/site/twiki/DG_SparkActionExtension.twiki
index 74875bb..94175cd 100644
--- a/docs/src/site/twiki/DG_SparkActionExtension.twiki
+++ b/docs/src/site/twiki/DG_SparkActionExtension.twiki
@@ -200,6 +200,42 @@ The =jar= element indicates python file. Refer to the file by it's localized nam
in PySpark. The py file should be in the lib/ folder next to the workflow.xml or added using the =file= element so that
it's localized to the working directory with just its name.
+---+++ Using Symlink in <jar>
+
+A symlink must be specified using =[[WorkflowFunctionalSpec#a3.2.2.
+1_Adding_Files_and_Archives_for_the_Job][file]]= element. Then, you can use
+the symlink name in =jar= element.
+
+*Example:*
+
+Specifying relative path for symlink:
+
+Make sure that the file is within the application directory i.e. =oozie.wf.application.path= .
+<verbatim>
+ <spark xmlns="uri:oozie:spark-action:0.2">
+ ...
+ <jar>py-spark-example-symlink.py</jar>
+ ...
+ ...
+ <file>py-spark.py#py-spark-example-symlink.py</file>
+ ...
+ </spark>
+</verbatim>
+
+Specifying full path for symlink:
+<verbatim>
+ <spark xmlns="uri:oozie:spark-action:0.2">
+ ...
+ <jar>spark-example-symlink.jar</jar>
+ ...
+ ...
+ <file>hdfs://localhost:8020/user/testjars/all-oozie-examples.jar#spark-example-symlink.jar</file>
+ ...
+ </spark>
+</verbatim>
+
+
+
---++ Appendix, Spark XML-Schema
---+++ AE.A Appendix A, Spark XML-Schema
http://git-wip-us.apache.org/repos/asf/oozie/blob/fb44f427/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
----------------------------------------------------------------------
diff --git a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
index d37053d..12eb61a 100644
--- a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
+++ b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
@@ -520,7 +520,7 @@ public class SparkMain extends LauncherMain {
/**
* This class is used for filtering out unwanted jars.
*/
- private static class JarFilter {
+ static class JarFilter {
private String sparkVersion = "1.X.X";
private String sparkYarnJar;
private String applicationJar;
@@ -547,9 +547,10 @@ public class SparkMain extends LauncherMain {
*
* @throws OozieActionConfiguratorException
*/
- private void filter() throws OozieActionConfiguratorException {
+ public void filter() throws OozieActionConfiguratorException {
Iterator<URI> iterator = listUris.iterator();
File matchedFile = null;
+ Path applJarPath = new Path(applicationJar);
while (iterator.hasNext()) {
URI uri = iterator.next();
Path p = new Path(uri);
@@ -575,13 +576,28 @@ public class SparkMain extends LauncherMain {
// Here we skip the application jar, because
// (if uris are same,) it will get distributed multiple times
// - one time with --files and another time as application jar.
- if (p.getName().equals(applicationJar) || uri.toString().equals(applicationJar)) {
- applicationJar = uri.toString();
+ if (isApplicationJar(p.getName(), uri, applJarPath)) {
+ String fragment = uri.getFragment();
+ applicationJar = fragment != null && fragment.length() > 0 ? fragment : uri.toString();
iterator.remove();
}
}
}
+ /**
+ * Checks if a file is application jar
+ *
+ * @param fileName fileName name of the file
+ * @param fileUri fileUri URI of the file
+ * @param applJarPath Path of application jar
+ * @return true if fileName or fileUri is the application jar
+ */
+ private boolean isApplicationJar(String fileName, URI fileUri, Path applJarPath) {
+ return (fileName.equals(applicationJar) || fileUri.toString().equals(applicationJar)
+ || applJarPath.getName().equals(fileName)
+ || applicationJar.equals(fileUri.getFragment()));
+ }
+
public String getApplicationJar() {
return applicationJar;
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/fb44f427/sharelib/spark/src/test/java/org/apache/oozie/action/hadoop/TestJarFilter.java
----------------------------------------------------------------------
diff --git a/sharelib/spark/src/test/java/org/apache/oozie/action/hadoop/TestJarFilter.java b/sharelib/spark/src/test/java/org/apache/oozie/action/hadoop/TestJarFilter.java
new file mode 100644
index 0000000..2d4c83c
--- /dev/null
+++ b/sharelib/spark/src/test/java/org/apache/oozie/action/hadoop/TestJarFilter.java
@@ -0,0 +1,109 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oozie.action.hadoop;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.LinkedList;
+import java.util.jar.Attributes;
+import java.util.jar.JarOutputStream;
+import java.util.jar.Manifest;
+
+import org.apache.oozie.action.hadoop.SparkMain.JarFilter;
+import org.junit.Test;
+
+public class TestJarFilter {
+
+ @Test
+ public void testJarFilter() throws URISyntaxException, IOException, OozieActionConfiguratorException {
+ LinkedList<URI> listUris = new LinkedList<URI>();
+ String sparkVersion = "2.1.0";
+ String sparkYarnJar = "spark-yarn-" + sparkVersion + ".jar";
+ String applicationJarName = "oozie-examples.jar";
+ String renamedApplicationJar = "renamed-oozie-examples.jar";
+ URI sparkYarnJarUri = new URI("hdfs://localhost:8020/user/sparkjars/" + sparkYarnJar);
+ URI applicationJarUri = new URI("hdfs://localhost:8020/user/sparkdata/" + applicationJarName);
+ createSparkYarnJar(sparkYarnJar, sparkVersion);
+ populateUris(listUris, sparkYarnJarUri, applicationJarUri);
+
+ // check application jar, spark yarn jar and spark version
+ JarFilter jarFilter = new JarFilter(listUris, applicationJarUri.getPath());
+ jarFilter.filter();
+ assertEquals(applicationJarUri.toString(), jarFilter.getApplicationJar());
+ assertEquals(sparkYarnJarUri.toString(), jarFilter.getSparkYarnJar());
+ assertEquals(sparkVersion, jarFilter.getSparkVersion());
+ checkFilteredUris(listUris, sparkYarnJarUri.toString(), applicationJarUri.toString());
+ listUris.clear();
+
+ // check application jar with fragmented URI
+ applicationJarUri = new URI("hdfs", "localhost", "/user/sparkdata/" + applicationJarName,
+ renamedApplicationJar);
+ populateUris(listUris, sparkYarnJarUri, applicationJarUri);
+ jarFilter = new JarFilter(listUris, applicationJarUri.getPath());
+ jarFilter.filter();
+ assertEquals(renamedApplicationJar, jarFilter.getApplicationJar());
+ assertEquals(sparkYarnJarUri.toString(), jarFilter.getSparkYarnJar());
+ assertEquals(sparkVersion, jarFilter.getSparkVersion());
+ checkFilteredUris(listUris, sparkYarnJarUri.toString(), renamedApplicationJar);
+ listUris.clear();
+
+ // application jar is present in <file> with symlink
+ // and user mentioned the symlink name in <jar>
+ populateUris(listUris, sparkYarnJarUri, applicationJarUri);
+ jarFilter = new JarFilter(listUris, renamedApplicationJar);
+ jarFilter.filter();
+ assertEquals(renamedApplicationJar, jarFilter.getApplicationJar());
+ assertEquals(sparkYarnJarUri.toString(), jarFilter.getSparkYarnJar());
+ assertEquals(sparkVersion, jarFilter.getSparkVersion());
+ checkFilteredUris(listUris, sparkYarnJarUri.toString(), renamedApplicationJar);
+ }
+
+ private void checkFilteredUris(LinkedList<URI> listUris, String sparkYarnJar, String applicationJar) {
+ for(URI uri : listUris) {
+ assertFalse(uri.toString().contains(sparkYarnJar));
+ assertFalse(uri.toString().contains(applicationJar));
+ }
+ }
+
+ private void createSparkYarnJar(String sparkYarnJar, String sparkVersion)
+ throws FileNotFoundException, IOException {
+ Manifest manifest = new Manifest();
+ manifest.getMainAttributes().put(Attributes.Name.MANIFEST_VERSION, "1.0");
+ manifest.getMainAttributes().put(Attributes.Name.SPECIFICATION_VERSION, sparkVersion);
+ JarOutputStream target = new JarOutputStream(new FileOutputStream(sparkYarnJar), manifest);
+ target.flush();
+ target.close();
+ }
+
+ private void populateUris(LinkedList<URI> listUris, URI sparkYarnJarUri, URI applicationJarUri)
+ throws URISyntaxException {
+ String fileNamePattern = "hdfs://localhost:8020/user/sparkdata/%s%s%s";
+ listUris.add(new URI(String.format(fileNamePattern, "_SUCCESS", "#", "ouputflag.txt")));
+ listUris.add(new URI(String.format(fileNamePattern, "dependency.jar", "", "")));
+ listUris.add(new URI(String.format(fileNamePattern, "helper.jar", "", "")));
+ listUris.add(sparkYarnJarUri);
+ listUris.add(applicationJarUri);
+ }
+}
\ No newline at end of file
[05/31] oozie git commit: ammending release-log.txt for OOZIE-2690
Posted by sa...@apache.org.
ammending release-log.txt for OOZIE-2690
(cherry picked from commit 53554e89aeb472b0ccecfa31c9788580010834c6)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/51eb9a8b
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/51eb9a8b
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/51eb9a8b
Branch: refs/heads/branch-4.3
Commit: 51eb9a8bac6cb7749eeb410013c99ca84f4ca4bc
Parents: 96f57b9
Author: jvishwakarma <jv...@walmartlabs.com>
Authored: Mon Nov 21 15:13:37 2016 +0530
Committer: satishsaley <sa...@apache.org>
Committed: Fri Dec 8 16:34:55 2017 -0800
----------------------------------------------------------------------
release-log.txt | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/51eb9a8b/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 29173e59..648e52b 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,7 @@
-- Oozie 4.3.1 release
+OOZIE-2690 OOZIE NPE while executing kill() (abhishekbafna via jaydeepvishwakarma)
+
-- Oozie 4.3.0 release
OOZIE-2742 Unable to kill applications based on tag (satishsaley via rohini)
[26/31] oozie git commit: OOZIE-2886 Ensure consistent versioning of
hadoop jars in sharelibs (dbist13 via rkanter)
Posted by sa...@apache.org.
OOZIE-2886 Ensure consistent versioning of hadoop jars in sharelibs (dbist13 via rkanter)
(cherry picked from commit 230e4261162389d40443a15b6d0b837557577baf)
pom.xml
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/bbac99cf
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/bbac99cf
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/bbac99cf
Branch: refs/heads/branch-4.3
Commit: bbac99cf8941fe3b321a7fdcb83f5ffdcd42e08b
Parents: 7ea8f19
Author: Robert Kanter <rk...@apache.org>
Authored: Thu Jun 1 15:51:45 2017 -0700
Committer: satishsaley <sa...@apache.org>
Committed: Fri Dec 8 16:34:55 2017 -0800
----------------------------------------------------------------------
pom.xml | 36 ++++++++++++++++++++++++++++++++++++
release-log.txt | 1 +
sharelib/hive/pom.xml | 28 ++++++++++++++++++++++++++++
sharelib/hive2/pom.xml | 28 ++++++++++++++++++++++++++++
4 files changed, 93 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/bbac99cf/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 49d55e9..288aaec 100644
--- a/pom.xml
+++ b/pom.xml
@@ -809,6 +809,42 @@
</dependency>
<dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-api</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-common</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-applicationhistoryservice</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-common</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-resourcemanager</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-web-proxy</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+
+ <dependency>
<groupId>org.apache.pig</groupId>
<artifactId>pig</artifactId>
<version>${pig.version}</version>
http://git-wip-us.apache.org/repos/asf/oozie/blob/bbac99cf/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index a79d8c2..9834dfe 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.3.1 release
+OOZIE-2886 Ensure consistent versioning of hadoop jars in sharelibs (dbist13 via rkanter)
OOZIE-2872 Address backward compatibility issue introduced by OOZIE-2748 (pbacsko)
OOZIE-2863 SLACalculatorMemory.loadOnRestart causing delay in server start (puru via satishsaley)
OOZIE-2862 Coord change command doesn't change job to running if job was killed without creating any actions (puru)
http://git-wip-us.apache.org/repos/asf/oozie/blob/bbac99cf/sharelib/hive/pom.xml
----------------------------------------------------------------------
diff --git a/sharelib/hive/pom.xml b/sharelib/hive/pom.xml
index 2df0c90..f5b3f5e 100644
--- a/sharelib/hive/pom.xml
+++ b/sharelib/hive/pom.xml
@@ -91,6 +91,34 @@
<groupId>org.apache.hive</groupId>
<artifactId>hive-exec</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-annotations</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-common</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-applicationhistoryservice</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-common</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-resourcemanager</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-web-proxy</artifactId>
+ </exclusion>
</exclusions>
</dependency>
http://git-wip-us.apache.org/repos/asf/oozie/blob/bbac99cf/sharelib/hive2/pom.xml
----------------------------------------------------------------------
diff --git a/sharelib/hive2/pom.xml b/sharelib/hive2/pom.xml
index d0af590..951a06c 100644
--- a/sharelib/hive2/pom.xml
+++ b/sharelib/hive2/pom.xml
@@ -91,6 +91,34 @@
<groupId>org.apache.hive</groupId>
<artifactId>hive-exec</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-annotations</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-common</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-applicationhistoryservice</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-common</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-resourcemanager</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-web-proxy</artifactId>
+ </exclusion>
</exclusions>
</dependency>
[08/31] oozie git commit: OOZIE-2748 NPE in
LauncherMapper.printArgs() (pbacsko via rkanter)
Posted by sa...@apache.org.
OOZIE-2748 NPE in LauncherMapper.printArgs() (pbacsko via rkanter)
(cherry picked from commit c7fa12bb1abac643c60c7282626c39b690ae378b)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/3b24d9dd
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/3b24d9dd
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/3b24d9dd
Branch: refs/heads/branch-4.3
Commit: 3b24d9ddeb0c76be91083e2d93d70adf0b321125
Parents: a0c0ed9
Author: Robert Kanter <rk...@cloudera.com>
Authored: Fri Jan 6 10:36:26 2017 -0800
Committer: satishsaley <sa...@apache.org>
Committed: Fri Dec 8 16:34:55 2017 -0800
----------------------------------------------------------------------
.../action/hadoop/TestJavaActionExecutor.java | 26 ++++++
release-log.txt | 1 +
sharelib/oozie/pom.xml | 6 ++
.../oozie/action/hadoop/LauncherMapper.java | 17 +++-
.../oozie/action/hadoop/TestLauncherMapper.java | 88 ++++++++++++++++++++
5 files changed, 136 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/3b24d9dd/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
index 8965cdf..1c4b429 100644
--- a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
+++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
@@ -2941,4 +2941,30 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase {
assertEquals("DEBUG", conf.get(oozieActionHiveRootLogger));
}
+ public void testEmptyArgs() throws Exception {
+ String actionXml = "<java>" +
+ "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" +
+ "<name-node>" + getNameNodeUri() + "</name-node>" +
+ "<main-class>" + LauncherMainTester.class.getName() + "</main-class>" +
+ "<arg></arg>" +
+ "</java>";
+
+ Context context = createContext(actionXml, null);
+ final RunningJob runningJob = submitAction(context);
+ waitFor(60 * 1000, new Predicate() {
+ @Override
+ public boolean evaluate() throws Exception {
+ return runningJob.isComplete();
+ }
+ });
+ assertTrue(runningJob.isSuccessful());
+ ActionExecutor ae = new JavaActionExecutor();
+ ae.check(context, context.getAction());
+ assertTrue(ae.isCompleted(context.getAction().getExternalStatus()));
+ assertEquals("SUCCEEDED", context.getAction().getExternalStatus());
+ assertNull(context.getAction().getData());
+
+ ae.end(context, context.getAction());
+ assertEquals(WorkflowAction.Status.OK, context.getAction().getStatus());
+ }
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/3b24d9dd/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 0f70852..e1026d6 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.3.1 release
+OOZIE-2748 NPE in LauncherMapper.printArgs() (pbacsko via rkanter)
OOZIE-2654 Zookeeper dependent services should not depend on Connectionstate to be valid before cleaning up (venkatnrangan via abhishekbafna)
OOZIE-2690 OOZIE NPE while executing kill() (abhishekbafna via jaydeepvishwakarma)
http://git-wip-us.apache.org/repos/asf/oozie/blob/3b24d9dd/sharelib/oozie/pom.xml
----------------------------------------------------------------------
diff --git a/sharelib/oozie/pom.xml b/sharelib/oozie/pom.xml
index 8dfebd4..c30e984 100644
--- a/sharelib/oozie/pom.xml
+++ b/sharelib/oozie/pom.xml
@@ -62,6 +62,12 @@
</dependency>
<dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
<groupId>org.apache.oozie</groupId>
<artifactId>oozie-hadoop-utils</artifactId>
<scope>compile</scope>
http://git-wip-us.apache.org/repos/asf/oozie/blob/3b24d9dd/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java
----------------------------------------------------------------------
diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java
index 7271486..8edebac 100644
--- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java
+++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java
@@ -50,6 +50,8 @@ import org.apache.hadoop.mapred.Mapper;
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reporter;
+import com.google.common.base.Strings;
+
public class LauncherMapper<K1, V1, K2, V2> implements Mapper<K1, V1, K2, V2>, Runnable {
static final String CONF_OOZIE_ACTION_MAIN_CLASS = "oozie.launcher.action.main.class";
@@ -494,10 +496,21 @@ public class LauncherMapper<K1, V1, K2, V2> implements Mapper<K1, V1, K2, V2>, R
public static String[] getMainArguments(Configuration conf) {
String[] args = new String[conf.getInt(CONF_OOZIE_ACTION_MAIN_ARG_COUNT, 0)];
+
+ int pos = 0;
for (int i = 0; i < args.length; i++) {
- args[i] = conf.get(CONF_OOZIE_ACTION_MAIN_ARG_PREFIX + i);
+ String arg = conf.get(CONF_OOZIE_ACTION_MAIN_ARG_PREFIX + i);
+ if (!Strings.isNullOrEmpty(arg)) {
+ args[pos++] = conf.get(CONF_OOZIE_ACTION_MAIN_ARG_PREFIX + i);
+ }
}
- return args;
+
+ // this is to skip null args, that is <arg></arg> in the workflow XML -- in this case,
+ // args[] might look like {"arg1", "arg2", null, null} at this point
+ String[] retArray = new String[pos];
+ System.arraycopy(args, 0, retArray, 0, pos);
+
+ return retArray;
}
private void setupHeartBeater(Reporter reporter) {
http://git-wip-us.apache.org/repos/asf/oozie/blob/3b24d9dd/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/TestLauncherMapper.java
----------------------------------------------------------------------
diff --git a/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/TestLauncherMapper.java b/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/TestLauncherMapper.java
new file mode 100644
index 0000000..1dd8002
--- /dev/null
+++ b/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/TestLauncherMapper.java
@@ -0,0 +1,88 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oozie.action.hadoop;
+
+import static org.apache.oozie.action.hadoop.LauncherMapper.CONF_OOZIE_ACTION_MAIN_ARG_COUNT;
+import static org.apache.oozie.action.hadoop.LauncherMapper.CONF_OOZIE_ACTION_MAIN_ARG_PREFIX;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.BDDMockito.given;
+import static org.mockito.Matchers.eq;
+
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.runners.MockitoJUnitRunner;
+
+import com.google.common.collect.Lists;
+
+@RunWith(MockitoJUnitRunner.class)
+public class TestLauncherMapper {
+ @Mock
+ private Configuration conf; // we have to use mock, because conf.set(null) throws exception
+
+ @Test
+ public void testLauncherMapperArgsHandlingWithoutNulls() {
+ setupConf(Lists.newArrayList("a", "b", "c"));
+
+ String args[] = LauncherMapper.getMainArguments(conf);
+
+ assertTrue(Arrays.equals(new String[] { "a", "b", "c"}, args));
+ }
+
+ @Test
+ public void testLauncherMapperArgsHandlingWhenArgsContainNulls() {
+ setupConf(Lists.newArrayList("a", null, "b", null, "c"));
+
+ String args[] = LauncherMapper.getMainArguments(conf);
+
+ assertTrue(Arrays.equals(new String[] { "a", "b", "c"}, args));
+ }
+
+ @Test
+ public void testLauncherMapperArgsHandlingWhenArgsContainsNullsOnly() {
+ setupConf(Lists.<String>newArrayList(null, null, null));
+
+ String args[] = LauncherMapper.getMainArguments(conf);
+
+ assertTrue(Arrays.equals(new String[] {}, args));
+ }
+
+ @Test
+ public void testLauncherMapperArgsHandlingWhenArgsContainsOneNull() {
+ setupConf(Lists.<String>newArrayList((String) null));
+
+ String args[] = LauncherMapper.getMainArguments(conf);
+
+ assertTrue(Arrays.equals(new String[] {}, args));
+ }
+
+ private void setupConf(List<String> argList) {
+ int argCount = argList.size();
+
+ given(conf.getInt(eq(CONF_OOZIE_ACTION_MAIN_ARG_COUNT), eq(0))).willReturn(argCount);
+
+ for (int i = 0; i < argCount; i++) {
+ given(conf.get(eq(CONF_OOZIE_ACTION_MAIN_ARG_PREFIX + i))).willReturn(argList.get(i));
+ }
+ }
+}
[12/31] oozie git commit: OOZIE-2807 Oozie gets RM delegation token
even for checking job status (satishsaley)
Posted by sa...@apache.org.
OOZIE-2807 Oozie gets RM delegation token even for checking job status (satishsaley)
(cherry picked from commit 9c0516608bdd0787b1fac3a69ccfe9d36cbb2d15)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/eeebb28e
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/eeebb28e
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/eeebb28e
Branch: refs/heads/branch-4.3
Commit: eeebb28ec6135ff8eaccf1fc8a26cbfd5f11c6f0
Parents: f603225
Author: satishsaley <sa...@apache.org>
Authored: Fri Mar 10 14:28:26 2017 -0800
Committer: satishsaley <sa...@apache.org>
Committed: Fri Dec 8 16:34:55 2017 -0800
----------------------------------------------------------------------
.../oozie/action/hadoop/JavaActionExecutor.java | 5 +----
.../oozie/service/HadoopAccessorService.java | 23 ++++++++++++++++++--
release-log.txt | 1 +
3 files changed, 23 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/eeebb28e/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
index 4beeb96..8a403dc 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
@@ -1151,10 +1151,7 @@ public class JavaActionExecutor extends ActionExecutor {
LOG.debug("Submitting the job through Job Client for action " + action.getId());
// setting up propagation of the delegation token.
- HadoopAccessorService has = Services.get().get(HadoopAccessorService.class);
- Token<DelegationTokenIdentifier> mrdt = jobClient.getDelegationToken(has
- .getMRDelegationTokenRenewer(launcherJobConf));
- launcherJobConf.getCredentials().addToken(HadoopAccessorService.MR_TOKEN_ALIAS, mrdt);
+ Services.get().get(HadoopAccessorService.class).addRMDelegationToken(jobClient, launcherJobConf);
// insert credentials tokens to launcher job conf if needed
if (needInjectCredentials() && credentialsConf != null) {
http://git-wip-us.apache.org/repos/asf/oozie/blob/eeebb28e/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java b/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java
index 5377127..23a9d92 100644
--- a/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java
+++ b/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java
@@ -514,8 +514,6 @@ public class HadoopAccessorService implements Service {
return new JobClient(conf);
}
});
- Token<DelegationTokenIdentifier> mrdt = jobClient.getDelegationToken(getMRDelegationTokenRenewer(conf));
- conf.getCredentials().addToken(MR_TOKEN_ALIAS, mrdt);
return jobClient;
}
catch (InterruptedException ex) {
@@ -527,6 +525,27 @@ public class HadoopAccessorService implements Service {
}
/**
+ * Get the RM delegation token using jobClient and add it to conf
+ *
+ * @param jobClient
+ * @param conf
+ * @throws HadoopAccessorException
+ */
+ public void addRMDelegationToken(JobClient jobClient, JobConf conf) throws HadoopAccessorException {
+ Token<DelegationTokenIdentifier> mrdt;
+ try {
+ mrdt = jobClient.getDelegationToken(getMRDelegationTokenRenewer(conf));
+ }
+ catch (IOException e) {
+ throw new HadoopAccessorException(ErrorCode.E0902, e.getMessage(), e);
+ }
+ catch (InterruptedException e) {
+ throw new HadoopAccessorException(ErrorCode.E0902, e.getMessage(), e);
+ }
+ conf.getCredentials().addToken(MR_TOKEN_ALIAS, mrdt);
+ }
+
+ /**
* Return a FileSystem created with the provided user for the specified URI.
*
*
http://git-wip-us.apache.org/repos/asf/oozie/blob/eeebb28e/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 43ecb06..b073ba9 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.3.1 release
+OOZIE-2807 Oozie gets RM delegation token even for checking job status (satishsaley)
OOZIE-2811 Add support for filtering out properties from SparkConfigurationService (gezapeti via rkanter)
OOZIE-2802 Spark action failure on Spark 2.1.0 due to duplicate sharelibs (gezapeti via rkanter)
OOZIE-2803 Mask passwords when printing out configs/args in MapReduceMain and SparkMain (pbacsko via rkanter)
[11/31] oozie git commit: OOZIE-2787 Oozie distributes application
jar twice making the spark job fail (satishsaley)
Posted by sa...@apache.org.
OOZIE-2787 Oozie distributes application jar twice making the spark job fail (satishsaley)
(cherry picked from commit d249fbf43cd75b97f594330884189e8d8df55ddd)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/aae4948f
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/aae4948f
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/aae4948f
Branch: refs/heads/branch-4.3
Commit: aae4948f115bafc48db7d5c47bd8e223cd337781
Parents: f5b835b
Author: Satish Subhashrao Saley <sa...@yahoo-inc.com>
Authored: Mon Feb 6 09:40:44 2017 -0800
Committer: satishsaley <sa...@apache.org>
Committed: Fri Dec 8 16:34:55 2017 -0800
----------------------------------------------------------------------
pom.xml | 2 +-
release-log.txt | 1 +
.../apache/oozie/action/hadoop/SparkMain.java | 184 ++++++++++++-------
3 files changed, 122 insertions(+), 65 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/aae4948f/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index bfd0af7..36631a1 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1946,7 +1946,7 @@
<activeByDefault>false</activeByDefault>
</activation>
<properties>
- <spark.version>2.0.0</spark.version>
+ <spark.version>2.1.0</spark.version>
<spark.streaming.kafka.version>1.6.2</spark.streaming.kafka.version>
<spark.bagel.version>1.6.2</spark.bagel.version>
</properties>
http://git-wip-us.apache.org/repos/asf/oozie/blob/aae4948f/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 3ee166a..9e74443 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.3.1 release
+OOZIE-2787 Oozie distributes application jar twice making the spark job fail (satishsaley)
OOZIE-2777 Config-default.xml longer than 64k results in java.io.UTFDataFormatException (gezapeti via harsh)
OOZIE-2771 Allow retrieving keystore and truststore passwords from Hadoop Credential Provider (asasvari via abhishekbafna)
OOZIE-2748 NPE in LauncherMapper.printArgs() (pbacsko via rkanter)
http://git-wip-us.apache.org/repos/asf/oozie/blob/aae4948f/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
----------------------------------------------------------------------
diff --git a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
index 0da74d4..8f2f438 100644
--- a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
+++ b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
@@ -72,8 +72,6 @@ public class SparkMain extends LauncherMain {
private static final Pattern SPARK_VERSION_1 = Pattern.compile("^1.*");
private static final String SPARK_YARN_JAR = "spark.yarn.jar";
private static final String SPARK_YARN_JARS = "spark.yarn.jars";
- private String sparkYarnJar = null;
- private String sparkVersion = "1.X.X";
public static void main(String[] args) throws Exception {
run(SparkMain.class, args);
}
@@ -216,19 +214,22 @@ public class SparkMain extends LauncherMain {
}
if ((yarnClusterMode || yarnClientMode)) {
- LinkedList<URI> fixedUris = fixFsDefaultUris(DistributedCache.getCacheFiles(actionConf), jarPath);
- String cachedFiles = filterSparkYarnJar(fixedUris);
+ LinkedList<URI> fixedUris = fixFsDefaultUris(DistributedCache.getCacheFiles(actionConf));
+ JarFilter jarfilter = new JarFilter(fixedUris, jarPath);
+ jarfilter.filter();
+ jarPath = jarfilter.getApplicationJar();
+ String cachedFiles = StringUtils.join(fixedUris, ",");
if (cachedFiles != null && !cachedFiles.isEmpty()) {
sparkArgs.add("--files");
sparkArgs.add(cachedFiles);
}
- fixedUris = fixFsDefaultUris(DistributedCache.getCacheArchives(actionConf), jarPath);
+ fixedUris = fixFsDefaultUris(DistributedCache.getCacheArchives(actionConf));
String cachedArchives = StringUtils.join(fixedUris, ",");
if (cachedArchives != null && !cachedArchives.isEmpty()) {
sparkArgs.add("--archives");
sparkArgs.add(cachedArchives);
}
- setSparkYarnJarsConf(sparkArgs);
+ setSparkYarnJarsConf(sparkArgs, jarfilter.getSparkYarnJar(), jarfilter.getSparkVersion());
}
if (!sparkArgs.contains(VERBOSE_OPTION)) {
@@ -319,7 +320,7 @@ public class SparkMain extends LauncherMain {
* @param fileNamePattern the pattern to look for
* @return the file if there is one else it returns null
*/
- private File getMatchingFile(Pattern fileNamePattern) throws OozieActionConfiguratorException {
+ private static File getMatchingFile(Pattern fileNamePattern) throws OozieActionConfiguratorException {
File localDir = new File(".");
for(String fileName : localDir.list()){
if(fileNamePattern.matcher(fileName).find()){
@@ -424,7 +425,7 @@ public class SparkMain extends LauncherMain {
* @throws IOException
* @throws URISyntaxException
*/
- private LinkedList<URI> fixFsDefaultUris(URI[] files, String jarPath) throws IOException, URISyntaxException {
+ private LinkedList<URI> fixFsDefaultUris(URI[] files) throws IOException, URISyntaxException {
if (files == null) {
return null;
}
@@ -432,70 +433,19 @@ public class SparkMain extends LauncherMain {
FileSystem fs = FileSystem.get(new Configuration(true));
for (int i = 0; i < files.length; i++) {
URI fileUri = files[i];
- // Spark compares URIs based on scheme, host and port.
- // Here we convert URIs into the default format so that Spark
- // won't think those belong to different file system.
- // This will avoid an extra copy of files which already exists on
- // same hdfs.
- if (!fileUri.toString().equals(jarPath) && fs.getUri().getScheme().equals(fileUri.getScheme())
- && (fs.getUri().getHost().equals(fileUri.getHost()) || fileUri.getHost() == null)
- && (fs.getUri().getPort() == -1 || fileUri.getPort() == -1
- || fs.getUri().getPort() == fileUri.getPort())) {
- URI uri = new URI(fs.getUri().getScheme(), fileUri.getUserInfo(), fs.getUri().getHost(),
- fs.getUri().getPort(), fileUri.getPath(), fileUri.getQuery(), fileUri.getFragment());
- // Here we skip the application jar, because
- // (if uris are same,) it will get distributed multiple times
- // - one time with --files and another time as application jar.
- if (!uri.toString().equals(jarPath)) {
- listUris.add(uri);
- }
- }
+ listUris.add(getFixedUri(fs, fileUri));
}
return listUris;
}
/**
- * Filters out the Spark yarn jar and records its version
- *
- * @param listUris string containing uris separated by comma
- * @return
- * @throws OozieActionConfiguratorException
- */
- private String filterSparkYarnJar(LinkedList<URI> listUris) throws OozieActionConfiguratorException {
- Iterator<URI> iterator = listUris.iterator();
- File matchedFile = null;
- while (iterator.hasNext()) {
- URI uri = iterator.next();
- Path p = new Path(uri);
- if (SPARK_YARN_JAR_PATTERN.matcher(p.getName()).find()) {
- matchedFile = getMatchingFile(SPARK_YARN_JAR_PATTERN);
- }
- else if (SPARK_ASSEMBLY_JAR_PATTERN.matcher(p.getName()).find()) {
- matchedFile = getMatchingFile(SPARK_ASSEMBLY_JAR_PATTERN);
- }
- if (matchedFile != null) {
- sparkYarnJar = uri.toString();
- try {
- sparkVersion = getJarVersion(matchedFile);
- System.out.println("Spark Version " + sparkVersion);
- }
- catch (IOException io) {
- System.out.println(
- "Unable to open " + matchedFile.getPath() + ". Default Spark Version " + sparkVersion);
- }
- iterator.remove();
- break;
- }
- }
- return StringUtils.join(listUris, ",");
- }
-
- /**
* Sets spark.yarn.jars for Spark 2.X. Sets spark.yarn.jar for Spark 1.X.
*
* @param sparkArgs
+ * @param sparkYarnJar
+ * @param sparkVersion
*/
- private void setSparkYarnJarsConf(List<String> sparkArgs) {
+ private void setSparkYarnJarsConf(List<String> sparkArgs, String sparkYarnJar, String sparkVersion) {
if (SPARK_VERSION_1.matcher(sparkVersion).find()) {
// In Spark 1.X.X, set spark.yarn.jar to avoid
// multiple distribution
@@ -509,7 +459,7 @@ public class SparkMain extends LauncherMain {
}
}
- private String getJarVersion(File jarFile) throws IOException {
+ private static String getJarVersion(File jarFile) throws IOException {
@SuppressWarnings("resource")
Manifest manifest = new JarFile(jarFile).getManifest();
return manifest.getMainAttributes().getValue("Specification-Version");
@@ -521,4 +471,110 @@ public class SparkMain extends LauncherMain {
}
to.append(what);
}
+
+ private static URI getFixedUri(URI fileUri) throws URISyntaxException, IOException {
+ FileSystem fs = FileSystem.get(new Configuration(true));
+ return getFixedUri(fs, fileUri);
+ }
+
+ /**
+ * Spark compares URIs based on scheme, host and port. Here we convert URIs
+ * into the default format so that Spark won't think those belong to
+ * different file system. This will avoid an extra copy of files which
+ * already exists on same hdfs.
+ *
+ * @param fs
+ * @param fileUri
+ * @return fixed uri
+ * @throws URISyntaxException
+ */
+ private static URI getFixedUri(FileSystem fs, URI fileUri) throws URISyntaxException {
+ if (fs.getUri().getScheme().equals(fileUri.getScheme())
+ && (fs.getUri().getHost().equals(fileUri.getHost()) || fileUri.getHost() == null)
+ && (fs.getUri().getPort() == -1 || fileUri.getPort() == -1
+ || fs.getUri().getPort() == fileUri.getPort())) {
+ return new URI(fs.getUri().getScheme(), fileUri.getUserInfo(), fs.getUri().getHost(), fs.getUri().getPort(),
+ fileUri.getPath(), fileUri.getQuery(), fileUri.getFragment());
+ }
+ return fileUri;
+ }
+
+ /**
+ * This class is used for filtering out unwanted jars.
+ */
+ private static class JarFilter {
+ private String sparkVersion = "1.X.X";
+ private String sparkYarnJar;
+ private String applicationJar;
+ private LinkedList<URI> listUris = null;
+
+ /**
+ * @param listUris List of URIs to be filtered
+ * @param jarPath Application jar
+ * @throws IOException
+ * @throws URISyntaxException
+ */
+ public JarFilter(LinkedList<URI> listUris, String jarPath) throws URISyntaxException, IOException {
+ this.listUris = listUris;
+ applicationJar = jarPath;
+ Path p = new Path(jarPath);
+ if (p.isAbsolute()) {
+ applicationJar = getFixedUri(p.toUri()).toString();
+ }
+ }
+
+ /**
+ * Filters out the Spark yarn jar and application jar. Also records
+ * spark yarn jar's version.
+ *
+ * @throws OozieActionConfiguratorException
+ */
+ private void filter() throws OozieActionConfiguratorException {
+ Iterator<URI> iterator = listUris.iterator();
+ File matchedFile = null;
+ while (iterator.hasNext()) {
+ URI uri = iterator.next();
+ Path p = new Path(uri);
+ if (SPARK_YARN_JAR_PATTERN.matcher(p.getName()).find()) {
+ matchedFile = getMatchingFile(SPARK_YARN_JAR_PATTERN);
+ }
+ else if (SPARK_ASSEMBLY_JAR_PATTERN.matcher(p.getName()).find()) {
+ matchedFile = getMatchingFile(SPARK_ASSEMBLY_JAR_PATTERN);
+ }
+ if (matchedFile != null) {
+ sparkYarnJar = uri.toString();
+ try {
+ sparkVersion = getJarVersion(matchedFile);
+ System.out.println("Spark Version " + sparkVersion);
+ }
+ catch (IOException io) {
+ System.out.println(
+ "Unable to open " + matchedFile.getPath() + ". Default Spark Version " + sparkVersion);
+ }
+ iterator.remove();
+ matchedFile = null;
+ }
+ // Here we skip the application jar, because
+ // (if uris are same,) it will get distributed multiple times
+ // - one time with --files and another time as application jar.
+ if (p.getName().equals(applicationJar) || uri.toString().equals(applicationJar)) {
+ applicationJar = uri.toString();
+ iterator.remove();
+ }
+ }
+ }
+
+ public String getApplicationJar() {
+ return applicationJar;
+ }
+
+ public String getSparkYarnJar() {
+ return sparkYarnJar;
+ }
+
+ public String getSparkVersion() {
+ return sparkVersion;
+ }
+
+ }
}
[03/31] oozie git commit: OOZIE-2825 Custom Authentication doc page
is not well formatted (Jan Hentschel via rkanter)
Posted by sa...@apache.org.
OOZIE-2825 Custom Authentication doc page is not well formatted (Jan Hentschel via rkanter)
(cherry picked from commit 5983cdfe3d976c46f18970d77b003b637a83faf8)
docs/src/site/twiki/ENG_Custom_Authentication.twiki
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/636e434e
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/636e434e
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/636e434e
Branch: refs/heads/branch-4.3
Commit: 636e434ee8ae5e616b5fbf61f70b807adcffbd8b
Parents: f232030
Author: Robert Kanter <rk...@apache.org>
Authored: Tue Jun 13 17:38:04 2017 -0700
Committer: satishsaley <sa...@apache.org>
Committed: Fri Dec 8 16:34:55 2017 -0800
----------------------------------------------------------------------
.../site/twiki/ENG_Custom_Authentication.twiki | 106 ++++---------------
release-log.txt | 1 +
2 files changed, 21 insertions(+), 86 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/636e434e/docs/src/site/twiki/ENG_Custom_Authentication.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/ENG_Custom_Authentication.twiki b/docs/src/site/twiki/ENG_Custom_Authentication.twiki
index 6bac3a6..7d967c3 100644
--- a/docs/src/site/twiki/ENG_Custom_Authentication.twiki
+++ b/docs/src/site/twiki/ENG_Custom_Authentication.twiki
@@ -13,108 +13,42 @@
The following authenticators are provided in hadoop-auth:
* KerberosAuthenticator : the authenticator implements the Kerberos SPNEGO authentication sequence.
- * PseudoAuthenticator : the authenticator implementation provides an authentication equivalent to Hadoop's Simple authentication, it trusts the value of the 'user.name' Java System property.
+ * PseudoAuthenticator : the authenticator implementation provides an authentication equivalent to Hadoop's Simple
+ authentication, it trusts the value of the 'user.name' Java System property.
2. =org.apache.hadoop.security.authentication.server.AuthenticationHandler:= Interface for server authentication mechanisms.
* KerberosAuthenticationHandler : the authenticator handler implements the Kerberos SPNEGO authentication mechanism for HTTP.
- * PseudoAuthenticationHandler : the authenticator handler provides a pseudo authentication mechanism that accepts the user name specified as a query string parameter.
* AltKerberosAuthenticationHandler: the authenticator handler allows for Kerberos SPNEGO authentication for non-browsers and an alternate form of authentication for browsers. A subclass must implement the alternate authentication (see [[ENG_Custom_Authentication#LoginServerExample][Example Login Server]])
+ * PseudoAuthenticationHandler : the authenticator handler provides a pseudo authentication mechanism that accepts the user
+ name specified as a query string parameter.
-3. =org.apache.hadoop.security.authentication.server.AuthenticationFilter:= A servlet filter enables protecting web application resources with different authentication mechanisms provided by AuthenticationHandler. To enable the filter, web application resources file (ex. web.xml) needs to include a filter class derived from =AuthenticationFilter=.
+3. =org.apache.hadoop.security.authentication.server.AuthenticationFilter:= A servlet filter enables protecting web application
+resources with different authentication mechanisms provided by AuthenticationHandler. To enable the filter, web application
+resources file (ex. web.xml) needs to include a filter class derived from =AuthenticationFilter=.
----++ Provide Custom Client Authenticator
-
-In client side, a custom authentication requires a extended =Authenticator= to retrieve authentication token or certificate and set it to 'token' instance in method 'authenticate()'.
-
-The following methods should be overridden by derived Authenticator.
-<verbatim>
-
- public void authenticate(URL url, AuthenticatedURL.Token token)
- throws IOException, AuthenticationException {
-
- TheAuthenticatorConf conf = TheAuthenticatorConf();
-
- HttpURLConnection conn = (HttpURLConnection) url.openConnection();
- conn.setRequestMethod("OPTIONS");
-
- //Depending on actual authenticationovide Custom Authentication to Oozie Server
-
-Eclipse and IntelliJ can use directly MiniOozie Maven project files. MiniOozie project can be imported to
-Eclipse and IntelliJ as independent project.
-
-overridden methods
-<verbatim>
- mechanism, retrieve the cert string or token.
- String encodedStr = URLEncoder.encode(aCertString, "UTF-8");
- // set to cookie with a key that can be recognized later in the server side.
- conn.addRequestProperty("Cookie", "NEWAUTH=" + encodedStr);
-
- // extract token from connection and set to token
- AuthenticatedURL.extractToken(conn, token);
-
- }
-</verbatim>
-
-The following shows an example of a singleton class which can be used at a class of Authenticator to set and get configuration which is required for authentication purpose.
-
-<verbatim>
- public static class TheAuthenticatorConf {
- private static final TheAuthenticatorConf instance = new TheAuthenticatorConf();
- private final Map<String, String> map = new HashMap<String, String>();
-
- private TheAuthenticatorConf() {
- }
-
- public static TheAuthenticatorConf getInstance() {
- return instance;
- }
-
- public void put(String key, String value) {
- map.put(key, value);
- }
-
- public String get(String key) {
- return map.get(key);
- }
-
- public void clear() {
- map.clear();
- }
- }
-</verbatim>
+For more information have a look at the appropriate
+[[https://hadoop.apache.org/docs/r2.7.2/hadoop-auth/index.html][Hadoop documentation]].
---++ Provide Custom Authentication to Oozie Client
-Apache Oozie contains a default class =org.apache.oozie.client.AuthOozieClient= to support Kerberos HTTP SPNEGO authentication, pseudo/simple authentication and anonymous access for client connections.
+Apache Oozie contains a default class =org.apache.oozie.client.AuthOozieClient= to support Kerberos HTTP SPNEGO authentication,
+pseudo/simple authentication and anonymous access for client connections.
-To provide other authentication mechanisms, a Oozie client should extend from =AuthOozieClient= and provide the following methods should be overridden by derived classes to provide custom authentication:
+To provide other authentication mechanisms, an Oozie client should extend from =AuthOozieClient= and provide the following
+methods should be overridden by derived classes to provide custom authentication:
* getAuthenticator() : return corresponding Authenticator based on value specified by user at =auth= command option.
- * createConnection() : create a singleton class at Authenticator to allow client set and get key-value configuration for authentication.
-
----++ Provide Custom Server AuthenticationHandler
-
-In server side, a custom authentication requires a extended AuthenticationHandler to retrieve authentication token or certificate from http request and verify it. After successful verification, an =AuthenticationToken= is created with user name and current authentication type. With this token, this request can be proceeded for response.
-
-The following methods should be overridden by derived AuthenticationHandler.
-<verbatim>
-
- public AuthenticationToken authenticate(HttpServletRequest request, HttpServletResponse response)
- throws IOException, AuthenticationException {
-
- // the certificate or token can be retrieved from request and verified.
-
- // use the information from the legal certificate or token to create AuthenticationToken
- AuthenticationToken token = new AuthenticationToken(userName, principal, type);
-
- return token;
- }
-</verbatim>
+ * createConnection() : create a singleton class at Authenticator to allow client set and get key-value configuration for
+ authentication.
---++ Provide Custom Authentication to Oozie Server
-To accept custom authentication in Oozie server, a filter extends from AuthenticationFilter must be provided. This filter delegates to the configured authentication handler for authentication and once it obtains an =AuthenticationToken= from it, sets a signed HTTP cookie with the token. If HTTP cookie is provided with different key name, its cookie value can be retrieved by overriding =getToken()= method. Please note, only when =getToken()= return NULL, a custom authentication can be invoked and processed in =AuthenticationFilter.doFilter()=.
+To accept custom authentication in Oozie server, a filter extends from AuthenticationFilter must be provided. This filter
+delegates to the configured authentication handler for authentication and once it obtains an =AuthenticationToken= from it, sets
+a signed HTTP cookie with the token. If HTTP cookie is provided with different key name, its cookie value can be retrieved by
+overriding =getToken()= method. Please note, only when =getToken()= return NULL, a custom authentication can be invoked and
+processed in =AuthenticationFilter.doFilter()=.
The following method explains how to read it and return NULL token.
<verbatim>
http://git-wip-us.apache.org/repos/asf/oozie/blob/636e434e/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index f831645..0995972 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.3.1 release
+OOZIE-2825 Custom Authentication doc page is not well formatted (Jan Hentschel via rkanter)
OOZIE-2747 README.txt is out of date (Jan Hentschel via rkanter)
OOZIE-2923 Improve Spark options parsing (andras.piros via gezapeti)
OOZIE-2886 Ensure consistent versioning of hadoop jars in sharelibs (dbist13 via rkanter)
[19/31] oozie git commit: OOZIE-2862 Coord change command doesn't
change job to running if job was killed without creating any actions
Posted by sa...@apache.org.
OOZIE-2862 Coord change command doesn't change job to running if job was killed without creating any actions
(cherry picked from commit cf3b400a2a8e66f8689856bfb16b2547526a012a)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/4b3b30d6
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/4b3b30d6
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/4b3b30d6
Branch: refs/heads/branch-4.3
Commit: 4b3b30d684939d5e98a75d82fa0927a6d3e0032a
Parents: 3f751ca
Author: puru <pu...@gmail.com>
Authored: Wed Apr 12 14:17:13 2017 -0700
Committer: satishsaley <sa...@apache.org>
Committed: Fri Dec 8 16:34:55 2017 -0800
----------------------------------------------------------------------
.../command/coord/CoordChangeXCommand.java | 4 +-
.../command/coord/TestCoordChangeXCommand.java | 40 ++++++++++++++++----
release-log.txt | 1 +
3 files changed, 35 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/4b3b30d6/core/src/main/java/org/apache/oozie/command/coord/CoordChangeXCommand.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/command/coord/CoordChangeXCommand.java b/core/src/main/java/org/apache/oozie/command/coord/CoordChangeXCommand.java
index e65b74f..ff7bf8a 100644
--- a/core/src/main/java/org/apache/oozie/command/coord/CoordChangeXCommand.java
+++ b/core/src/main/java/org/apache/oozie/command/coord/CoordChangeXCommand.java
@@ -436,8 +436,8 @@ public class CoordChangeXCommand extends CoordinatorXCommand<Void> {
LOG.info("Coord status is changed to " + jobStatus + " from " + prevStatus);
if (jobStatus.equals(CoordinatorJob.Status.RUNNING)) {
coordJob.setPending();
- if (coordJob.getNextMaterializedTime() != null
- && coordJob.getEndTime().after(coordJob.getNextMaterializedTime())) {
+ if (coordJob.getNextMaterializedTime() == null
+ || coordJob.getEndTime().after(coordJob.getNextMaterializedTime())) {
coordJob.resetDoneMaterialization();
}
} else if (jobStatus.equals(CoordinatorJob.Status.IGNORED)) {
http://git-wip-us.apache.org/repos/asf/oozie/blob/4b3b30d6/core/src/test/java/org/apache/oozie/command/coord/TestCoordChangeXCommand.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/command/coord/TestCoordChangeXCommand.java b/core/src/test/java/org/apache/oozie/command/coord/TestCoordChangeXCommand.java
index 8034bbe..c071000 100644
--- a/core/src/test/java/org/apache/oozie/command/coord/TestCoordChangeXCommand.java
+++ b/core/src/test/java/org/apache/oozie/command/coord/TestCoordChangeXCommand.java
@@ -59,6 +59,7 @@ import org.apache.oozie.util.XCallable;
public class TestCoordChangeXCommand extends XDataTestCase {
private Services services;
+ public static final int HOURS_IN_MS = 60 * 60 * 1000;
/**
* Return the UTC date and time in W3C format down to second
@@ -538,7 +539,7 @@ public class TestCoordChangeXCommand extends XDataTestCase {
// Testcase to check status for coord whose enddate is set before startdate.
public void testCoordChangeEndTimeBeforeStart() throws Exception {
Date start = new Date();
- Date end = new Date(start.getTime() + (4 * 60 * 60 * 1000)); // 4 hrs
+ Date end = new Date(start.getTime() + (4 * HOURS_IN_MS)); // 4 hrs
Date endTime = new Date(start.getTime() - 3000);
String endTimeChangeStr = "endtime=" + DateUtils.formatDateOozieTZ(endTime);
@@ -624,7 +625,7 @@ public class TestCoordChangeXCommand extends XDataTestCase {
Date startTime = DateUtils.parseDateOozieTZ("2013-08-01T00:00Z");
Date endTime = DateUtils.parseDateOozieTZ("2013-08-01T04:59Z");
- Date pauseTime = new Date(startTime.getTime() + (3 * 60 * 60 * 1000)); //2 hrs
+ Date pauseTime = new Date(startTime.getTime() + (3 * HOURS_IN_MS)); //2 hrs
String pauseTimeChangeStr = "pausetime=" + DateUtils.formatDateOozieTZ(pauseTime);
final CoordinatorJobBean job = addRecordToCoordJobTableForPauseTimeTest(CoordinatorJob.Status.RUNNING, startTime,
endTime, endTime, true, false, 4);
@@ -729,7 +730,7 @@ public class TestCoordChangeXCommand extends XDataTestCase {
public void testChangeTimeDeleteRunning() throws Exception {
Date startTime = DateUtils.parseDateOozieTZ("2013-08-01T00:00Z");
Date endTime = DateUtils.parseDateOozieTZ("2013-08-01T04:59Z");
- Date pauseTime = new Date(startTime.getTime() + (2 * 60 * 60 * 1000)); // 2 hrs
+ Date pauseTime = new Date(startTime.getTime() + (2 * HOURS_IN_MS)); // 2 hrs
String pauseTimeChangeStr = "pausetime=" + DateUtils.formatDateOozieTZ(pauseTime);
final CoordinatorJobBean job = addRecordToCoordJobTableForPauseTimeTest(CoordinatorJob.Status.RUNNING,
startTime, endTime, endTime, true, false, 4);
@@ -763,7 +764,7 @@ public class TestCoordChangeXCommand extends XDataTestCase {
public void testCoordStatus_Ignored() throws Exception {
Date start = new Date();
- Date end = new Date(start.getTime() + (5 * 60 * 60 * 1000)); // 5 hrs
+ Date end = new Date(start.getTime() + (5 * HOURS_IN_MS)); // 5 hrs
String statusToRUNNING = "status=RUNNING";
String statusToIGNORED = "status=IGNORED";
final CoordinatorJobBean job1 = addRecordToCoordJobTableForPauseTimeTest(CoordinatorJob.Status.IGNORED, start,
@@ -808,7 +809,7 @@ public class TestCoordChangeXCommand extends XDataTestCase {
// Status change from failed- successful
public void testCoordStatus_Failed() throws Exception {
Date start = new Date();
- Date end = new Date(start.getTime() + (5 * 60 * 60 * 1000)); // 5 hrs
+ Date end = new Date(start.getTime() + (5 * HOURS_IN_MS)); // 5 hrs
String status = "status=RUNNING";
final CoordinatorJobBean job = addRecordToCoordJobTableForPauseTimeTest(CoordinatorJob.Status.FAILED, start,
end, end, true, false, 4);
@@ -837,7 +838,7 @@ public class TestCoordChangeXCommand extends XDataTestCase {
// Status change from Killed- successful
public void testCoordStatus_Killed() throws Exception {
Date start = new Date();
- Date end = new Date(start.getTime() + (5 * 60 * 60 * 1000)); // 5 hrs
+ Date end = new Date(start.getTime() + (5 * HOURS_IN_MS)); // 5 hrs
String status = "status=RUNNING";
final CoordinatorJobBean job = addRecordToCoordJobTableForPauseTimeTest(CoordinatorJob.Status.KILLED, start,
end, end, true, false, 4);
@@ -866,7 +867,7 @@ public class TestCoordChangeXCommand extends XDataTestCase {
// Check status change from Succeeded- exception
public void testCoordStatus_Changefailed() throws Exception {
Date start = new Date();
- Date end = new Date(start.getTime() + (4 * 60 * 60 * 1000)); // 5 hrs
+ Date end = new Date(start.getTime() + (4 * HOURS_IN_MS)); // 5 hrs
String status = "status=RUNNING";
final CoordinatorJobBean job = addRecordToCoordJobTableForPauseTimeTest(CoordinatorJob.Status.SUCCEEDED, start,
end, end, true, false, 4);
@@ -886,7 +887,7 @@ public class TestCoordChangeXCommand extends XDataTestCase {
// Check status change - with multiple option. Pause can't be applied to killed job, old behavior.
public void testCoord_throwException() throws Exception {
Date start = new Date();
- Date end = new Date(start.getTime() + (4 * 60 * 60 * 1000)); // 4 hrs
+ Date end = new Date(start.getTime() + (4 * HOURS_IN_MS)); // 4 hrs
String status = "status=RUNNING;pausetime=" + DateUtils.formatDateOozieTZ(end);
final CoordinatorJobBean job = addRecordToCoordJobTableForPauseTimeTest(CoordinatorJob.Status.KILLED, start,
end, end, true, false, 4);
@@ -903,6 +904,29 @@ public class TestCoordChangeXCommand extends XDataTestCase {
assertTrue(e.getMessage().contains("Cannot change a killed coordinator job"));
}
}
+
+ // Check status change - when job is killed with no action created
+ public void testRunningStatusWithNoAction() throws Exception {
+ Date now = new Date();
+ Date start = new Date(now.getTime() - (4 * HOURS_IN_MS)); // 4 hrs
+ Date end = new Date(now.getTime() + (4 * HOURS_IN_MS)); // 4 hrs
+
+ String status = "status=RUNNING";
+ CoordinatorJobBean job = addRecordToCoordJobTableForPauseTimeTest(CoordinatorJob.Status.KILLED, start, end,
+ null, true, false, 0);
+ job = CoordJobQueryExecutor.getInstance().get(CoordJobQueryExecutor.CoordJobQuery.GET_COORD_JOB, job.getId());
+ assertEquals("KILLED", job.getStatusStr());
+ assertNull(job.getNextMaterializedTime());
+ new CoordChangeXCommand(job.getId(), status).call();
+ job = CoordJobQueryExecutor.getInstance().get(CoordJobQueryExecutor.CoordJobQuery.GET_COORD_JOB, job.getId());
+ assertEquals("RUNNING", job.getStatusStr());
+ // make sure that action is created
+ new CoordMaterializeTransitionXCommand(job.getId(), 3600).call();
+ job = CoordJobQueryExecutor.getInstance().get(CoordJobQueryExecutor.CoordJobQuery.GET_COORD_JOB, job.getId());
+ assertNotNull(job.getNextMaterializedTime());
+ assertEquals(1, job.getLastActionNumber());
+ }
+
protected CoordinatorJobBean addRecordToCoordJobTableForPauseTimeTest(CoordinatorJob.Status status, Date start,
Date end, Date lastActionTime, boolean pending, boolean doneMatd, int lastActionNum) throws Exception {
CoordinatorJobBean coordJob = createCoordJob(status, start, end, pending, doneMatd, lastActionNum);
http://git-wip-us.apache.org/repos/asf/oozie/blob/4b3b30d6/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 5427bad..0649b46 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.3.1 release
+OOZIE-2862 Coord change command doesn't change job to running if job was killed without creating any actions (puru)
OOZIE-2811 amend Add support for filtering out properties from SparkConfigurationService
OOZIE-2807 Oozie gets RM delegation token even for checking job status (satishsaley)
OOZIE-2811 Add support for filtering out properties from SparkConfigurationService (gezapeti via rkanter)
[14/31] oozie git commit: OOZIE-2786 Pass Oozie workflow ID and
settings to Spark application configuration (zhengxb2005 via rkanter)
Posted by sa...@apache.org.
OOZIE-2786 Pass Oozie workflow ID and settings to Spark application configuration (zhengxb2005 via rkanter)
(cherry picked from commit a8c3392570ab46517c995660c922257059272f70)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/3709ea2f
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/3709ea2f
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/3709ea2f
Branch: refs/heads/branch-4.3
Commit: 3709ea2f5abb5c5b0de3fba064e3fe4e39157c6d
Parents: 7350858
Author: Robert Kanter <rk...@cloudera.com>
Authored: Tue Feb 7 13:50:42 2017 -0800
Committer: satishsaley <sa...@apache.org>
Committed: Fri Dec 8 16:34:55 2017 -0800
----------------------------------------------------------------------
release-log.txt | 1 +
.../org/apache/oozie/action/hadoop/SparkMain.java | 17 +++++++++++++++++
.../apache/oozie/action/hadoop/TestSparkMain.java | 17 +++++++++++++++++
3 files changed, 35 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/3709ea2f/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index c39b3ba..76bab5f 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.3.1 release
+OOZIE-2786 Pass Oozie workflow ID and settings to Spark application configuration (zhengxb2005 via rkanter)
OOZIE-2790 log4j configuration is not passed to spark executors (satishsaley)
OOZIE-2787 Oozie distributes application jar twice making the spark job fail (satishsaley)
OOZIE-2777 Config-default.xml longer than 64k results in java.io.UTFDataFormatException (gezapeti via harsh)
http://git-wip-us.apache.org/repos/asf/oozie/blob/3709ea2f/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
----------------------------------------------------------------------
diff --git a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
index 911c99d..d37053d 100644
--- a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
+++ b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
@@ -29,6 +29,7 @@ import java.util.ArrayList;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
+import java.util.Map;
import java.util.Properties;
import java.util.jar.JarFile;
import java.util.jar.Manifest;
@@ -43,6 +44,8 @@ import org.apache.hadoop.fs.Path;
import org.apache.log4j.PropertyConfigurator;
import org.apache.spark.deploy.SparkSubmit;
+import com.google.common.annotations.VisibleForTesting;
+
public class SparkMain extends LauncherMain {
private static final String MASTER_OPTION = "--master";
private static final String MODE_OPTION = "--deploy-mode";
@@ -114,6 +117,8 @@ public class SparkMain extends LauncherMain {
sparkArgs.add(className);
}
+ appendOoziePropertiesToSparkConf(sparkArgs, actionConf);
+
String jarPath = actionConf.get(SparkActionExecutor.SPARK_JAR);
if(jarPath!=null && jarPath.endsWith(".py")){
isPyspark = true;
@@ -466,6 +471,18 @@ public class SparkMain extends LauncherMain {
return manifest.getMainAttributes().getValue("Specification-Version");
}
+ /*
+ * Get properties that needs to be passed to Spark as Spark configuration from actionConf.
+ */
+ @VisibleForTesting
+ protected void appendOoziePropertiesToSparkConf(List<String> sparkArgs, Configuration actionConf) {
+ for (Map.Entry<String, String> oozieConfig : actionConf
+ .getValByRegex("^oozie\\.(?!launcher|spark).+").entrySet()) {
+ sparkArgs.add("--conf");
+ sparkArgs.add(String.format("spark.%s=%s", oozieConfig.getKey(), oozieConfig.getValue()));
+ }
+ }
+
private void appendWithPathSeparator(String what, StringBuilder to){
if(to.length() > 0){
to.append(File.pathSeparator);
http://git-wip-us.apache.org/repos/asf/oozie/blob/3709ea2f/sharelib/spark/src/test/java/org/apache/oozie/action/hadoop/TestSparkMain.java
----------------------------------------------------------------------
diff --git a/sharelib/spark/src/test/java/org/apache/oozie/action/hadoop/TestSparkMain.java b/sharelib/spark/src/test/java/org/apache/oozie/action/hadoop/TestSparkMain.java
index f044048..bf4adf9 100644
--- a/sharelib/spark/src/test/java/org/apache/oozie/action/hadoop/TestSparkMain.java
+++ b/sharelib/spark/src/test/java/org/apache/oozie/action/hadoop/TestSparkMain.java
@@ -26,13 +26,17 @@ import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.util.ArrayList;
+import java.util.List;
import java.util.regex.Pattern;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.oozie.util.IOUtils;
import org.apache.oozie.util.XConfiguration;
+import com.google.common.collect.Lists;
+
public class TestSparkMain extends MainTestCase {
private static final String INPUT = "input.txt";
@@ -120,4 +124,17 @@ public class TestSparkMain extends MainTestCase {
assertFalse(pattern.matcher(s).find());
}
}
+
+ public void testAppendOoziePropertiesToSparkConf() throws Exception {
+ SparkMain instance = SparkMain.class.newInstance();
+ List<String> sparkArgs = new ArrayList<String>();
+ Configuration actionConf = new Configuration();
+ actionConf.set("foo", "foo-not-to-include");
+ actionConf.set("oozie.launcher", "launcher-not-to-include");
+ actionConf.set("oozie.spark", "spark-not-to-include");
+ actionConf.set("oozie.bar", "bar");
+
+ instance.appendOoziePropertiesToSparkConf(sparkArgs, actionConf);
+ assertEquals(Lists.newArrayList("--conf", "spark.oozie.bar=bar"), sparkArgs);
+ }
}
\ No newline at end of file
[13/31] oozie git commit: OOZIE-2790 log4j configuration is not
passed to spark executors (satishsaley)
Posted by sa...@apache.org.
OOZIE-2790 log4j configuration is not passed to spark executors (satishsaley)
(cherry picked from commit 4478d53b929bfe20e0946446886d8eb1673139c2)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/73508586
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/73508586
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/73508586
Branch: refs/heads/branch-4.3
Commit: 73508586293259550eef2b43a1abf8240f0be58d
Parents: 38e6478
Author: Satish Subhashrao Saley <sa...@yahoo-inc.com>
Authored: Mon Feb 6 21:58:36 2017 -0800
Committer: satishsaley <sa...@apache.org>
Committed: Fri Dec 8 16:34:55 2017 -0800
----------------------------------------------------------------------
release-log.txt | 1 +
.../src/main/java/org/apache/oozie/action/hadoop/SparkMain.java | 1 +
2 files changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/73508586/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 9e74443..c39b3ba 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.3.1 release
+OOZIE-2790 log4j configuration is not passed to spark executors (satishsaley)
OOZIE-2787 Oozie distributes application jar twice making the spark job fail (satishsaley)
OOZIE-2777 Config-default.xml longer than 64k results in java.io.UTFDataFormatException (gezapeti via harsh)
OOZIE-2771 Allow retrieving keystore and truststore passwords from Hadoop Credential Provider (asasvari via abhishekbafna)
http://git-wip-us.apache.org/repos/asf/oozie/blob/73508586/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
----------------------------------------------------------------------
diff --git a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
index 8f2f438..911c99d 100644
--- a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
+++ b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
@@ -218,6 +218,7 @@ public class SparkMain extends LauncherMain {
JarFilter jarfilter = new JarFilter(fixedUris, jarPath);
jarfilter.filter();
jarPath = jarfilter.getApplicationJar();
+ fixedUris.add(new Path(SPARK_LOG4J_PROPS).toUri());
String cachedFiles = StringUtils.join(fixedUris, ",");
if (cachedFiles != null && !cachedFiles.isEmpty()) {
sparkArgs.add("--files");
[06/31] oozie git commit: OOZIE-2811 Add support for filtering out
properties from SparkConfigurationService (gezapeti via rkanter)
Posted by sa...@apache.org.
OOZIE-2811 Add support for filtering out properties from SparkConfigurationService (gezapeti via rkanter)
(cherry picked from commit 98ad14bdc021132bd4b0ad54e8edc9f71e0e2636)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/f603225d
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/f603225d
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/f603225d
Branch: refs/heads/branch-4.3
Commit: f603225dadb6379242d9dc90a9cf9207e75bbd5d
Parents: ca9f337
Author: Robert Kanter <rk...@apache.org>
Authored: Tue Mar 7 15:18:14 2017 -0800
Committer: satishsaley <sa...@apache.org>
Committed: Fri Dec 8 16:34:55 2017 -0800
----------------------------------------------------------------------
.../action/hadoop/SparkActionExecutor.java | 10 +-
.../service/SparkConfigurationService.java | 144 +++++++++++--------
core/src/main/resources/oozie-default.xml | 18 ++-
.../service/TestSparkConfigurationService.java | 130 +++++++++--------
release-log.txt | 1 +
5 files changed, 176 insertions(+), 127 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/f603225d/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java
index e2e023e..1a3197a 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java
@@ -32,7 +32,7 @@ import org.jdom.Namespace;
import java.util.ArrayList;
import java.util.List;
-import java.util.Map;
+import java.util.Properties;
public class SparkActionExecutor extends JavaActionExecutor {
public static final String SPARK_MAIN_CLASS_NAME = "org.apache.oozie.action.hadoop.SparkMain";
@@ -79,9 +79,11 @@ public class SparkActionExecutor extends JavaActionExecutor {
StringBuilder sparkOptsSb = new StringBuilder();
if (master.startsWith("yarn")) {
String resourceManager = actionConf.get(HADOOP_JOB_TRACKER);
- Map<String, String> sparkConfig = Services.get().get(SparkConfigurationService.class).getSparkConfig(resourceManager);
- for (Map.Entry<String, String> entry : sparkConfig.entrySet()) {
- sparkOptsSb.append("--conf ").append(entry.getKey()).append("=").append(entry.getValue()).append(" ");
+ Properties sparkConfig =
+ Services.get().get(SparkConfigurationService.class).getSparkConfig(resourceManager);
+ for (String property : sparkConfig.stringPropertyNames()) {
+ sparkOptsSb.append("--conf ")
+ .append(property).append("=").append(sparkConfig.getProperty(property)).append(" ");
}
}
String sparkOpts = actionXml.getChildTextTrim("spark-opts", ns);
http://git-wip-us.apache.org/repos/asf/oozie/blob/f603225d/core/src/main/java/org/apache/oozie/service/SparkConfigurationService.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/service/SparkConfigurationService.java b/core/src/main/java/org/apache/oozie/service/SparkConfigurationService.java
index b29ab8d..b15cce0 100644
--- a/core/src/main/java/org/apache/oozie/service/SparkConfigurationService.java
+++ b/core/src/main/java/org/apache/oozie/service/SparkConfigurationService.java
@@ -18,19 +18,19 @@
package org.apache.oozie.service;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.oozie.ErrorCode;
-import org.apache.oozie.util.IOUtils;
-import org.apache.oozie.util.XConfiguration;
import org.apache.oozie.util.XLog;
import java.io.File;
+import java.io.FileInputStream;
import java.io.FileReader;
import java.io.IOException;
-import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.Map;
import java.util.Properties;
+import java.util.Set;
public class SparkConfigurationService implements Service {
@@ -39,20 +39,25 @@ public class SparkConfigurationService implements Service {
public static final String CONF_PREFIX = Service.CONF_PREFIX + "SparkConfigurationService.";
public static final String SPARK_CONFIGURATIONS = CONF_PREFIX + "spark.configurations";
public static final String SPARK_CONFIGURATIONS_IGNORE_SPARK_YARN_JAR
- = CONF_PREFIX + "spark.configurations.ignore.spark.yarn.jar";
+ = SPARK_CONFIGURATIONS + ".ignore.spark.yarn.jar";
+ public static final String SPARK_CONFIGURATIONS_BLACKLIST = SPARK_CONFIGURATIONS + ".blacklist";
- private Map<String, Map<String, String>> sparkConfigs;
private static final String SPARK_CONFIG_FILE = "spark-defaults.conf";
private static final String SPARK_YARN_JAR_PROP = "spark.yarn.jar";
+ private static final String HOST_WILDCARD = "*";
+ private Map<String, Properties> sparkConfigs;
+ private Set<String> blacklist;
@Override
public void init(Services services) throws ServiceException {
+ loadBlacklist();
loadSparkConfigs();
}
@Override
public void destroy() {
sparkConfigs.clear();
+ blacklist.clear();
}
@Override
@@ -60,76 +65,89 @@ public class SparkConfigurationService implements Service {
return SparkConfigurationService.class;
}
+ private void loadBlacklist() {
+ blacklist = new HashSet<>();
+ for(String s : ConfigurationService.getStrings(SPARK_CONFIGURATIONS_BLACKLIST)) {
+ blacklist.add(s.trim());
+ }
+ // spark.yarn.jar is added if the old property to ignore it is set.
+ if(ConfigurationService.getBoolean(SPARK_CONFIGURATIONS_IGNORE_SPARK_YARN_JAR)){
+ LOG.warn("Deprecated property found in configuration: " + SPARK_CONFIGURATIONS_IGNORE_SPARK_YARN_JAR +
+ "Use "+SPARK_CONFIGURATIONS_BLACKLIST+" instead.");
+ blacklist.add(SPARK_YARN_JAR_PROP);
+ }
+ }
+
private void loadSparkConfigs() throws ServiceException {
- sparkConfigs = new HashMap<String, Map<String, String>>();
- File configDir = new File(ConfigurationService.getConfigurationDirectory());
+ sparkConfigs = new HashMap<>();
String[] confDefs = ConfigurationService.getStrings(SPARK_CONFIGURATIONS);
- if (confDefs != null) {
- boolean ignoreSparkYarnJar = ConfigurationService.getBoolean(SPARK_CONFIGURATIONS_IGNORE_SPARK_YARN_JAR);
- for (String confDef : confDefs) {
- if (confDef.trim().length() > 0) {
- String[] parts = confDef.split("=");
- if (parts.length == 2) {
- String hostPort = parts[0];
- String confDir = parts[1];
- File dir = new File(confDir);
- if (!dir.isAbsolute()) {
- dir = new File(configDir, confDir);
- }
- if (dir.exists()) {
- File file = new File(dir, SPARK_CONFIG_FILE);
- if (file.exists()) {
- Properties props = new Properties();
- FileReader fr = null;
- try {
- fr = new FileReader(file);
- props.load(fr);
- fr.close();
- if (ignoreSparkYarnJar) {
- // Ignore spark.yarn.jar because it may interfere with the Spark Sharelib jars
- props.remove(SPARK_YARN_JAR_PROP);
- }
- sparkConfigs.put(hostPort, propsToMap(props));
- LOG.info("Loaded Spark Configuration: {0}={1}", hostPort, file.getAbsolutePath());
- } catch (IOException ioe) {
- LOG.warn("Spark Configuration could not be loaded for {0}: {1}",
- hostPort, ioe.getMessage(), ioe);
- } finally {
- IOUtils.closeSafely(fr);
- }
- } else {
- LOG.warn("Spark Configuration could not be loaded for {0}: {1} does not exist",
- hostPort, file.getAbsolutePath());
- }
- } else {
- LOG.warn("Spark Configuration could not be loaded for {0}: {1} does not exist",
- hostPort, dir.getAbsolutePath());
- }
- } else {
- LOG.warn("Spark Configuration could not be loaded: invalid value found: {0}", confDef);
- }
+ for (String confDef : confDefs) {
+ readEntry(confDef.trim());
+ }
+ }
+
+ private void readEntry(String confDef) throws ServiceException {
+ String[] parts = confDef.split("=");
+ if (parts.length == 2) {
+ String hostPort = parts[0];
+ String confDir = parts[1];
+ File dir = getAbsoluteDir(confDir);
+ if (dir.exists()) {
+ Properties sparkDefaults = readSparkConfigFile(hostPort, dir);
+ filterBlackList(sparkDefaults);
+ if(!sparkDefaults.isEmpty()) {
+ sparkConfigs.put(hostPort, sparkDefaults);
}
+ } else {
+ LOG.warn("Spark Configuration could not be loaded for {0}: {1} does not exist",
+ hostPort, dir.getAbsolutePath());
}
} else {
- LOG.info("Spark Configuration(s) not specified");
+ LOG.warn("Spark Configuration could not be loaded: invalid value found: {0}", confDef);
}
}
- private Map<String, String> propsToMap(Properties props) {
- Map<String, String> map = new HashMap<String, String>(props.size());
- for (String key : props.stringPropertyNames()) {
- map.put(key, props.getProperty(key));
+ private File getAbsoluteDir(String confDir) throws ServiceException {
+ File dir = new File(confDir);
+ if (!dir.isAbsolute()) {
+ File configDir = new File(ConfigurationService.getConfigurationDirectory());
+ dir = new File(configDir, confDir);
+ }
+ return dir;
+ }
+
+ private void filterBlackList(Properties sparkDefaults) {
+ for(String property : blacklist){
+ sparkDefaults.remove(property);
+ }
+ }
+
+ private Properties readSparkConfigFile(String hostPort, File dir) {
+ File file = new File(dir, SPARK_CONFIG_FILE);
+ Properties props = new Properties();
+ if (file.exists()) {
+ try (FileInputStream stream = new FileInputStream(file);
+ InputStreamReader reader = new InputStreamReader(stream, StandardCharsets.UTF_8.name())) {
+ props.load(reader);
+ LOG.info("Loaded Spark Configuration: {0}={1}", hostPort, file.getAbsolutePath());
+ } catch (IOException ioe) {
+ LOG.warn("Spark Configuration could not be loaded for {0}: {1}",
+ hostPort, ioe.getMessage(), ioe);
+ }
+ } else {
+ LOG.warn("Spark Configuration could not be loaded for {0}: {1} does not exist",
+ hostPort, file.getAbsolutePath());
}
- return map;
+ return props;
}
- public Map<String, String> getSparkConfig(String resourceManagerHostPort) {
+ public Properties getSparkConfig(String resourceManagerHostPort) {
resourceManagerHostPort = (resourceManagerHostPort != null) ? resourceManagerHostPort.toLowerCase() : null;
- Map<String, String> config = sparkConfigs.get(resourceManagerHostPort);
+ Properties config = sparkConfigs.get(resourceManagerHostPort);
if (config == null) {
- config = sparkConfigs.get("*");
+ config = sparkConfigs.get(HOST_WILDCARD);
if (config == null) {
- config = new HashMap<String, String>();
+ config = new Properties();
}
}
return config;
http://git-wip-us.apache.org/repos/asf/oozie/blob/f603225d/core/src/main/resources/oozie-default.xml
----------------------------------------------------------------------
diff --git a/core/src/main/resources/oozie-default.xml b/core/src/main/resources/oozie-default.xml
index b61f163..8c55a0d 100644
--- a/core/src/main/resources/oozie-default.xml
+++ b/core/src/main/resources/oozie-default.xml
@@ -2746,13 +2746,23 @@ will be the requeue interval for the actions which are waiting for a long time w
</property>
<property>
+ <name>oozie.service.SparkConfigurationService.spark.configurations.blacklist</name>
+ <value>spark.yarn.jar,spark.yarn.jars</value>
+ <description>
+ Comma separated list of properties to ignore from any Spark configurations specified in
+ oozie.service.SparkConfigurationService.spark.configurations property.
+ </description>
+ </property>
+
+ <property>
<name>oozie.service.SparkConfigurationService.spark.configurations.ignore.spark.yarn.jar</name>
<value>true</value>
<description>
- If true, Oozie will ignore the "spark.yarn.jar" property from any Spark configurations specified in
- oozie.service.SparkConfigurationService.spark.configurations. If false, Oozie will not ignore it. It is recommended
- to leave this as true because it can interfere with the jars in the Spark sharelib.
- </description>
+ Deprecated. Use oozie.service.SparkConfigurationService.spark.configurations.blacklist instead.
+ If true, Oozie will ignore the "spark.yarn.jar" property from any Spark configurations specified in
+ oozie.service.SparkConfigurationService.spark.configurations. If false, Oozie will not ignore it. It is recommended
+ to leave this as true because it can interfere with the jars in the Spark sharelib.
+ </description>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/oozie/blob/f603225d/core/src/test/java/org/apache/oozie/service/TestSparkConfigurationService.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/service/TestSparkConfigurationService.java b/core/src/test/java/org/apache/oozie/service/TestSparkConfigurationService.java
index 9d82fdc..0e00a45 100644
--- a/core/src/test/java/org/apache/oozie/service/TestSparkConfigurationService.java
+++ b/core/src/test/java/org/apache/oozie/service/TestSparkConfigurationService.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -18,21 +18,11 @@
package org.apache.oozie.service;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.JobClient;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.oozie.ErrorCode;
import org.apache.oozie.test.XTestCase;
-import org.apache.oozie.util.IOUtils;
import java.io.File;
import java.io.FileOutputStream;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.net.URI;
-import java.util.Map;
+import java.io.IOException;
import java.util.Properties;
public class TestSparkConfigurationService extends XTestCase {
@@ -53,40 +43,14 @@ public class TestSparkConfigurationService extends XTestCase {
scs.destroy();
ConfigurationService.set("oozie.service.SparkConfigurationService.spark.configurations", "");
scs.init(Services.get());
- Map<String, String> sparkConfigs = scs.getSparkConfig("foo");
+ Properties sparkConfigs = scs.getSparkConfig("foo");
assertEquals(0, sparkConfigs.size());
}
public void testSparkConfigs() throws Exception {
- File sparkConf1Dir = new File(getTestCaseConfDir(), "spark-conf-1");
- File sparkConf3Dir = new File(getTestCaseConfDir(), "spark-conf-3");
- File sparkConf4Dir = new File(getTestCaseConfDir(), "spark-conf-4");
- sparkConf1Dir.mkdirs();
- sparkConf3Dir.mkdirs();
- sparkConf4Dir.mkdirs();
- File sparkConf1 = new File(sparkConf1Dir, "spark-defaults.conf");
- Properties sparkConf1Props = new Properties();
- sparkConf1Props.setProperty("a", "A");
- sparkConf1Props.setProperty("b", "B");
- sparkConf1Props.setProperty("spark.yarn.jar", "foo"); // should be ignored by default
- FileOutputStream fos = null;
- try {
- fos = new FileOutputStream(sparkConf1);
- sparkConf1Props.store(fos, "");
- } finally {
- IOUtils.closeSafely(fos);
- }
- File sparkConf4 = new File(sparkConf4Dir, "spark-defaults.conf");
- Properties sparkConf4Props = new Properties();
- sparkConf4Props.setProperty("y", "Y");
- sparkConf4Props.setProperty("z", "Z");
- fos = null;
- try {
- fos = new FileOutputStream(sparkConf4);
- sparkConf4Props.store(fos, "");
- } finally {
- IOUtils.closeSafely(fos);
- }
+ File sparkConf1Dir = createSparkConfsInDir("spark-conf-1", "a", "A", "b", "B", "spark.yarn.jar", "foo");
+ File sparkConf3Dir = createSparkConfsInDir("spark-conf-3");
+ File sparkConf4Dir = createSparkConfsInDir("spark-conf-4", "y", "Y", "z", "Z", "spark.yarn.jars", "foo2");
SparkConfigurationService scs = Services.get().get(SparkConfigurationService.class);
scs.destroy();
@@ -96,48 +60,102 @@ public class TestSparkConfigurationService extends XTestCase {
",rm3=" + sparkConf3Dir.getAbsolutePath() + // missing file
",rm4=" + sparkConf4Dir.getName()); // relative path
scs.init(Services.get());
- Map<String, String> sparkConfigs = scs.getSparkConfig("foo");
- assertEquals(0, sparkConfigs.size());
+ Properties sparkConfigs = scs.getSparkConfig("foo");
+ assertEquals(sparkConfigs.toString(), 0, sparkConfigs.size());
sparkConfigs = scs.getSparkConfig("rm1");
- assertEquals(2, sparkConfigs.size());
+ assertEquals(sparkConfigs.toString(), 2, sparkConfigs.size());
assertEquals("A", sparkConfigs.get("a"));
assertEquals("B", sparkConfigs.get("b"));
sparkConfigs = scs.getSparkConfig("rm2");
- assertEquals(0, sparkConfigs.size());
+ assertEquals(sparkConfigs.toString(), 0, sparkConfigs.size());
sparkConfigs = scs.getSparkConfig("rm3");
- assertEquals(0, sparkConfigs.size());
+ assertEquals(sparkConfigs.toString(), 0, sparkConfigs.size());
sparkConfigs = scs.getSparkConfig("rm4");
- assertEquals(2, sparkConfigs.size());
+ assertEquals(sparkConfigs.toString(), 2, sparkConfigs.size());
assertEquals("Y", sparkConfigs.get("y"));
assertEquals("Z", sparkConfigs.get("z"));
scs.destroy();
// Setting this to false should make it not ignore spark.yarn.jar
ConfigurationService.setBoolean("oozie.service.SparkConfigurationService.spark.configurations.ignore.spark.yarn.jar",
false);
+ ConfigurationService.set("oozie.service.SparkConfigurationService.spark.configurations.blacklist", " ");
scs.init(Services.get());
sparkConfigs = scs.getSparkConfig("rm1");
- assertEquals(3, sparkConfigs.size());
+ assertEquals(sparkConfigs.toString(), 3, sparkConfigs.size());
assertEquals("A", sparkConfigs.get("a"));
assertEquals("B", sparkConfigs.get("b"));
assertEquals("foo", sparkConfigs.get("spark.yarn.jar"));
ConfigurationService.setBoolean("oozie.service.SparkConfigurationService.spark.configurations.ignore.spark.yarn.jar", true);
-
+ ConfigurationService.set("oozie.service.SparkConfigurationService.spark.configurations.blacklist", "spark.yarn.jar,spark.yarn.jars");
scs.destroy();
ConfigurationService.set("oozie.service.SparkConfigurationService.spark.configurations",
- "rm1=" + sparkConf1Dir.getAbsolutePath() + // defined
+ "rm1=" + sparkConf1Dir.getAbsolutePath() + // define
",*=" + sparkConf4Dir.getAbsolutePath()); // wildcard
scs.init(Services.get());
sparkConfigs = scs.getSparkConfig("rm1");
- assertEquals(2, sparkConfigs.size());
+ assertEquals(sparkConfigs.toString(), 2, sparkConfigs.size());
assertEquals("A", sparkConfigs.get("a"));
assertEquals("B", sparkConfigs.get("b"));
sparkConfigs = scs.getSparkConfig("rm2");
- assertEquals(2, sparkConfigs.size());
+ assertEquals(sparkConfigs.toString(), 2, sparkConfigs.size());
assertEquals("Y", sparkConfigs.get("y"));
assertEquals("Z", sparkConfigs.get("z"));
sparkConfigs = scs.getSparkConfig("foo");
- assertEquals(2, sparkConfigs.size());
+ assertEquals(sparkConfigs.toString(), 2, sparkConfigs.size());
assertEquals("Y", sparkConfigs.get("y"));
assertEquals("Z", sparkConfigs.get("z"));
}
+
+ private File createSparkConfsInDir(String directory, String... props) throws IOException {
+ File sparkConf1Dir = new File(getTestCaseConfDir(), directory);
+ sparkConf1Dir.mkdirs();
+ File sparkConf1 = new File(sparkConf1Dir, "spark-defaults.conf");
+ Properties sparkConf1Props = new Properties();
+ for (int i = 0; i < props.length; i += 2) {
+ sparkConf1Props.setProperty(props[i], props[i + 1]);
+ }
+ if (!sparkConf1Props.isEmpty()) {
+ try (FileOutputStream fos = new FileOutputStream(sparkConf1)) {
+ sparkConf1Props.store(fos, "");
+ }
+ }
+ return sparkConf1Dir;
+ }
+
+ public void testBlackList() throws Exception {
+ File sparkConf1Dir = createSparkConfsInDir("spark-conf-1", "a", "A", "b", "B",
+ "spark.yarn.jar", "foo");
+ File sparkConf3Dir = createSparkConfsInDir("spark-conf-3");
+ File sparkConf4Dir = createSparkConfsInDir("spark-conf-4", "y", "Y", "z", "Z",
+ "spark.yarn.jars", "foo2");
+
+ SparkConfigurationService scs = Services.get().get(SparkConfigurationService.class);
+ scs.destroy();
+ ConfigurationService.set("oozie.service.SparkConfigurationService.spark.configurations",
+ "rm1=" + sparkConf1Dir.getAbsolutePath() + // absolute path
+ ",rm2" + // invalid entry
+ ",rm3=" + sparkConf3Dir.getAbsolutePath() + // missing file
+ ",rm4=" + sparkConf4Dir.getName()); // relative path
+ ConfigurationService.setBoolean("oozie.service.SparkConfigurationService.spark.configurations.ignore.spark.yarn.jar",
+ false);
+ ConfigurationService.set("oozie.service.SparkConfigurationService.spark.configurations.blacklist", "a,z");
+ scs.init(Services.get());
+ Properties sparkConfigs = scs.getSparkConfig("foo");
+ assertEquals(sparkConfigs.toString(), 0, sparkConfigs.size());
+ sparkConfigs = scs.getSparkConfig("rm1");
+ assertEquals(sparkConfigs.toString(), 2, sparkConfigs.size());
+ assertEquals("B", sparkConfigs.get("b"));
+ assertEquals("foo", sparkConfigs.get("spark.yarn.jar"));
+ sparkConfigs = scs.getSparkConfig("rm2");
+ assertEquals(sparkConfigs.toString(), 0, sparkConfigs.size());
+ sparkConfigs = scs.getSparkConfig("rm3");
+ assertEquals(sparkConfigs.toString(), 0, sparkConfigs.size());
+ sparkConfigs = scs.getSparkConfig("rm4");
+ assertEquals(sparkConfigs.toString(), 2, sparkConfigs.size());
+ assertEquals("Y", sparkConfigs.get("y"));
+ assertEquals("foo2", sparkConfigs.get("spark.yarn.jars"));
+ scs.destroy();
+ }
+
+
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/f603225d/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index cf43d61..43ecb06 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.3.1 release
+OOZIE-2811 Add support for filtering out properties from SparkConfigurationService (gezapeti via rkanter)
OOZIE-2802 Spark action failure on Spark 2.1.0 due to duplicate sharelibs (gezapeti via rkanter)
OOZIE-2803 Mask passwords when printing out configs/args in MapReduceMain and SparkMain (pbacsko via rkanter)
OOZIE-2799 Setting log location for spark sql on hive (satishsaley)
[02/31] oozie git commit: OOZIE-3028 Oozie Pig Action fails with no
python dependencies (dbist13 via rohini)
Posted by sa...@apache.org.
OOZIE-3028 Oozie Pig Action fails with no python dependencies (dbist13 via rohini)
(cherry picked from commit 630171297cc75b56aafef27efdf2b9b746d86e33)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/d64a4c39
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/d64a4c39
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/d64a4c39
Branch: refs/heads/branch-4.3
Commit: d64a4c39f4a89741e0712390f51457bd352b96be
Parents: 0f293ea
Author: Rohini Palaniswamy <ro...@apache.org>
Authored: Thu Aug 10 20:35:13 2017 -0700
Committer: satishsaley <sa...@apache.org>
Committed: Fri Dec 8 16:34:55 2017 -0800
----------------------------------------------------------------------
pom.xml | 4 ++--
release-log.txt | 1 +
sharelib/pig/pom.xml | 2 +-
3 files changed, 4 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/d64a4c39/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 288aaec..7613bc2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1342,8 +1342,8 @@
<dependency>
<groupId>org.python</groupId>
- <artifactId>jython</artifactId>
- <version>2.5.3</version>
+ <artifactId>jython-standalone</artifactId>
+ <version>2.7.1</version>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/oozie/blob/d64a4c39/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index a0946fe..557f657 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.3.1 release
+OOZIE-3028 Oozie Pig Action fails with no python dependencies (dbist13 via rohini)
OOZIE-2984 Parse spark-defaults.conf values with spaces without needing the quotes (andras.piros via gezapeti)
OOZIE-2825 Custom Authentication doc page is not well formatted (Jan Hentschel via rkanter)
OOZIE-2747 README.txt is out of date (Jan Hentschel via rkanter)
http://git-wip-us.apache.org/repos/asf/oozie/blob/d64a4c39/sharelib/pig/pom.xml
----------------------------------------------------------------------
diff --git a/sharelib/pig/pom.xml b/sharelib/pig/pom.xml
index f0ed83b..3cc197b 100644
--- a/sharelib/pig/pom.xml
+++ b/sharelib/pig/pom.xml
@@ -52,7 +52,7 @@
</dependency>
<dependency>
<groupId>org.python</groupId>
- <artifactId>jython</artifactId>
+ <artifactId>jython-standalone</artifactId>
<scope>compile</scope>
</dependency>
[18/31] oozie git commit: OOZIE-2792 amend to fix regex
Posted by sa...@apache.org.
OOZIE-2792 amend to fix regex
(cherry picked from commit 64a58212f3549f865682c75e45710f11578ae9e7)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/4363cca3
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/4363cca3
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/4363cca3
Branch: refs/heads/branch-4.3
Commit: 4363cca33e7554f3044629712fefeac30fbd43f9
Parents: b721224
Author: Robert Kanter <rk...@apache.org>
Authored: Mon Feb 27 08:05:14 2017 -0800
Committer: satishsaley <sa...@apache.org>
Committed: Fri Dec 8 16:34:55 2017 -0800
----------------------------------------------------------------------
.../main/java/org/apache/oozie/action/hadoop/Hive2Main.java | 2 +-
.../java/org/apache/oozie/action/hadoop/TestHive2Main.java | 6 ++++--
2 files changed, 5 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/4363cca3/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java
----------------------------------------------------------------------
diff --git a/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java b/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java
index 8e214fa..a3a07bd 100644
--- a/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java
+++ b/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java
@@ -44,7 +44,7 @@ public class Hive2Main extends LauncherMain {
static final Pattern[] HIVE2_JOB_IDS_PATTERNS = {
Pattern.compile("Ended Job = (job_\\S*)"),
Pattern.compile("Submitted application (application[0-9_]*)"),
- Pattern.compile("Running with YARN Application = (application_[0-9]+)")
+ Pattern.compile("Running with YARN Application = (application[0-9_]*)")
};
private static final Set<String> DISALLOWED_BEELINE_OPTIONS = new HashSet<String>();
http://git-wip-us.apache.org/repos/asf/oozie/blob/4363cca3/sharelib/hive2/src/test/java/org/apache/oozie/action/hadoop/TestHive2Main.java
----------------------------------------------------------------------
diff --git a/sharelib/hive2/src/test/java/org/apache/oozie/action/hadoop/TestHive2Main.java b/sharelib/hive2/src/test/java/org/apache/oozie/action/hadoop/TestHive2Main.java
index b94b7c3..b6ad294 100644
--- a/sharelib/hive2/src/test/java/org/apache/oozie/action/hadoop/TestHive2Main.java
+++ b/sharelib/hive2/src/test/java/org/apache/oozie/action/hadoop/TestHive2Main.java
@@ -30,13 +30,13 @@ public class TestHive2Main extends XTestCase {
List<String> lines = new ArrayList<String>();
lines.add("Ended Job = job_001");
lines.add("Running with YARN Application = application_002");
+ lines.add("Running with YARN Application = application_002_2");
lines.add("Submitted application application_003");
+ lines.add("Submitted application application_003_2");
// Non-matching ones
lines.add("Ended Job = ");
lines.add("Ended Job = abc");
lines.add("Running with YARN Application = job_004");
- lines.add("Running with YARN Application = application005");
- lines.add("Running with YARN Application = application_abc");
lines.add("Submitted application job_006");
Set<String> jobIds = new LinkedHashSet<String>();
for (String line : lines) {
@@ -46,7 +46,9 @@ public class TestHive2Main extends XTestCase {
Set<String> expected = new LinkedHashSet<String>();
expected.add("job_001");
expected.add("job_002");
+ expected.add("job_002_2");
expected.add("job_003");
+ expected.add("job_003_2");
assertEquals(expected, jobIds);
}
}
\ No newline at end of file
[21/31] oozie git commit: OOZIE-2777 amend to fix RAT report
Posted by sa...@apache.org.
OOZIE-2777 amend to fix RAT report
(cherry picked from commit b1bfa109b3acc2ede4533edfe3d5b5224e7e9f04)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/38e64789
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/38e64789
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/38e64789
Branch: refs/heads/branch-4.3
Commit: 38e64789662bfd3fc8d66148a081b521115f7a94
Parents: aae4948
Author: Robert Kanter <rk...@cloudera.com>
Authored: Mon Feb 6 17:44:53 2017 -0800
Committer: satishsaley <sa...@apache.org>
Committed: Fri Dec 8 16:34:55 2017 -0800
----------------------------------------------------------------------
pom.xml | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/38e64789/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 36631a1..49d55e9 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1843,6 +1843,7 @@
<exclude>docs/src/site/twiki/*.twiki</exclude>
<exclude>examples/src/main/data/*</exclude>
<exclude>core/src/test/resources/test-ioutils.txt</exclude>
+ <exclude>core/src/test/resources/oldWorkFlowApp.serialized</exclude>
<exclude>sharelib/pig/src/test/resources/PigMain.txt</exclude>
<exclude>sharelib/hive/src/test/resources/HiveMain.txt</exclude>
<exclude>webapp/src/main/resources/.gitignore</exclude>
[17/31] oozie git commit: OOZIE-2803 Mask passwords when printing out
configs/args in MapReduceMain and SparkMain (pbacsko via rkanter)
Posted by sa...@apache.org.
OOZIE-2803 Mask passwords when printing out configs/args in MapReduceMain and SparkMain (pbacsko via rkanter)
(cherry picked from commit ccbf692d90f8e52ab30afcdc7e411209f3d2b94b)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/b721224f
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/b721224f
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/b721224f
Branch: refs/heads/branch-4.3
Commit: b721224f2288b51feba33ef6264be5274266d51f
Parents: 4ed56b6
Author: Robert Kanter <rk...@apache.org>
Authored: Fri Feb 24 13:18:11 2017 -0800
Committer: satishsaley <sa...@apache.org>
Committed: Fri Dec 8 16:34:55 2017 -0800
----------------------------------------------------------------------
.../org/apache/oozie/util/Instrumentation.java | 1 +
.../org/apache/oozie/util/PasswordMasker.java | 121 ----------------
.../apache/oozie/util/TestPasswordMasker.java | 92 ------------
.../test/resources/instrumentation-os-env.json | 47 ------
.../instrumentation-system-properties.json | 88 ------------
release-log.txt | 1 +
.../oozie/action/hadoop/MapReduceMain.java | 13 +-
.../oozie/action/hadoop/PasswordMasker.java | 144 +++++++++++++++++++
.../oozie/action/hadoop/TestPasswordMasker.java | 140 ++++++++++++++++++
.../test/resources/instrumentation-os-env.json | 47 ++++++
.../instrumentation-system-properties.json | 88 ++++++++++++
.../apache/oozie/action/hadoop/SparkMain.java | 3 +-
12 files changed, 434 insertions(+), 351 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/b721224f/core/src/main/java/org/apache/oozie/util/Instrumentation.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/util/Instrumentation.java b/core/src/main/java/org/apache/oozie/util/Instrumentation.java
index 99d64ac..92de2c9 100644
--- a/core/src/main/java/org/apache/oozie/util/Instrumentation.java
+++ b/core/src/main/java/org/apache/oozie/util/Instrumentation.java
@@ -20,6 +20,7 @@ package org.apache.oozie.util;
import com.google.common.collect.Maps;
import org.apache.hadoop.conf.Configuration;
+import org.apache.oozie.action.hadoop.PasswordMasker;
import org.apache.oozie.service.ConfigurationService;
import org.apache.oozie.service.Services;
http://git-wip-us.apache.org/repos/asf/oozie/blob/b721224f/core/src/main/java/org/apache/oozie/util/PasswordMasker.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/util/PasswordMasker.java b/core/src/main/java/org/apache/oozie/util/PasswordMasker.java
deleted file mode 100644
index 1f8a0ab..0000000
--- a/core/src/main/java/org/apache/oozie/util/PasswordMasker.java
+++ /dev/null
@@ -1,121 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.oozie.util;
-
-import com.google.common.collect.Maps;
-
-import javax.annotation.Nullable;
-import java.util.Map;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-/**
- * A generic password masker that masks {@code Map<String, String>} values given that its keys are considered password keys.
- * <p/>
- * Tested with {@see System#getProperties()} and {@see System#getenv()}.
- */
-class PasswordMasker {
-
- /**
- * The mask that is applied to recognized passwords.
- **/
- private static final String PASSWORD_MASK = "*****";
-
- /**
- * A key is considered a password key, if it contains {{pass}}, case ignored.
- **/
- private static final String PASSWORD_KEY = "pass";
-
- /**
- * Tells us whether an OS environment variable that contains a password fragment.
- * <p/>
- * E.g. {{-Djavax.net.ssl.trustStorePassword=password}} from {{$CATALINA_OPTS}}.
- **/
- private static final String REGEX_CONTAINING_PASSWORD_FRAGMENT_OS_ENV_STYLE =
- ".*[((\\s)+-[D|X][\\w[.\\w]*]*(?i)pass[\\w[.\\w]*]*=)([\\w]+)]+.*";
-
- /**
- * Extracts a password fragment from an OS environment variable. Can be used iteratively to get all fragments.
- * <p/>
- * E.g. {{-Doozie.https.keystore.pass=password}} and {{-Djavax.net.ssl.trustStorePassword=password}} from {{$CATALINA_OPTS}}.
- * {@see java.util.Matcher#find()}
- **/
- private static final String REGEX_EXTRACTING_PASSWORD_FRAGMENTS_OS_ENV_STYLE =
- "((\\s)+-[D|X][\\w[.\\w]*]*(?i)pass[\\w[.\\w]*]*=)([\\w]+)";
-
- private static final Pattern PATTERN_CONTAINING_PASSWORD_FRAGMENTS = Pattern
- .compile(REGEX_CONTAINING_PASSWORD_FRAGMENT_OS_ENV_STYLE);
-
- private static final Pattern PATTERN_EXTRACTING_PASSWORD_FRAGMENTS = Pattern
- .compile(REGEX_EXTRACTING_PASSWORD_FRAGMENTS_OS_ENV_STYLE);
-
- Map<String, String> mask(Map<String, String> unmasked) {
- return Maps.transformEntries(unmasked, new Maps.EntryTransformer<String, String, String>() {
- @Override
- public String transformEntry(@Nullable String key, @Nullable String value) {
- checkNotNull(key, "key has to be set");
- checkNotNull(value, "value has to be set");
-
- if (isPasswordKey(key)) {
- return PASSWORD_MASK;
- }
-
- if (containsPasswordFragment(value)) {
- return maskPasswordFragments(value);
- }
-
- return value;
- }
- });
- }
-
- private boolean isPasswordKey(String key) {
- return key.toLowerCase().contains(PASSWORD_KEY);
-
- }
-
- private boolean containsPasswordFragment(String maybePasswordFragments) {
- return PATTERN_CONTAINING_PASSWORD_FRAGMENTS
- .matcher(maybePasswordFragments)
- .matches();
- }
-
- private String maskPasswordFragments(String maybePasswordFragments) {
- StringBuilder maskedBuilder = new StringBuilder();
- Matcher passwordFragmentsMatcher = PATTERN_EXTRACTING_PASSWORD_FRAGMENTS
- .matcher(maybePasswordFragments);
-
- int start = 0, end;
- while (passwordFragmentsMatcher.find()) {
- end = passwordFragmentsMatcher.start();
-
- maskedBuilder.append(maybePasswordFragments.substring(start, end));
- maskedBuilder.append(passwordFragmentsMatcher.group(1));
- maskedBuilder.append(PASSWORD_MASK);
-
- start = passwordFragmentsMatcher.end();
- }
-
- maskedBuilder.append(maybePasswordFragments.substring(start));
-
- return maskedBuilder.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/oozie/blob/b721224f/core/src/test/java/org/apache/oozie/util/TestPasswordMasker.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/util/TestPasswordMasker.java b/core/src/test/java/org/apache/oozie/util/TestPasswordMasker.java
deleted file mode 100644
index b00cce7..0000000
--- a/core/src/test/java/org/apache/oozie/util/TestPasswordMasker.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.oozie.util;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-public class TestPasswordMasker {
-
- @Test
- public void testWhenJavaSystemPropertiesAreAskedPasswordsAppearMasked() throws Exception {
- Map<String, String> masked = new PasswordMasker().mask(jsonToMap("/instrumentation-system-properties.json"));
-
- assertPasswordValueIsMasked(masked, "javax.net.ssl.trustStorePassword");
- assertPasswordValueIsMasked(masked, "oozie.https.keystore.pass");
- }
-
- @Test
- public void testWhenOSEnvIsAskedPasswordsAppearMasked() throws Exception {
- Map<String, String> masked = new PasswordMasker().mask(jsonToMap("/instrumentation-os-env.json"));
-
- assertPasswordValueIsMasked(masked, "HADOOP_CREDSTORE_PASSWORD");
- assertPasswordValueIsMasked(masked, "OOZIE_HTTPS_KEYSTORE_PASSWORD");
- assertPasswordValueIsMasked(masked, "OOZIE_HTTPS_TRUSTSTORE_PASSWORD");
-
- assertPasswordValueFragmentIsMasked(masked, "CATALINA_OPTS", "-Doozie.https.keystore.pass=");
- assertPasswordValueFragmentIsMasked(masked, "CATALINA_OPTS", "-Djavax.net.ssl.trustStorePassword=");
-
- assertValueFragmentIsPresent(masked, "CATALINA_OPTS", "-Xmx1024m");
- assertValueFragmentIsPresent(masked, "CATALINA_OPTS", "-Doozie.https.keystore.file=/Users/forsage/.keystore");
- assertValueFragmentIsPresent(masked, "CATALINA_OPTS", "-Djava.library.path=");
- }
-
- @SuppressWarnings("unchecked")
- private Map<String, String> jsonToMap(String jsonPath) throws IOException {
- return new ObjectMapper().readValue(getClass().getResourceAsStream(jsonPath), HashMap.class);
- }
-
- private void assertPasswordValueIsMasked(Map<String, String> mapContainingMaskedPassword, String passwordKey) {
- assertEquals(String.format("Value of key '%s' should be masked.", passwordKey),
- "*****",
- mapContainingMaskedPassword.get(passwordKey));
- }
-
- private void assertPasswordValueFragmentIsMasked(Map<String, String> mapContainingMaskedPassword, String passwordKey,
- String passwordFragmentKey) {
- assertEquals(
- String.format("Value fragment of password key '%s' and password fragment key '%s' should be masked.",
- passwordKey,
- passwordFragmentKey),
- "*****",
- getFragmentValue(mapContainingMaskedPassword.get(passwordKey), passwordFragmentKey));
- }
-
- private String getFragmentValue(String base, String fragmentKey) {
- for (String fragment : base.split(" ")) {
- if (fragment.startsWith(fragmentKey)) {
- return fragment.substring(fragmentKey.length());
- }
- }
-
- return null;
- }
-
- private void assertValueFragmentIsPresent(Map<String, String> masked, String key, String valueFragment) {
- assertTrue(String.format("For key '%s' value fragment '%s' should be present.", key, valueFragment),
- masked.get(key).contains(valueFragment));
- }
-}
http://git-wip-us.apache.org/repos/asf/oozie/blob/b721224f/core/src/test/resources/instrumentation-os-env.json
----------------------------------------------------------------------
diff --git a/core/src/test/resources/instrumentation-os-env.json b/core/src/test/resources/instrumentation-os-env.json
deleted file mode 100644
index e85cd8d..0000000
--- a/core/src/test/resources/instrumentation-os-env.json
+++ /dev/null
@@ -1,47 +0,0 @@
-{
- "HADOOP_CREDSTORE_PASSWORD": "password",
- "OOZIE_HTTPS_KEYSTORE_PASSWORD": "password",
- "OOZIE_HTTPS_TRUSTSTORE_PASSWORD": "password",
- "PATH": "/opt/local/bin:/opt/local/sbin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin",
- "HISTCONTROL": "ignoreboth",
- "OOZIE_DATA": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/data",
- "CATALINA_PID": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/oozie-server/temp/oozie.pid",
- "MC_SID": "8597",
- "OOZIE_INSTANCE_ID": "Budapests-MacBook-Pro.local",
- "OOZIE_HTTP_HOSTNAME": "Budapests-MacBook-Pro.local",
- "JAVA_HOME": "/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home",
- "CATALINA_OUT": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/logs/catalina.out",
- "TERM": "xterm-256color",
- "LANG": "en_US.UTF-8",
- "CATALINA_BASE": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/oozie-server",
- "OOZIE_CONFIG_FILE": "oozie-site.xml",
- "LOGNAME": "forsage",
- "OOZIE_HOME": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT",
- "XPC_SERVICE_NAME": "0",
- "PWD": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT",
- "TERM_PROGRAM_VERSION": "361.1",
- "JAVA_MAIN_CLASS_33220": "org.apache.catalina.startup.Bootstrap",
- "_": "/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home/bin/java",
- "SHELL": "/bin/bash",
- "OOZIE_CONFIG": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/conf",
- "TERM_PROGRAM": "Apple_Terminal",
- "OOZIE_ADMIN_PORT": "11001",
- "CATALINA_OPTS": " -Xmx1024m -Dderby.stream.error.file=/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/logs/derby.log -Doozie.home.dir=/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT -Doozie.config.dir=/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/conf -Doozie.log.dir=/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/logs -Doozie.data.dir=/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/data -Doozie.instance.id=Budapests-MacBook-Pro.local -Doozie.config.file=oozie-site.xml -Doozie.log4j.file=oozie-log4j.properties -Doozie.log4j.reload=10 -Doozie.http.hostname=Budapests-MacBook-Pro.local -Doozie.admin.port=11001 -Doozie.http.port=11000 -Doozie.https.port=11443 -Doozie.base.url=http://Budapests-MacBook-Pro.local:11000/oozie -Doozie.https.keystore.file=/Users
/forsage/.keystore -Doozie.https.keystore.pass=password -Djavax.net.ssl.trustStorePassword=password -Djava.library.path=",
- "USER": "forsage",
- "OOZIE_LOG": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/logs",
- "OOZIE_LOG4J_RELOAD": "10",
- "TMPDIR": "/var/folders/yy/gkvmmzn91vv_lb2_bmymxz600000gp/T/",
- "SSH_AUTH_SOCK": "/private/tmp/com.apple.launchd.NvNvd0j95Z/Listeners",
- "MC_TMPDIR": "/var/folders/yy/gkvmmzn91vv_lb2_bmymxz600000gp/T/mc-forsage",
- "XPC_FLAGS": "0x0",
- "OOZIE_BASE_URL": "http://Budapests-MacBook-Pro.local:11000/oozie",
- "TERM_SESSION_ID": "283A05FC-7501-4B9D-B3E3-BDDD3521593C",
- "OOZIE_HTTPS_KEYSTORE_FILE": "/Users/forsage/.keystore",
- "__CF_USER_TEXT_ENCODING": "0x1F6:0x0:0x0",
- "Apple_PubSub_Socket_Render": "/private/tmp/com.apple.launchd.6kR2bgiMHn/Render",
- "OOZIE_HTTP_PORT": "11000",
- "OOZIE_HTTPS_PORT": "11443",
- "SHLVL": "3",
- "HOME": "/Users/forsage",
- "OOZIE_LOG4J_FILE": "oozie-log4j.properties"
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/oozie/blob/b721224f/core/src/test/resources/instrumentation-system-properties.json
----------------------------------------------------------------------
diff --git a/core/src/test/resources/instrumentation-system-properties.json b/core/src/test/resources/instrumentation-system-properties.json
deleted file mode 100644
index 61430d2..0000000
--- a/core/src/test/resources/instrumentation-system-properties.json
+++ /dev/null
@@ -1,88 +0,0 @@
-{
- "javax.net.ssl.trustStorePassword": "password",
- "oozie.https.keystore.pass": "password",
- "gopherProxySet": "false",
- "awt.toolkit": "sun.lwawt.macosx.LWCToolkit",
- "oozie.base.url": "http://Budapests-MacBook-Pro.local:11000/oozie",
- "file.encoding.pkg": "sun.io",
- "java.specification.version": "1.8",
- "sun.cpu.isalist": "",
- "sun.jnu.encoding": "UTF-8",
- "java.class.path": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/oozie-server/bin/bootstrap.jar",
- "java.vm.vendor": "Oracle Corporation",
- "sun.arch.data.model": "64",
- "sun.font.fontmanager": "sun.font.CFontManager",
- "catalina.useNaming": "true",
- "java.vendor.url": "http://java.oracle.com/",
- "user.timezone": "Europe/Budapest",
- "os.name": "Mac OS X",
- "java.vm.specification.version": "1.8",
- "oozie.http.hostname": "Budapests-MacBook-Pro.local",
- "oozie.instance.id": "Budapests-MacBook-Pro.local",
- "sun.java.launcher": "SUN_STANDARD",
- "user.country": "US",
- "oozie.log.dir": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/logs",
- "oozie.home.dir": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT",
- "sun.boot.library.path": "/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home/jre/lib",
- "sun.java.command": "org.apache.catalina.startup.Bootstrap start",
- "http.nonProxyHosts": "local|*.local|169.254/16|*.169.254/16",
- "sun.cpu.endian": "little",
- "user.home": "/Users/forsage",
- "user.language": "en",
- "java.specification.vendor": "Oracle Corporation",
- "java.naming.factory.url.pkgs": "org.apache.naming",
- "java.home": "/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home/jre",
- "oozie.config.file": "oozie-site.xml",
- "oozie.log4j.reload": "10",
- "file.separator": "/",
- "oozie.https.keystore.file": "/Users/forsage/.keystore",
- "line.separator": "\n",
- "java.vm.specification.vendor": "Oracle Corporation",
- "java.specification.name": "Java Platform API Specification",
- "derby.stream.error.file": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/logs/derby.log",
- "oozie.log4j.file": "oozie-log4j.properties",
- "oozie.admin.port": "11001",
- "java.awt.graphicsenv": "sun.awt.CGraphicsEnvironment",
- "package.access": "sun.,org.apache.catalina.,org.apache.coyote.,org.apache.jasper.,org.apache.naming.resources.,org.apache.tomcat.,sun.beans.",
- "package.definition": "sun.,java.,org.apache.catalina.,org.apache.coyote.,org.apache.jasper.,org.apache.naming.,org.apache.tomcat.",
- "sun.boot.class.path": "/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home/jre/lib/resources.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home/jre/lib/rt.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home/jre/lib/sunrsasign.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home/jre/lib/jsse.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home/jre/lib/jce.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home/jre/lib/charsets.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home/jre/lib/jfr.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home/jre/classes",
- "server.loader": "",
- "java.util.logging.config.file": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/oozie-server/conf/logging.properties",
- "sun.management.compiler": "HotSpot 64-Bit Tiered Compilers",
- "oozie.data.dir": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/data",
- "ftp.nonProxyHosts": "local|*.local|169.254/16|*.169.254/16",
- "java.runtime.version": "1.8.0_102-b14",
- "java.naming.factory.initial": "org.apache.naming.java.javaURLContextFactory",
- "user.name": "forsage",
- "oozie.https.port": "11443",
- "path.separator": ":",
- "common.loader": "${catalina.base}/lib,${catalina.base}/lib/*.jar,${catalina.home}/lib,${catalina.home}/lib/*.jar",
- "os.version": "10.11.6",
- "java.endorsed.dirs": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/oozie-server/endorsed",
- "java.runtime.name": "Java(TM) SE Runtime Environment",
- "file.encoding": "UTF-8",
- "java.vm.name": "Java HotSpot(TM) 64-Bit Server VM",
- "java.vendor.url.bug": "http://bugreport.sun.com/bugreport/",
- "java.io.tmpdir": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/oozie-server/temp",
- "oozie.http.port": "11000",
- "catalina.home": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/oozie-server",
- "java.version": "1.8.0_102",
- "user.dir": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT",
- "oozie.config.dir": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/conf",
- "os.arch": "x86_64",
- "java.vm.specification.name": "Java Virtual Machine Specification",
- "java.awt.printerjob": "sun.lwawt.macosx.CPrinterJob",
- "sun.os.patch.level": "unknown",
- "catalina.base": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/oozie-server",
- "shared.loader": "",
- "java.util.logging.manager": "org.apache.juli.ClassLoaderLogManager",
- "java.library.path": "",
- "java.vendor": "Oracle Corporation",
- "java.vm.info": "mixed mode",
- "java.vm.version": "25.102-b14",
- "sun.io.unicode.encoding": "UnicodeBig",
- "java.ext.dirs": "/Users/forsage/Library/Java/Extensions:/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home/jre/lib/ext:/Library/Java/Extensions:/Network/Library/Java/Extensions:/System/Library/Java/Extensions:/usr/lib/java",
- "tomcat.util.buf.StringCache.byte.enabled": "true",
- "java.class.version": "52.0",
- "socksNonProxyHosts": "local|*.local|169.254/16|*.169.254/16"
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/oozie/blob/b721224f/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index db89107..222c48f 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.3.1 release
+OOZIE-2803 Mask passwords when printing out configs/args in MapReduceMain and SparkMain (pbacsko via rkanter)
OOZIE-2799 Setting log location for spark sql on hive (satishsaley)
OOZIE-2792 Hive2 action is not parsing Spark application ID from log file properly when Hive is on Spark (zhengxb2005 via rkanter)
OOZIE-2786 Pass Oozie workflow ID and settings to Spark application configuration (zhengxb2005 via rkanter)
http://git-wip-us.apache.org/repos/asf/oozie/blob/b721224f/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java
----------------------------------------------------------------------
diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java
index 23447cf..d376057 100644
--- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java
+++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java
@@ -23,8 +23,10 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.RunningJob;
-import java.util.HashSet;
+
+import java.util.ArrayList;
import java.util.Map;
+import java.util.Map.Entry;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.File;
@@ -55,7 +57,14 @@ public class MapReduceMain extends LauncherMain {
// Run a config class if given to update the job conf
runConfigClass(jobConf);
- logMasking("Map-Reduce job configuration:", new HashSet<String>(), jobConf);
+ PasswordMasker passwordMasker = new PasswordMasker();
+ // Temporary JobConf object, we mask out possible passwords before we print key-value pairs
+ JobConf maskedJobConf = new JobConf(false);
+ for (Entry<String, String> entry : jobConf) {
+ maskedJobConf.set(entry.getKey(), passwordMasker.maskPasswordsIfNecessary(entry.getValue()));
+ }
+
+ logMasking("Map-Reduce job configuration:", new ArrayList<String>(), maskedJobConf);
File idFile = new File(System.getProperty(LauncherMapper.ACTION_PREFIX + LauncherMapper.ACTION_DATA_NEW_ID));
System.out.println("Submitting Oozie action Map-Reduce job");
http://git-wip-us.apache.org/repos/asf/oozie/blob/b721224f/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/PasswordMasker.java
----------------------------------------------------------------------
diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/PasswordMasker.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/PasswordMasker.java
new file mode 100644
index 0000000..eb60aac
--- /dev/null
+++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/PasswordMasker.java
@@ -0,0 +1,144 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oozie.action.hadoop;
+
+import com.google.common.collect.Maps;
+
+import javax.annotation.Nonnull;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+/**
+ * A generic password masker that masks {@code Map<String, String>} values given that its keys are considered password keys.
+ * <p/>
+ * Tested with {@see System#getProperties()} and {@see System#getenv()}.
+ */
+public class PasswordMasker {
+
+ /**
+ * The mask that is applied to recognized passwords.
+ **/
+ private static final String PASSWORD_MASK = "*****";
+
+ /**
+ * A key is considered a password key, if it contains {{pass}}, case ignored.
+ **/
+ private static final String PASSWORD_KEY = "pass";
+
+ /**
+ * Tells us whether a given string contains a password fragment. A password fragment is something that looks
+ * like {{-Djavax.net.ssl.trustStorePassword=password}} or {{HADOOP_CREDSTORE_PASSWORD=pwd123}}
+ *
+ **/
+ private static final String PASSWORD_CONTAINING_REGEX =
+ "(.*)([\\w[.\\w]*]*(?i)" + PASSWORD_KEY + "[\\w]*=)([\\w]+)(.*)";
+
+ private static final Pattern PASSWORD_CONTAINING_PATTERN = Pattern
+ .compile(PASSWORD_CONTAINING_REGEX);
+
+ /**
+ * Extracts a password fragment from a given string.
+ * <p/>
+ * {@see java.util.Matcher#find()}
+ **/
+ private static final String PASSWORD_EXTRACTING_REGEX =
+ "([\\w[.\\w]*]*(?i)pass[\\w]*=)([\\w]+)";
+
+ private static final Pattern PASSWORD_EXTRACTING_PATTERN = Pattern
+ .compile(PASSWORD_EXTRACTING_REGEX);
+
+ /**
+ * Returns a map where keys are masked if they are considered a password.
+ * There are two cases when passwords are masked:
+ * 1. The key contains the string "pass". In this case, the entire value is considered a password and replaced completely with
+ * a masking string.
+ * 2. The value matches a regular expression. Strings like "HADOOP_CREDSTORE_PASSWORD=pwd123" or
+ * "-Djavax.net.ssl.trustStorePassword=password" are considered password definition strings and the text after the equal sign
+ * is replaced with a masking string.
+ *
+ * @param unmasked key-value map
+ * @return A new map where values are changed based on the replace algorithm described above
+ */
+ public Map<String, String> mask(Map<String, String> unmasked) {
+ return Maps.transformEntries(unmasked, new Maps.EntryTransformer<String, String, String>() {
+ @Override
+ public String transformEntry(@Nonnull String key, @Nonnull String value) {
+ checkNotNull(key, "key has to be set");
+ checkNotNull(value, "value has to be set");
+
+ if (isPasswordKey(key)) {
+ return PASSWORD_MASK;
+ }
+
+ return maskPasswordsIfNecessary(value);
+ }
+ });
+ }
+
+ /**
+ * Masks passwords inside a string. A substring is subject to password masking if it looks like
+ * "HADOOP_CREDSTORE_PASSWORD=pwd123" or "-Djavax.net.ssl.trustStorePassword=password". The text after the equal sign is
+ * replaced with a masking string.
+ *
+ * @param unmasked String which might contain passwords
+ * @return The same string where passwords are replaced with a masking string. If there is no password inside, the original
+ * string is returned.
+ */
+ public String maskPasswordsIfNecessary(String unmasked) {
+ if (containsPasswordFragment(unmasked)) {
+ return maskPasswordFragments(unmasked);
+ } else {
+ return unmasked;
+ }
+ }
+
+ private boolean isPasswordKey(String key) {
+ return key.toLowerCase().contains(PASSWORD_KEY);
+ }
+
+ private boolean containsPasswordFragment(String maybePasswordFragments) {
+ return PASSWORD_CONTAINING_PATTERN
+ .matcher(maybePasswordFragments)
+ .matches();
+ }
+
+ private String maskPasswordFragments(String maybePasswordFragments) {
+ StringBuilder maskedBuilder = new StringBuilder();
+ Matcher passwordFragmentsMatcher = PASSWORD_EXTRACTING_PATTERN
+ .matcher(maybePasswordFragments);
+
+ int start = 0, end;
+ while (passwordFragmentsMatcher.find()) {
+ end = passwordFragmentsMatcher.start();
+
+ maskedBuilder.append(maybePasswordFragments.substring(start, end));
+ maskedBuilder.append(passwordFragmentsMatcher.group(1));
+ maskedBuilder.append(PASSWORD_MASK);
+
+ start = passwordFragmentsMatcher.end();
+ }
+
+ maskedBuilder.append(maybePasswordFragments.substring(start));
+
+ return maskedBuilder.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/oozie/blob/b721224f/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/TestPasswordMasker.java
----------------------------------------------------------------------
diff --git a/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/TestPasswordMasker.java b/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/TestPasswordMasker.java
new file mode 100644
index 0000000..08e55e1
--- /dev/null
+++ b/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/TestPasswordMasker.java
@@ -0,0 +1,140 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oozie.action.hadoop;
+
+import org.codehaus.jackson.map.ObjectMapper;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class TestPasswordMasker {
+ private PasswordMasker passwordMasker;
+
+ @Before
+ public void setup() {
+ passwordMasker = new PasswordMasker();
+ }
+
+ @Test
+ public void testWhenJavaSystemPropertiesAreAskedPasswordsAppearMasked() throws Exception {
+ Map<String, String> masked = passwordMasker.mask(jsonToMap("/instrumentation-system-properties.json"));
+
+ assertPasswordValueIsMasked(masked, "javax.net.ssl.trustStorePassword");
+ assertPasswordValueIsMasked(masked, "oozie.https.keystore.pass");
+ }
+
+ @Test
+ public void testWhenOSEnvIsAskedPasswordsAppearMasked() throws Exception {
+ Map<String, String> masked = passwordMasker.mask(jsonToMap("/instrumentation-os-env.json"));
+
+ assertPasswordValueIsMasked(masked, "HADOOP_CREDSTORE_PASSWORD");
+ assertPasswordValueIsMasked(masked, "OOZIE_HTTPS_KEYSTORE_PASSWORD");
+ assertPasswordValueIsMasked(masked, "OOZIE_HTTPS_TRUSTSTORE_PASSWORD");
+
+ assertPasswordValueFragmentIsMasked(masked, "CATALINA_OPTS", "-Doozie.https.keystore.pass=");
+ assertPasswordValueFragmentIsMasked(masked, "CATALINA_OPTS", "-Djavax.net.ssl.trustStorePassword=");
+
+ assertValueFragmentIsPresent(masked, "CATALINA_OPTS", "-Xmx1024m");
+ assertValueFragmentIsPresent(masked, "CATALINA_OPTS", "-Doozie.https.keystore.file=/Users/forsage/.keystore");
+ assertValueFragmentIsPresent(masked, "CATALINA_OPTS", "-Djava.library.path=");
+ }
+
+ @Test
+ public void testMaskNothing() {
+ assertEquals("abcd", passwordMasker.maskPasswordsIfNecessary("abcd"));
+ assertEquals("abcd abcd", passwordMasker.maskPasswordsIfNecessary("abcd abcd"));
+ assertEquals("-Djava.net.pasX=pwd1", passwordMasker.maskPasswordsIfNecessary("-Djava.net.pasX=pwd1"));
+ }
+
+ @Test
+ public void testMaskJavaSystemProp() {
+ assertEquals("-Djava.sysprop.password=*****", passwordMasker.maskPasswordsIfNecessary("-Djava.sysprop.password=pwd123"));
+ }
+
+ @Test
+ public void testMaskJavaSystemPropWithWhiteSpaces() {
+ assertEquals(" -Djava.sysprop.password=***** ",
+ passwordMasker.maskPasswordsIfNecessary(" -Djava.sysprop.password=pwd123 "));
+ }
+
+ @Test
+ public void testMaskTwoJavaSystemProps() {
+ assertEquals("-Djava.sysprop.password=***** -Djava.another.password=*****",
+ passwordMasker.maskPasswordsIfNecessary("-Djava.sysprop.password=pwd123 -Djava.another.password=pwd456"));
+ }
+
+ @Test
+ public void testMaskEnvironmentVariable() {
+ assertEquals("DUMMY_PASSWORD=*****", passwordMasker.maskPasswordsIfNecessary("DUMMY_PASSWORD=dummy"));
+ }
+
+ @Test
+ public void testMaskTwoEnvironmentVariables() {
+ assertEquals("DUMMY_PASSWORD=*****:ANOTHER_PASSWORD=*****",
+ passwordMasker.maskPasswordsIfNecessary("DUMMY_PASSWORD=dummy:ANOTHER_PASSWORD=pwd123"));
+ }
+
+ @Test
+ public void testMaskRandomMatchingStuff() {
+ assertEquals("aa -Djava.sysprop.password=***** bb DUMMY_PASSWORD=***** cc",
+ passwordMasker.maskPasswordsIfNecessary("aa -Djava.sysprop.password=1234 bb DUMMY_PASSWORD=dummy cc"));
+ }
+
+ @SuppressWarnings("unchecked")
+ private Map<String, String> jsonToMap(String jsonPath) throws IOException {
+ return new ObjectMapper().readValue(getClass().getResourceAsStream(jsonPath), HashMap.class);
+ }
+
+ private void assertPasswordValueIsMasked(Map<String, String> mapContainingMaskedPassword, String passwordKey) {
+ assertEquals(String.format("Value of key '%s' should be masked.", passwordKey),
+ "*****",
+ mapContainingMaskedPassword.get(passwordKey));
+ }
+
+ private void assertPasswordValueFragmentIsMasked(Map<String, String> mapContainingMaskedPassword, String passwordKey,
+ String passwordFragmentKey) {
+ assertEquals(
+ String.format("Value fragment of password key '%s' and password fragment key '%s' should be masked.",
+ passwordKey,
+ passwordFragmentKey),
+ "*****",
+ getFragmentValue(mapContainingMaskedPassword.get(passwordKey), passwordFragmentKey));
+ }
+
+ private String getFragmentValue(String base, String fragmentKey) {
+ for (String fragment : base.split(" ")) {
+ if (fragment.startsWith(fragmentKey)) {
+ return fragment.substring(fragmentKey.length());
+ }
+ }
+
+ return null;
+ }
+
+ private void assertValueFragmentIsPresent(Map<String, String> masked, String key, String valueFragment) {
+ assertTrue(String.format("For key '%s' value fragment '%s' should be present.", key, valueFragment),
+ masked.get(key).contains(valueFragment));
+ }
+}
http://git-wip-us.apache.org/repos/asf/oozie/blob/b721224f/sharelib/oozie/src/test/resources/instrumentation-os-env.json
----------------------------------------------------------------------
diff --git a/sharelib/oozie/src/test/resources/instrumentation-os-env.json b/sharelib/oozie/src/test/resources/instrumentation-os-env.json
new file mode 100644
index 0000000..e85cd8d
--- /dev/null
+++ b/sharelib/oozie/src/test/resources/instrumentation-os-env.json
@@ -0,0 +1,47 @@
+{
+ "HADOOP_CREDSTORE_PASSWORD": "password",
+ "OOZIE_HTTPS_KEYSTORE_PASSWORD": "password",
+ "OOZIE_HTTPS_TRUSTSTORE_PASSWORD": "password",
+ "PATH": "/opt/local/bin:/opt/local/sbin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin",
+ "HISTCONTROL": "ignoreboth",
+ "OOZIE_DATA": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/data",
+ "CATALINA_PID": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/oozie-server/temp/oozie.pid",
+ "MC_SID": "8597",
+ "OOZIE_INSTANCE_ID": "Budapests-MacBook-Pro.local",
+ "OOZIE_HTTP_HOSTNAME": "Budapests-MacBook-Pro.local",
+ "JAVA_HOME": "/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home",
+ "CATALINA_OUT": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/logs/catalina.out",
+ "TERM": "xterm-256color",
+ "LANG": "en_US.UTF-8",
+ "CATALINA_BASE": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/oozie-server",
+ "OOZIE_CONFIG_FILE": "oozie-site.xml",
+ "LOGNAME": "forsage",
+ "OOZIE_HOME": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT",
+ "XPC_SERVICE_NAME": "0",
+ "PWD": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT",
+ "TERM_PROGRAM_VERSION": "361.1",
+ "JAVA_MAIN_CLASS_33220": "org.apache.catalina.startup.Bootstrap",
+ "_": "/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home/bin/java",
+ "SHELL": "/bin/bash",
+ "OOZIE_CONFIG": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/conf",
+ "TERM_PROGRAM": "Apple_Terminal",
+ "OOZIE_ADMIN_PORT": "11001",
+ "CATALINA_OPTS": " -Xmx1024m -Dderby.stream.error.file=/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/logs/derby.log -Doozie.home.dir=/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT -Doozie.config.dir=/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/conf -Doozie.log.dir=/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/logs -Doozie.data.dir=/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/data -Doozie.instance.id=Budapests-MacBook-Pro.local -Doozie.config.file=oozie-site.xml -Doozie.log4j.file=oozie-log4j.properties -Doozie.log4j.reload=10 -Doozie.http.hostname=Budapests-MacBook-Pro.local -Doozie.admin.port=11001 -Doozie.http.port=11000 -Doozie.https.port=11443 -Doozie.base.url=http://Budapests-MacBook-Pro.local:11000/oozie -Doozie.https.keystore.file=/Users
/forsage/.keystore -Doozie.https.keystore.pass=password -Djavax.net.ssl.trustStorePassword=password -Djava.library.path=",
+ "USER": "forsage",
+ "OOZIE_LOG": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/logs",
+ "OOZIE_LOG4J_RELOAD": "10",
+ "TMPDIR": "/var/folders/yy/gkvmmzn91vv_lb2_bmymxz600000gp/T/",
+ "SSH_AUTH_SOCK": "/private/tmp/com.apple.launchd.NvNvd0j95Z/Listeners",
+ "MC_TMPDIR": "/var/folders/yy/gkvmmzn91vv_lb2_bmymxz600000gp/T/mc-forsage",
+ "XPC_FLAGS": "0x0",
+ "OOZIE_BASE_URL": "http://Budapests-MacBook-Pro.local:11000/oozie",
+ "TERM_SESSION_ID": "283A05FC-7501-4B9D-B3E3-BDDD3521593C",
+ "OOZIE_HTTPS_KEYSTORE_FILE": "/Users/forsage/.keystore",
+ "__CF_USER_TEXT_ENCODING": "0x1F6:0x0:0x0",
+ "Apple_PubSub_Socket_Render": "/private/tmp/com.apple.launchd.6kR2bgiMHn/Render",
+ "OOZIE_HTTP_PORT": "11000",
+ "OOZIE_HTTPS_PORT": "11443",
+ "SHLVL": "3",
+ "HOME": "/Users/forsage",
+ "OOZIE_LOG4J_FILE": "oozie-log4j.properties"
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/oozie/blob/b721224f/sharelib/oozie/src/test/resources/instrumentation-system-properties.json
----------------------------------------------------------------------
diff --git a/sharelib/oozie/src/test/resources/instrumentation-system-properties.json b/sharelib/oozie/src/test/resources/instrumentation-system-properties.json
new file mode 100644
index 0000000..61430d2
--- /dev/null
+++ b/sharelib/oozie/src/test/resources/instrumentation-system-properties.json
@@ -0,0 +1,88 @@
+{
+ "javax.net.ssl.trustStorePassword": "password",
+ "oozie.https.keystore.pass": "password",
+ "gopherProxySet": "false",
+ "awt.toolkit": "sun.lwawt.macosx.LWCToolkit",
+ "oozie.base.url": "http://Budapests-MacBook-Pro.local:11000/oozie",
+ "file.encoding.pkg": "sun.io",
+ "java.specification.version": "1.8",
+ "sun.cpu.isalist": "",
+ "sun.jnu.encoding": "UTF-8",
+ "java.class.path": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/oozie-server/bin/bootstrap.jar",
+ "java.vm.vendor": "Oracle Corporation",
+ "sun.arch.data.model": "64",
+ "sun.font.fontmanager": "sun.font.CFontManager",
+ "catalina.useNaming": "true",
+ "java.vendor.url": "http://java.oracle.com/",
+ "user.timezone": "Europe/Budapest",
+ "os.name": "Mac OS X",
+ "java.vm.specification.version": "1.8",
+ "oozie.http.hostname": "Budapests-MacBook-Pro.local",
+ "oozie.instance.id": "Budapests-MacBook-Pro.local",
+ "sun.java.launcher": "SUN_STANDARD",
+ "user.country": "US",
+ "oozie.log.dir": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/logs",
+ "oozie.home.dir": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT",
+ "sun.boot.library.path": "/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home/jre/lib",
+ "sun.java.command": "org.apache.catalina.startup.Bootstrap start",
+ "http.nonProxyHosts": "local|*.local|169.254/16|*.169.254/16",
+ "sun.cpu.endian": "little",
+ "user.home": "/Users/forsage",
+ "user.language": "en",
+ "java.specification.vendor": "Oracle Corporation",
+ "java.naming.factory.url.pkgs": "org.apache.naming",
+ "java.home": "/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home/jre",
+ "oozie.config.file": "oozie-site.xml",
+ "oozie.log4j.reload": "10",
+ "file.separator": "/",
+ "oozie.https.keystore.file": "/Users/forsage/.keystore",
+ "line.separator": "\n",
+ "java.vm.specification.vendor": "Oracle Corporation",
+ "java.specification.name": "Java Platform API Specification",
+ "derby.stream.error.file": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/logs/derby.log",
+ "oozie.log4j.file": "oozie-log4j.properties",
+ "oozie.admin.port": "11001",
+ "java.awt.graphicsenv": "sun.awt.CGraphicsEnvironment",
+ "package.access": "sun.,org.apache.catalina.,org.apache.coyote.,org.apache.jasper.,org.apache.naming.resources.,org.apache.tomcat.,sun.beans.",
+ "package.definition": "sun.,java.,org.apache.catalina.,org.apache.coyote.,org.apache.jasper.,org.apache.naming.,org.apache.tomcat.",
+ "sun.boot.class.path": "/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home/jre/lib/resources.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home/jre/lib/rt.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home/jre/lib/sunrsasign.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home/jre/lib/jsse.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home/jre/lib/jce.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home/jre/lib/charsets.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home/jre/lib/jfr.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home/jre/classes",
+ "server.loader": "",
+ "java.util.logging.config.file": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/oozie-server/conf/logging.properties",
+ "sun.management.compiler": "HotSpot 64-Bit Tiered Compilers",
+ "oozie.data.dir": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/data",
+ "ftp.nonProxyHosts": "local|*.local|169.254/16|*.169.254/16",
+ "java.runtime.version": "1.8.0_102-b14",
+ "java.naming.factory.initial": "org.apache.naming.java.javaURLContextFactory",
+ "user.name": "forsage",
+ "oozie.https.port": "11443",
+ "path.separator": ":",
+ "common.loader": "${catalina.base}/lib,${catalina.base}/lib/*.jar,${catalina.home}/lib,${catalina.home}/lib/*.jar",
+ "os.version": "10.11.6",
+ "java.endorsed.dirs": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/oozie-server/endorsed",
+ "java.runtime.name": "Java(TM) SE Runtime Environment",
+ "file.encoding": "UTF-8",
+ "java.vm.name": "Java HotSpot(TM) 64-Bit Server VM",
+ "java.vendor.url.bug": "http://bugreport.sun.com/bugreport/",
+ "java.io.tmpdir": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/oozie-server/temp",
+ "oozie.http.port": "11000",
+ "catalina.home": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/oozie-server",
+ "java.version": "1.8.0_102",
+ "user.dir": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT",
+ "oozie.config.dir": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/conf",
+ "os.arch": "x86_64",
+ "java.vm.specification.name": "Java Virtual Machine Specification",
+ "java.awt.printerjob": "sun.lwawt.macosx.CPrinterJob",
+ "sun.os.patch.level": "unknown",
+ "catalina.base": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/oozie-server",
+ "shared.loader": "",
+ "java.util.logging.manager": "org.apache.juli.ClassLoaderLogManager",
+ "java.library.path": "",
+ "java.vendor": "Oracle Corporation",
+ "java.vm.info": "mixed mode",
+ "java.vm.version": "25.102-b14",
+ "sun.io.unicode.encoding": "UnicodeBig",
+ "java.ext.dirs": "/Users/forsage/Library/Java/Extensions:/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home/jre/lib/ext:/Library/Java/Extensions:/Network/Library/Java/Extensions:/System/Library/Java/Extensions:/usr/lib/java",
+ "tomcat.util.buf.StringCache.byte.enabled": "true",
+ "java.class.version": "52.0",
+ "socksNonProxyHosts": "local|*.local|169.254/16|*.169.254/16"
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/oozie/blob/b721224f/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
----------------------------------------------------------------------
diff --git a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
index db1e197..88ac64e 100644
--- a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
+++ b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
@@ -263,8 +263,9 @@ public class SparkMain extends LauncherMain {
System.out.println("Oozie Spark action configuration");
System.out.println("=================================================================");
System.out.println();
+ PasswordMasker passwordMasker = new PasswordMasker();
for (String arg : sparkArgs) {
- System.out.println(" " + arg);
+ System.out.println(" " + passwordMasker.maskPasswordsIfNecessary(arg));
}
System.out.println();
try {
[23/31] oozie git commit: OOZIE-2872 Address backward compatibility
issue introduced by OOZIE-2748 (pbacsko)
Posted by sa...@apache.org.
OOZIE-2872 Address backward compatibility issue introduced by OOZIE-2748 (pbacsko)
(cherry picked from commit e0b7cde711b1b9e1a03660ec635041eeb9755049)
core/src/main/resources/oozie-default.xml
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/7ea8f19a
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/7ea8f19a
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/7ea8f19a
Branch: refs/heads/branch-4.3
Commit: 7ea8f19a756fd8775671bd9bd71b210666547e6d
Parents: f553c4e
Author: Peter Bacsko <pb...@cloudera.com>
Authored: Mon May 15 12:50:28 2017 +0200
Committer: satishsaley <sa...@apache.org>
Committed: Fri Dec 8 16:34:55 2017 -0800
----------------------------------------------------------------------
.../oozie/action/hadoop/JavaActionExecutor.java | 6 ++++
core/src/main/resources/oozie-default.xml | 8 +++++
.../action/hadoop/TestJavaActionExecutor.java | 17 +++++++--
release-log.txt | 1 +
.../oozie/action/hadoop/LauncherMapper.java | 35 ++++++++++++------
.../oozie/action/hadoop/TestLauncherMapper.java | 37 +++++++++++++++++---
6 files changed, 87 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/7ea8f19a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
index 8a403dc..50577fd 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
@@ -993,6 +993,12 @@ public class JavaActionExecutor extends ActionExecutor {
args[i] = list.get(i).getTextTrim();
}
LauncherMapperHelper.setupMainArguments(launcherJobConf, args);
+ // backward compatibility flag - see OOZIE-2872
+ if (ConfigurationService.getBoolean(LauncherMapper.CONF_OOZIE_NULL_ARGS_ALLOWED)) {
+ launcherJobConf.setBoolean(LauncherMapper.CONF_OOZIE_NULL_ARGS_ALLOWED, true);
+ } else {
+ launcherJobConf.setBoolean(LauncherMapper.CONF_OOZIE_NULL_ARGS_ALLOWED, false);
+ }
// Make mapred.child.java.opts and mapreduce.map.java.opts equal, but give values from the latter priority; also append
// <java-opt> and <java-opts> and give those highest priority
http://git-wip-us.apache.org/repos/asf/oozie/blob/7ea8f19a/core/src/main/resources/oozie-default.xml
----------------------------------------------------------------------
diff --git a/core/src/main/resources/oozie-default.xml b/core/src/main/resources/oozie-default.xml
index 8c55a0d..dbab18b 100644
--- a/core/src/main/resources/oozie-default.xml
+++ b/core/src/main/resources/oozie-default.xml
@@ -2850,4 +2850,12 @@ will be the requeue interval for the actions which are waiting for a long time w
</description>
</property>
+ <property>
+ <name>oozie.actions.null.args.allowed</name>
+ <value>true</value>
+ <description>
+ When set to true, empty arguments (like <arg></arg>) will be passed as "null" to the main method of a
+ given action. That is, the args[] array will contain "null" elements. When set to false, then "nulls" are removed.
+ </description>
+ </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/oozie/blob/7ea8f19a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
index 1c4b429..a6e1d50 100644
--- a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
+++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
@@ -2941,7 +2941,18 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase {
assertEquals("DEBUG", conf.get(oozieActionHiveRootLogger));
}
- public void testEmptyArgs() throws Exception {
+ public void testEmptyArgsWithNullArgsNotAllowed() throws Exception {
+ testEmptyArgs(false, "SUCCEEDED", WorkflowAction.Status.OK);
+ }
+
+ public void testEmptyArgsWithNullArgsAllowed() throws Exception {
+ testEmptyArgs(true, "FAILED/KILLED", WorkflowAction.Status.ERROR);
+ }
+
+ private void testEmptyArgs(boolean nullArgsAllowed, String expectedExternalStatus, WorkflowAction.Status expectedStatus)
+ throws Exception {
+ ConfigurationService.setBoolean(LauncherMapper.CONF_OOZIE_NULL_ARGS_ALLOWED, nullArgsAllowed);
+
String actionXml = "<java>" +
"<job-tracker>" + getJobTrackerUri() + "</job-tracker>" +
"<name-node>" + getNameNodeUri() + "</name-node>" +
@@ -2961,10 +2972,10 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase {
ActionExecutor ae = new JavaActionExecutor();
ae.check(context, context.getAction());
assertTrue(ae.isCompleted(context.getAction().getExternalStatus()));
- assertEquals("SUCCEEDED", context.getAction().getExternalStatus());
+ assertEquals(expectedExternalStatus, context.getAction().getExternalStatus());
assertNull(context.getAction().getData());
ae.end(context, context.getAction());
- assertEquals(WorkflowAction.Status.OK, context.getAction().getStatus());
+ assertEquals(expectedStatus, context.getAction().getStatus());
}
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/7ea8f19a/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 2b36ec0..a79d8c2 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.3.1 release
+OOZIE-2872 Address backward compatibility issue introduced by OOZIE-2748 (pbacsko)
OOZIE-2863 SLACalculatorMemory.loadOnRestart causing delay in server start (puru via satishsaley)
OOZIE-2862 Coord change command doesn't change job to running if job was killed without creating any actions (puru)
OOZIE-2811 amend Add support for filtering out properties from SparkConfigurationService
http://git-wip-us.apache.org/repos/asf/oozie/blob/7ea8f19a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java
----------------------------------------------------------------------
diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java
index 8edebac..8657c67 100644
--- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java
+++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java
@@ -63,6 +63,7 @@ public class LauncherMapper<K1, V1, K2, V2> implements Mapper<K1, V1, K2, V2>, R
static final String CONF_OOZIE_EXTERNAL_STATS_MAX_SIZE = "oozie.external.stats.max.size";
static final String OOZIE_ACTION_CONFIG_CLASS = ACTION_PREFIX + "config.class";
static final String CONF_OOZIE_ACTION_FS_GLOB_MAX = ACTION_PREFIX + "fs.glob.max";
+ static final String CONF_OOZIE_NULL_ARGS_ALLOWED = ACTION_PREFIX + "null.args.allowed";
static final String COUNTER_GROUP = "oozie.launcher";
static final String COUNTER_LAUNCHER_ERROR = "oozie.launcher.error";
@@ -497,18 +498,28 @@ public class LauncherMapper<K1, V1, K2, V2> implements Mapper<K1, V1, K2, V2>, R
public static String[] getMainArguments(Configuration conf) {
String[] args = new String[conf.getInt(CONF_OOZIE_ACTION_MAIN_ARG_COUNT, 0)];
- int pos = 0;
- for (int i = 0; i < args.length; i++) {
- String arg = conf.get(CONF_OOZIE_ACTION_MAIN_ARG_PREFIX + i);
- if (!Strings.isNullOrEmpty(arg)) {
- args[pos++] = conf.get(CONF_OOZIE_ACTION_MAIN_ARG_PREFIX + i);
+ String[] retArray;
+
+ if (conf.getBoolean(CONF_OOZIE_NULL_ARGS_ALLOWED, true)) {
+ for (int i = 0; i < args.length; i++) {
+ args[i] = conf.get(CONF_OOZIE_ACTION_MAIN_ARG_PREFIX + i);
+ }
+
+ retArray = args;
+ } else {
+ int pos = 0;
+ for (int i = 0; i < args.length; i++) {
+ String arg = conf.get(CONF_OOZIE_ACTION_MAIN_ARG_PREFIX + i);
+ if (!Strings.isNullOrEmpty(arg)) {
+ args[pos++] = conf.get(CONF_OOZIE_ACTION_MAIN_ARG_PREFIX + i);
+ }
}
- }
- // this is to skip null args, that is <arg></arg> in the workflow XML -- in this case,
- // args[] might look like {"arg1", "arg2", null, null} at this point
- String[] retArray = new String[pos];
- System.arraycopy(args, 0, retArray, 0, pos);
+ // this is to skip null args, that is <arg></arg> in the workflow XML -- in this case,
+ // args[] might look like {"arg1", "arg2", null, null} at this point
+ retArray = new String[pos];
+ System.arraycopy(args, 0, retArray, 0, pos);
+ }
return retArray;
}
@@ -632,6 +643,10 @@ public class LauncherMapper<K1, V1, K2, V2> implements Mapper<K1, V1, K2, V2>, R
System.out.println(banner);
boolean maskNextArg = false;
for (String arg : args) {
+ if (arg == null) {
+ arg = "null"; // prevent NPE in pwd masking
+ }
+
if (maskNextArg) {
System.out.println(" " + "********");
maskNextArg = false;
http://git-wip-us.apache.org/repos/asf/oozie/blob/7ea8f19a/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/TestLauncherMapper.java
----------------------------------------------------------------------
diff --git a/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/TestLauncherMapper.java b/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/TestLauncherMapper.java
index 1dd8002..51b1d6f 100644
--- a/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/TestLauncherMapper.java
+++ b/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/TestLauncherMapper.java
@@ -23,6 +23,7 @@ import static org.apache.oozie.action.hadoop.LauncherMapper.CONF_OOZIE_ACTION_MA
import static org.junit.Assert.assertTrue;
import static org.mockito.BDDMockito.given;
import static org.mockito.Matchers.eq;
+import static org.mockito.Matchers.anyBoolean;
import java.util.Arrays;
import java.util.List;
@@ -41,8 +42,9 @@ public class TestLauncherMapper {
private Configuration conf; // we have to use mock, because conf.set(null) throws exception
@Test
- public void testLauncherMapperArgsHandlingWithoutNulls() {
+ public void testArgsHandlingWithoutNullsAndNullsNotAllowed() {
setupConf(Lists.newArrayList("a", "b", "c"));
+ setEnableNullArgsAllowed(false);
String args[] = LauncherMapper.getMainArguments(conf);
@@ -50,8 +52,9 @@ public class TestLauncherMapper {
}
@Test
- public void testLauncherMapperArgsHandlingWhenArgsContainNulls() {
+ public void testHandlingWhenArgsContainNullsAndNullsNotAllowed() {
setupConf(Lists.newArrayList("a", null, "b", null, "c"));
+ setEnableNullArgsAllowed(false);
String args[] = LauncherMapper.getMainArguments(conf);
@@ -59,8 +62,9 @@ public class TestLauncherMapper {
}
@Test
- public void testLauncherMapperArgsHandlingWhenArgsContainsNullsOnly() {
+ public void testArgsHandlingWhenArgsContainsNullsOnlyAndNullsNotAllowed() {
setupConf(Lists.<String>newArrayList(null, null, null));
+ setEnableNullArgsAllowed(false);
String args[] = LauncherMapper.getMainArguments(conf);
@@ -68,14 +72,35 @@ public class TestLauncherMapper {
}
@Test
- public void testLauncherMapperArgsHandlingWhenArgsContainsOneNull() {
+ public void testArgsHandlingWhenArgsContainsOneNullAndNullsNotAllowed() {
setupConf(Lists.<String>newArrayList((String) null));
+ setEnableNullArgsAllowed(false);
String args[] = LauncherMapper.getMainArguments(conf);
assertTrue(Arrays.equals(new String[] {}, args));
}
+ @Test
+ public void testHandlingWhenArgsContainNullsAndNullAllowed() {
+ setupConf(Lists.newArrayList("a", null, "b", null, "c"));
+ setEnableNullArgsAllowed(true);
+
+ String args[] = LauncherMapper.getMainArguments(conf);
+
+ assertTrue(Arrays.equals(new String[] { "a", null, "b", null, "c"}, args));
+ }
+
+ @Test
+ public void testArgsHandlingWhenArgsContainsOneNullAndNullsAllowed() {
+ setupConf(Lists.<String>newArrayList((String) null));
+ setEnableNullArgsAllowed(true);
+
+ String args[] = LauncherMapper.getMainArguments(conf);
+
+ assertTrue(Arrays.equals(new String[] { null }, args));
+ }
+
private void setupConf(List<String> argList) {
int argCount = argList.size();
@@ -85,4 +110,8 @@ public class TestLauncherMapper {
given(conf.get(eq(CONF_OOZIE_ACTION_MAIN_ARG_PREFIX + i))).willReturn(argList.get(i));
}
}
+
+ private void setEnableNullArgsAllowed(boolean nullArgsAllowed) {
+ given(conf.getBoolean(eq(LauncherMapper.CONF_OOZIE_NULL_ARGS_ALLOWED), anyBoolean())).willReturn(nullArgsAllowed);
+ }
}