You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@eagle.apache.org by ha...@apache.org on 2016/10/11 08:36:41 UTC
incubator-eagle git commit: [EAGLE-605] Fix MR feeder provider.xml
and remove useless configure
Repository: incubator-eagle
Updated Branches:
refs/heads/master 97978c60d -> fa85dc3e2
[EAGLE-605] Fix MR feeder provider.xml and remove useless configure
https://issues.apache.org/jira/browse/EAGLE-605
1. add missing configure from application.conf to provider.xml
2. remove useless configure like : pathContainsJobTrackerName/jobTrackerName/dryRun
3. Fix bug: add IsExecutable to ApplicationDesc so that web ui will know some applications can not be started, just static
Author: wujinhu <wu...@126.com>
Closes #490 from wujinhu/ui-test.
Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/fa85dc3e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/fa85dc3e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/fa85dc3e
Branch: refs/heads/master
Commit: fa85dc3e20f163ff84f39ca51d16c5d54fad0482
Parents: 97978c6
Author: wujinhu <wu...@126.com>
Authored: Tue Oct 11 16:36:24 2016 +0800
Committer: Hao Chen <ha...@apache.org>
Committed: Tue Oct 11 16:36:24 2016 +0800
----------------------------------------------------------------------
.../service/ApplicationOperationContext.java | 1 -
.../impl/ApplicationManagementServiceImpl.java | 22 ++++--
.../impl/ApplicationProviderSPILoader.java | 1 +
.../eagle/metadata/model/ApplicationDesc.java | 9 +++
.../eagle/metadata/model/ApplicationEntity.java | 11 +--
.../jpm/mr/history/MRHistoryJobConfig.java | 6 --
.../history/crawler/AbstractJobHistoryDAO.java | 42 +----------
.../history/crawler/JHFCrawlerDriverImpl.java | 21 ++----
.../mr/history/crawler/JobHistoryDAOImpl.java | 2 +-
....history.MRHistoryJobApplicationProvider.xml | 76 +++++++++----------
.../src/main/resources/application.conf | 5 --
....running.MRRunningJobApplicationProvider.xml | 77 ++++++++++++--------
.../src/main/resources/application.conf | 2 -
.../jpm/util/jobrecover/RunningJobManager.java | 4 +-
eagle-topology-assembly/pom.xml | 2 +-
15 files changed, 117 insertions(+), 164 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fa85dc3e/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationOperationContext.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationOperationContext.java b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationOperationContext.java
index 33a407c..3561374 100644
--- a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationOperationContext.java
+++ b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationOperationContext.java
@@ -90,7 +90,6 @@ public class ApplicationOperationContext implements Serializable, ApplicationLif
@Override
public void onInstall() {
- metadata.setExecutable(application.isExecutable());
if (metadata.getDescriptor().getStreams() != null) {
List<StreamDesc> streamDescToInstall = metadata.getDescriptor().getStreams().stream().map((streamDefinition -> {
StreamDefinition copied = streamDefinition.copy();
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fa85dc3e/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationManagementServiceImpl.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationManagementServiceImpl.java b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationManagementServiceImpl.java
index 04a7007..abd9197 100644
--- a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationManagementServiceImpl.java
+++ b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationManagementServiceImpl.java
@@ -21,6 +21,7 @@ import com.google.inject.Inject;
import com.google.inject.Singleton;
import com.typesafe.config.Config;
import org.apache.eagle.alert.metadata.IMetadataDao;
+import org.apache.eagle.app.Application;
import org.apache.eagle.app.service.ApplicationManagementService;
import org.apache.eagle.app.service.ApplicationOperations;
import org.apache.eagle.app.service.ApplicationOperationContext;
@@ -141,12 +142,14 @@ public class ApplicationManagementServiceImpl implements ApplicationManagementSe
@Override
public ApplicationEntity start(ApplicationOperations.StartOperation operation) throws ApplicationWrongStatusException {
ApplicationEntity applicationEntity = applicationEntityService.getByUUIDOrAppId(operation.getUuid(), operation.getAppId());
+ Application application = applicationProviderService.getApplicationProviderByType(applicationEntity.getDescriptor().getType()).getApplication();
+ Preconditions.checkArgument(application.isExecutable(), "Application is not executable");
+
ApplicationEntity.Status currentStatus = applicationEntity.getStatus();
try {
if (currentStatus == ApplicationEntity.Status.INITIALIZED || currentStatus == ApplicationEntity.Status.STOPPED) {
ApplicationOperationContext applicationOperationContext = new ApplicationOperationContext(
- applicationProviderService.getApplicationProviderByType(applicationEntity.getDescriptor().getType()).getApplication(),
- applicationEntity, config, alertMetadataService);
+ application, applicationEntity, config, alertMetadataService);
applicationOperationContext.onStart();
//Only when topology submitted successfully can the state change to STARTING
@@ -172,9 +175,11 @@ public class ApplicationManagementServiceImpl implements ApplicationManagementSe
@Override
public ApplicationEntity stop(ApplicationOperations.StopOperation operation) throws ApplicationWrongStatusException {
ApplicationEntity applicationEntity = applicationEntityService.getByUUIDOrAppId(operation.getUuid(), operation.getAppId());
+ Application application = applicationProviderService.getApplicationProviderByType(applicationEntity.getDescriptor().getType()).getApplication();
+ Preconditions.checkArgument(application.isExecutable(), "Application is not executable");
+
ApplicationOperationContext applicationOperationContext = new ApplicationOperationContext(
- applicationProviderService.getApplicationProviderByType(applicationEntity.getDescriptor().getType()).getApplication(),
- applicationEntity, config, alertMetadataService);
+ application, applicationEntity, config, alertMetadataService);
ApplicationEntity.Status currentStatus = applicationEntity.getStatus();
try {
if (currentStatus == ApplicationEntity.Status.RUNNING) {
@@ -200,12 +205,13 @@ public class ApplicationManagementServiceImpl implements ApplicationManagementSe
@Override
public ApplicationEntity.Status getStatus(ApplicationOperations.CheckStatusOperation operation) {
- ApplicationEntity applicationEntity = null;
try {
- applicationEntity = applicationEntityService.getByUUIDOrAppId(operation.getUuid(), operation.getAppId());
+ ApplicationEntity applicationEntity = applicationEntityService.getByUUIDOrAppId(operation.getUuid(), operation.getAppId());
+ Application application = applicationProviderService.getApplicationProviderByType(applicationEntity.getDescriptor().getType()).getApplication();
+ Preconditions.checkArgument(application.isExecutable(), "Application is not executable");
+
ApplicationOperationContext applicationOperationContext = new ApplicationOperationContext(
- applicationProviderService.getApplicationProviderByType(applicationEntity.getDescriptor().getType()).getApplication(),
- applicationEntity, config, alertMetadataService);
+ application, applicationEntity, config, alertMetadataService);
ApplicationEntity.Status topologyStatus = applicationOperationContext.getStatus();
return topologyStatus;
} catch (IllegalArgumentException e) {
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fa85dc3e/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationProviderSPILoader.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationProviderSPILoader.java b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationProviderSPILoader.java
index bd96e28..9f52c9c 100644
--- a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationProviderSPILoader.java
+++ b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationProviderSPILoader.java
@@ -80,6 +80,7 @@ public class ApplicationProviderSPILoader extends ApplicationProviderLoader {
ApplicationProviderConfig providerConfig = new ApplicationProviderConfig();
providerConfig.setClassName(applicationProvider.getClass().getCanonicalName());
providerConfig.setJarPath(jarFileSupplier.apply(applicationProvider));
+ applicationProvider.getApplicationDesc().setExecutable(applicationProvider.getApplication().isExecutable());
applicationProvider.prepare(providerConfig, getConfig());
registerProvider(applicationProvider);
LOG.warn("Loaded {}:{} ({}) from {}",
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fa85dc3e/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/ApplicationDesc.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/ApplicationDesc.java b/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/ApplicationDesc.java
index 506bdc6..8bc6ee3 100644
--- a/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/ApplicationDesc.java
+++ b/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/ApplicationDesc.java
@@ -36,6 +36,15 @@ public class ApplicationDesc implements Serializable {
private Configuration configuration;
private List<StreamDefinition> streams;
private ApplicationDocs docs;
+ private boolean executable;
+
+ public boolean isExecutable() {
+ return executable;
+ }
+
+ public void setExecutable(boolean executable) {
+ this.executable = executable;
+ }
private List<ApplicationDependency> dependencies;
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fa85dc3e/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/ApplicationEntity.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/ApplicationEntity.java b/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/ApplicationEntity.java
index bae6ba7..5297de1 100644
--- a/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/ApplicationEntity.java
+++ b/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/ApplicationEntity.java
@@ -33,7 +33,6 @@ public class ApplicationEntity extends PersistenceEntity {
private String appId;
private SiteEntity site;
private ApplicationDesc descriptor;
- private boolean executable = true;
private Map<String, Object> configuration = new HashMap<>();
private Map<String, String> context = new HashMap<>();
@@ -153,15 +152,7 @@ public class ApplicationEntity extends PersistenceEntity {
this.streams = streams;
}
- public boolean isExecutable() {
- return executable;
- }
-
- public void setExecutable(boolean executable) {
- this.executable = executable;
- }
-
- public static enum Status {
+ public enum Status {
INITIALIZED("INITIALIZED"),
STARTING("STARTING"),
RUNNING("RUNNING"),
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fa85dc3e/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobConfig.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobConfig.java b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobConfig.java
index 839d2e4..37df5ad 100644
--- a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobConfig.java
+++ b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobConfig.java
@@ -85,13 +85,10 @@ public class MRHistoryJobConfig implements Serializable {
public static class JobHistoryEndpointConfig implements Serializable {
public String mrHistoryServerUrl;
public String basePath;
- public boolean pathContainsJobTrackerName;
- public String jobTrackerName;
public Map<String, String> hdfs;
}
public static class ControlConfig implements Serializable {
- public boolean dryRun;
public Class<? extends JobIdPartitioner> partitionerCls;
public boolean zeroBasedMonth;
public String timeZone;
@@ -158,14 +155,11 @@ public class MRHistoryJobConfig implements Serializable {
//parse job history endpoint
this.jobHistoryEndpointConfig.basePath = config.getString("endpointConfig.basePath");
- this.jobHistoryEndpointConfig.jobTrackerName = config.getString("endpointConfig.jobTrackerName");
this.jobHistoryEndpointConfig.mrHistoryServerUrl = config.getString("endpointConfig.mrHistoryServerUrl");
- this.jobHistoryEndpointConfig.pathContainsJobTrackerName = config.getBoolean("endpointConfig.pathContainsJobTrackerName");
for (Map.Entry<String, ConfigValue> entry : config.getConfig("endpointConfig.hdfs").entrySet()) {
this.jobHistoryEndpointConfig.hdfs.put(entry.getKey(), entry.getValue().unwrapped().toString());
}
//parse control config
- this.controlConfig.dryRun = config.getBoolean("controlConfig.dryRun");
try {
this.controlConfig.partitionerCls = (Class<? extends JobIdPartitioner>) Class.forName(config.getString("controlConfig.partitionerCls"));
assert this.controlConfig.partitionerCls != null;
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fa85dc3e/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/crawler/AbstractJobHistoryDAO.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/crawler/AbstractJobHistoryDAO.java b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/crawler/AbstractJobHistoryDAO.java
index 74489cd..9ed2ed2 100644
--- a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/crawler/AbstractJobHistoryDAO.java
+++ b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/crawler/AbstractJobHistoryDAO.java
@@ -45,35 +45,19 @@ public abstract class AbstractJobHistoryDAO implements JobHistoryLCM {
protected static final Pattern JOBID_PATTERN = Pattern.compile("job_\\d+_\\d+");
protected final String basePath;
- protected volatile String jobTrackerName;
public static final String JOB_CONF_POSTFIX = "_conf.xml";
private static final Timer timer = new Timer(true);
private static final long JOB_TRACKER_SYNC_DURATION = 10 * 60 * 1000; // 10 minutes
- private boolean pathContainsJobTrackerName;
-
- public AbstractJobHistoryDAO(String basePath, boolean pathContainsJobTrackerName, String startingJobTrackerName) throws Exception {
+ public AbstractJobHistoryDAO(String basePath) throws Exception {
this.basePath = basePath;
- this.pathContainsJobTrackerName = pathContainsJobTrackerName;
- jobTrackerName = startingJobTrackerName;
- if (this.pathContainsJobTrackerName) {
- if (startingJobTrackerName == null || startingJobTrackerName.isEmpty()) {
- throw new IllegalStateException("startingJobTrackerName should not be null or empty");
- }
- // start background thread to check what is current job tracker
- startThread(this.basePath);
- }
}
protected String buildWholePathToYearMonthDay(int year, int month, int day) {
StringBuilder sb = new StringBuilder();
sb.append(basePath);
- if (!pathContainsJobTrackerName && jobTrackerName != null && !jobTrackerName.isEmpty()) {
- sb.append("/");
- sb.append(jobTrackerName);
- }
sb.append(String.format(YEAR_MONTH_DAY_URL_FORMAT, year, month, day));
return sb.toString();
}
@@ -110,30 +94,6 @@ public abstract class AbstractJobHistoryDAO implements JobHistoryLCM {
return null;
}
- private void startThread(final String basePath) throws Exception {
- LOG.info("start an every-" + JOB_TRACKER_SYNC_DURATION / (60 * 1000) + "min timer task to check current jobTrackerName in background");
- // Automatically update current job tracker name in background every 30 minutes
- timer.scheduleAtFixedRate(new TimerTask() {
- @Override
- public void run() {
- try {
- LOG.info("regularly checking current jobTrackerName in background");
- final String _jobTrackerName = calculateJobTrackerName(basePath);
- if (_jobTrackerName != null && !_jobTrackerName.equals(jobTrackerName)) {
- LOG.info("jobTrackerName changed from " + jobTrackerName + " to " + _jobTrackerName);
- jobTrackerName = _jobTrackerName;
- }
- LOG.info("Current jobTrackerName is: " + jobTrackerName);
- } catch (Exception e) {
- LOG.error("failed to figure out current job tracker name that is not configured due to: " + e.getMessage(), e);
- } catch (Throwable t) {
- LOG.error("failed to figure out current job tracker name that is not configured due to: " + t.getMessage(), t);
- }
- }
- }, JOB_TRACKER_SYNC_DURATION, JOB_TRACKER_SYNC_DURATION);
- }
-
-
@Override
public void readFileContent(int year, int month, int day, int serialNumber, String jobHistoryFileName, JHFInputStreamCallback reader) throws Exception {
InputStream downloadIs;
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fa85dc3e/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/crawler/JHFCrawlerDriverImpl.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/crawler/JHFCrawlerDriverImpl.java b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/crawler/JHFCrawlerDriverImpl.java
index ae9df37..a33abeb 100644
--- a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/crawler/JHFCrawlerDriverImpl.java
+++ b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/crawler/JHFCrawlerDriverImpl.java
@@ -52,7 +52,6 @@ public class JHFCrawlerDriverImpl implements JHFCrawlerDriver {
private Set<String> processedJobFileNames = new HashSet<>();
private final JobProcessDate processDate = new JobProcessDate();
- private boolean dryRun;
private JHFInputStreamCallback reader;
protected boolean zeroBasedMonth = true;
@@ -65,10 +64,6 @@ public class JHFCrawlerDriverImpl implements JHFCrawlerDriver {
public JHFCrawlerDriverImpl(JHFInputStreamCallback reader,
JobHistoryLCM historyLCM, JobIdFilter jobFilter, int partitionId) throws Exception {
this.zeroBasedMonth = MRHistoryJobConfig.get().getControlConfig().zeroBasedMonth;
- this.dryRun = MRHistoryJobConfig.get().getControlConfig().dryRun;
- if (this.dryRun) {
- LOG.info("this is a dry run");
- }
this.reader = reader;
jhfLCM = historyLCM;//new JobHistoryDAOImpl(jobHistoryConfig);
this.partitionId = partitionId;
@@ -169,15 +164,15 @@ public class JHFCrawlerDriverImpl implements JHFCrawlerDriver {
LOG.warn("illegal job history file name : " + jobHistoryFile);
return -1;
}
- if (!dryRun) {
- jhfLCM.readFileContent(
- processDate.year,
- getActualMonth(processDate.month),
- processDate.day,
- Integer.valueOf(serialNumber),
- jobHistoryFile,
+
+ jhfLCM.readFileContent(
+ processDate.year,
+ getActualMonth(processDate.month),
+ processDate.day,
+ Integer.valueOf(serialNumber),
+ jobHistoryFile,
reader);
- }
+
JobHistoryZKStateManager.instance().addProcessedJob(String.format(FORMAT_JOB_PROCESS_DATE,
this.processDate.year,
this.processDate.month + 1,
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fa85dc3e/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/crawler/JobHistoryDAOImpl.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/crawler/JobHistoryDAOImpl.java b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/crawler/JobHistoryDAOImpl.java
index a035357..07ca5c3 100644
--- a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/crawler/JobHistoryDAOImpl.java
+++ b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/crawler/JobHistoryDAOImpl.java
@@ -40,7 +40,7 @@ public class JobHistoryDAOImpl extends AbstractJobHistoryDAO {
private FileSystem hdfs;
public JobHistoryDAOImpl(JobHistoryEndpointConfig endpointConfig) throws Exception {
- super(endpointConfig.basePath, endpointConfig.pathContainsJobTrackerName, endpointConfig.jobTrackerName);
+ super(endpointConfig.basePath);
for (Map.Entry<String, String> entry : endpointConfig.hdfs.entrySet()) {
this.conf.set(entry.getKey(), entry.getValue());
LOG.info("conf key {}, conf value {}", entry.getKey(), entry.getValue());
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fa85dc3e/eagle-jpm/eagle-jpm-mr-history/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.history.MRHistoryJobApplicationProvider.xml
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.history.MRHistoryJobApplicationProvider.xml b/eagle-jpm/eagle-jpm-mr-history/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.history.MRHistoryJobApplicationProvider.xml
index 02e8c05..66111eb 100644
--- a/eagle-jpm/eagle-jpm-mr-history/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.history.MRHistoryJobApplicationProvider.xml
+++ b/eagle-jpm/eagle-jpm-mr-history/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.history.MRHistoryJobApplicationProvider.xml
@@ -28,73 +28,73 @@
<value>sandbox</value>
</property>
<property>
- <name>jobExtractorConfig.mrVersion</name>
- <value>MRVer2</value>
+ <name>workers</name>
+ <displayName>storm worker number</displayName>
+ <value>4</value>
</property>
<property>
- <name>jobExtractorConfig.readTimeOutSeconds</name>
- <displayName>zkPort</displayName>
- <value>10</value>
+ <name>envContextConfig.parallelismConfig.mrHistoryJobExecutor</name>
+ <value>4</value>
</property>
<property>
- <name>dataSourceConfig.zkQuorum</name>
- <value>sandbox.hortonworks.com:2181</value>
+ <name>envContextConfig.tasks.mrHistoryJobExecutor</name>
+ <value>4</value>
</property>
<property>
- <name>dataSourceConfig.zkPort</name>
- <value>2181</value>
+ <name>jobExtractorConfig.mrVersion</name>
+ <value>MRVer2</value>
</property>
<property>
- <name>dataSourceConfig.zkSessionTimeoutMs</name>
- <value>15000</value>
+ <name>jobExtractorConfig.readTimeOutSeconds</name>
+ <value>60</value>
</property>
<property>
- <name>dataSourceConfig.zkRetryTimes</name>
- <value>3</value>
+ <name>zkStateConfig.zkQuorum</name>
+ <value>sandbox.hortonworks.com:2181</value>
</property>
<property>
- <name>dataSourceConfig.zkRetryInterval</name>
- <value>20000</value>
+ <name>zkStateConfig.zkPort</name>
+ <value>2181</value>
</property>
<property>
- <name>dataSourceConfig.zkRoot</name>
- <value>/test_mrjobhistory</value>
+ <name>zkStateConfig.zkRoot</name>
+ <value>/mrjobhistory_sandbox</value>
</property>
<property>
- <name>dataSourceConfig.basePath</name>
- <value>/mr-history/done</value>
+ <name>zkStateConfig.zkSessionTimeoutMs</name>
+ <value>15000</value>
</property>
<property>
- <name>dataSourceConfig.jobTrackerName</name>
- <value></value>
+ <name>zkStateConfig.zkRetryTimes</name>
+ <value>3</value>
</property>
<property>
- <name>dataSourceConfig.nnEndpoint</name>
- <value>hdfs://sandbox.hortonworks.com:8020</value>
+ <name>zkStateConfig.zkRetryInterval</name>
+ <value>20000</value>
</property>
<property>
- <name>dataSourceConfig.pathContainsJobTrackerName</name>
+ <name>controlConfig.zeroBasedMonth</name>
<value>false</value>
</property>
<property>
- <name>dataSourceConfig.principal</name>
- <value></value>
+ <name>controlConfig.partitionerCls</name>
+ <value>org.apache.eagle.jpm.util.DefaultJobIdPartitioner</value>
</property>
<property>
- <name>dataSourceConfig.keytab</name>
- <value></value>
+ <name>controlConfig.timeZone</name>
+ <value>Etc/GMT+7</value>
</property>
<property>
- <name>dataSourceConfig.dryRun</name>
- <value>false</value>
+ <name>endpointConfig.mrHistoryServerUrl</name>
+ <value>http://sandbox.hortonworks.com:19888</value>
</property>
<property>
- <name>dataSourceConfig.partitionerCls</name>
- <value>org.apache.eagle.jpm.util.DefaultJobIdPartitioner</value>
+ <name>endpointConfig.basePath</name>
+ <value>/mr-history/done</value>
</property>
<property>
- <name>dataSourceConfig.zeroBasedMonth</name>
- <value>false</value>
+ <name>endpointConfig.hdfs.fs.defaultFS</name>
+ <value>hdfs://sandbox.hortonworks.com:8020</value>
</property>
<property>
<name>MRConfigureKeys.jobConfigKey</name>
@@ -115,14 +115,6 @@
<value>eagle.job.name</value>
</property>
<property>
- <name>envContextConfig.parallelismConfig.mrHistoryJobExecutor</name>
- <value>6</value>
- </property>
- <property>
- <name>envContextConfig.tasks.mrHistoryJobExecutor</name>
- <value>6</value>
- </property>
- <property>
<name>eagleProps.eagleService.host</name>
<description>eagleProps.eagleService.host</description>
<value>sandbox.hortonworks.com</value>
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fa85dc3e/eagle-jpm/eagle-jpm-mr-history/src/main/resources/application.conf
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/resources/application.conf b/eagle-jpm/eagle-jpm-mr-history/src/main/resources/application.conf
index 1440227..fa53dfb 100644
--- a/eagle-jpm/eagle-jpm-mr-history/src/main/resources/application.conf
+++ b/eagle-jpm/eagle-jpm-mr-history/src/main/resources/application.conf
@@ -40,7 +40,6 @@
"controlConfig" : {
"zeroBasedMonth" : false,
- "dryRun" : false,
"partitionerCls" : "org.apache.eagle.jpm.util.DefaultJobIdPartitioner",
"timeZone" : "UTC"
},
@@ -48,8 +47,6 @@
"endpointConfig" : {
"mrHistoryServerUrl" : "http://sandbox.hortonworks.com:19888",
"basePath" : "/mr-history/done",
- "pathContainsJobTrackerName" : false,
- "jobTrackerName" : "",
"hdfs" : {
fs.defaultFS : "hdfs://sandbox.hortonworks.com:8020",
#if not need, then do not set
@@ -60,8 +57,6 @@
},
"eagleProps" : {
- "mailHost" : "abc.com",
- "mailDebug" : "true",
"eagleService": {
"host": "sandbox.hortonworks.com",
"port": 9099,
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fa85dc3e/eagle-jpm/eagle-jpm-mr-running/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.running.MRRunningJobApplicationProvider.xml
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-mr-running/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.running.MRRunningJobApplicationProvider.xml b/eagle-jpm/eagle-jpm-mr-running/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.running.MRRunningJobApplicationProvider.xml
index 51e9eb8..4b95b36 100644
--- a/eagle-jpm/eagle-jpm-mr-running/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.running.MRRunningJobApplicationProvider.xml
+++ b/eagle-jpm/eagle-jpm-mr-running/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.running.MRRunningJobApplicationProvider.xml
@@ -23,10 +23,30 @@
<configuration>
<!-- org.apache.eagle.jpm.spark.running.SparkRunningJobAppConfig -->
<property>
- <name>envContextConfig.env</name>
- <value>local</value>
- <displayName>Environment</displayName>
- <description>Execution environment</description>
+ <name>jobExtractorConfig.site</name>
+ <displayName>Site ID</displayName>
+ <value>sandbox</value>
+ </property>
+ <property>
+ <name>workers</name>
+ <displayName>storm worker number</displayName>
+ <value>4</value>
+ </property>
+ <property>
+ <name>envContextConfig.parallelismConfig.mrRunningJobFetchSpout</name>
+ <value>1</value>
+ </property>
+ <property>
+ <name>envContextConfig.parallelismConfig.mrRunningJobParseBolt</name>
+ <value>8</value>
+ </property>
+ <property>
+ <name>envContextConfig.tasks.mrRunningJobFetchSpout</name>
+ <value>1</value>
+ </property>
+ <property>
+ <name>envContextConfig.tasks.mrRunningJobParseBolt</name>
+ <value>8</value>
</property>
<property>
<name>zookeeperConfig.zkQuorum</name>
@@ -60,7 +80,7 @@
</property>
<property>
<name>zookeeperConfig.zkRoot</name>
- <value>/apps/mr/running</value>
+ <value>/apps/mr/runningSandbox</value>
</property>
<property>
<name>eagleProps.eagleService.host</name>
@@ -85,26 +105,21 @@
<property>
<name>eagleProps.eagleService.readTimeOutSeconds</name>
<description>eagleProps.eagleService.readTimeOutSeconds</description>
- <value>20</value>
+ <value>60</value>
</property>
<property>
<name>eagleProps.eagleService.maxFlushNum</name>
<description>eagleProps.eagleService.maxFlushNum</description>
- <value>500</value>
- </property>
- <property>
- <name>jobExtractorConfig.site</name>
- <description>jobExtractorConfig.site</description>
- <value>sandbox</value>
+ <value>1000</value>
</property>
<property>
<name>jobExtractorConfig.fetchRunningJobInterval</name>
<description>jobExtractorConfig.fetchRunningJobInterval</description>
- <value>15</value>
+ <value>60</value>
</property>
<property>
- <name>jobExtractorConfig.parseThreadPoolSize</name>
- <description>jobExtractorConfig.parseThreadPoolSize</description>
+ <name>jobExtractorConfig.parseJobThreadPoolSize</name>
+ <description>jobExtractorConfig.parseJobThreadPoolSize</description>
<value>5</value>
</property>
<property>
@@ -115,27 +130,25 @@
<property>
<name>dataSourceConfig.rmUrls</name>
<description>dataSourceConfig.rmUrls</description>
- <value>http://sandbox.hortonworks.com:50030</value>
- </property>
- <property>
- <name>dataSourceConfig.nnEndpoint</name>
- <description>dataSourceConfig.nnEndpoint</description>
- <value>hdfs://sandbox.hortonworks.com:8020</value>
- </property>
- <property>
- <name>dataSourceConfig.keytab</name>
- <description>dataSourceConfig.keytab</description>
- <value></value>
+ <value>http://sandbox.hortonworks.com:8088</value>
</property>
<property>
- <name>dataSourceConfig.principal</name>
- <description>dataSourceConfig.principal</description>
- <value></value>
+ <name>MRConfigureKeys.jobConfigKey</name>
+ <value>mapreduce.map.output.compress,
+ mapreduce.map.output.compress.codec,
+ mapreduce.output.fileoutputformat.compress,
+ mapreduce.output.fileoutputformat.compress.type,
+ mapreduce.output.fileoutputformat.compress.codec,
+ mapred.output.format.class,
+ dataplatform.etl.info,
+ mapreduce.map.memory.mb,
+ mapreduce.reduce.memory.mb,
+ mapreduce.map.java.opts,
+ mapreduce.reduce.java.opts</value>
</property>
<property>
- <name>dataSourceConfig.rmUrls</name>
- <description>dataSourceConfig.rmUrls</description>
- <value>http://sandbox.hortonworks.com:8088</value>
+ <name>MRConfigureKeys.jobNameKey</name>
+ <value>eagle.job.name</value>
</property>
</configuration>
<docs>
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fa85dc3e/eagle-jpm/eagle-jpm-mr-running/src/main/resources/application.conf
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-mr-running/src/main/resources/application.conf b/eagle-jpm/eagle-jpm-mr-running/src/main/resources/application.conf
index 830c72b..9c354c7 100644
--- a/eagle-jpm/eagle-jpm-mr-running/src/main/resources/application.conf
+++ b/eagle-jpm/eagle-jpm-mr-running/src/main/resources/application.conf
@@ -51,8 +51,6 @@
},
"eagleProps" : {
- "mailHost" : "abc.com",
- "mailDebug" : "true",
"eagleService": {
"host": "sandbox.hortonworks.com",
"port": 9099,
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fa85dc3e/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/jobrecover/RunningJobManager.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/jobrecover/RunningJobManager.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/jobrecover/RunningJobManager.java
index 253c61a..a2d97bf 100644
--- a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/jobrecover/RunningJobManager.java
+++ b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/jobrecover/RunningJobManager.java
@@ -159,8 +159,8 @@ public class RunningJobManager implements Serializable {
appInfo.put("startedTime", app.getStartedTime() + "");
appInfo.put("finishedTime", app.getFinishedTime() + "");
appInfo.put("elapsedTime", app.getElapsedTime() + "");
- appInfo.put("amContainerLogs", app.getAmContainerLogs());
- appInfo.put("amHostHttpAddress", app.getAmHostHttpAddress());
+ appInfo.put("amContainerLogs", app.getAmContainerLogs() == null ? "" : app.getAmContainerLogs());
+ appInfo.put("amHostHttpAddress", app.getAmHostHttpAddress() == null ? "" : app.getAmHostHttpAddress());
appInfo.put("allocatedMB", app.getAllocatedMB() + "");
appInfo.put("allocatedVCores", app.getAllocatedVCores() + "");
appInfo.put("runningContainers", app.getRunningContainers() + "");
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fa85dc3e/eagle-topology-assembly/pom.xml
----------------------------------------------------------------------
diff --git a/eagle-topology-assembly/pom.xml b/eagle-topology-assembly/pom.xml
index 6a87a98..80ca265 100644
--- a/eagle-topology-assembly/pom.xml
+++ b/eagle-topology-assembly/pom.xml
@@ -177,4 +177,4 @@
</plugin>
</plugins>
</build>
-</project>
+</project>
\ No newline at end of file