You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@eagle.apache.org by ha...@apache.org on 2015/11/19 11:47:48 UTC
[41/55] [abbrv] [partial] incubator-eagle git commit: [EAGLE-46]
Rename package name as "org.apache.eagle"
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/afe86834/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/storm/JobRunningSpout.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/storm/JobRunningSpout.java b/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/storm/JobRunningSpout.java
deleted file mode 100644
index 670a2e9..0000000
--- a/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/storm/JobRunningSpout.java
+++ /dev/null
@@ -1,200 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package eagle.jobrunning.storm;
-
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
-import eagle.jobrunning.callback.DefaultRunningJobInputStreamCallback;
-import eagle.jobrunning.callback.RunningJobMessageId;
-import eagle.jobrunning.config.RunningJobCrawlConfig;
-import eagle.jobrunning.crawler.RunningJobCrawler;
-import eagle.jobrunning.crawler.RunningJobCrawlerImpl;
-import eagle.jobrunning.zkres.JobRunningZKStateManager;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import backtype.storm.spout.SpoutOutputCollector;
-import backtype.storm.task.TopologyContext;
-import backtype.storm.topology.OutputFieldsDeclarer;
-import backtype.storm.topology.base.BaseRichSpout;
-
-import eagle.job.JobFilter;
-import eagle.job.JobFilterByPartition;
-import eagle.job.JobPartitioner;
-import eagle.jobrunning.callback.RunningJobCallback;
-import eagle.jobrunning.common.JobConstants.ResourceType;
-import eagle.jobrunning.crawler.JobContext;
-
-public class JobRunningSpout extends BaseRichSpout {
- private static final long serialVersionUID = 1L;
- private static final Logger LOG = LoggerFactory.getLogger(JobRunningSpout.class);
-
- private RunningJobCrawlConfig config;
- private JobRunningZKStateManager zkStateManager;
- private transient RunningJobCrawler crawler;
- private JobRunningSpoutCollectorInterceptor interceptor;
- private RunningJobCallback callback;
- private ReadWriteLock readWriteLock;
- private static final int DEFAULT_WAIT_SECONDS_BETWEEN_ROUNDS = 10;
-
- public JobRunningSpout(RunningJobCrawlConfig config){
- this(config, new JobRunningSpoutCollectorInterceptor());
- }
-
- /**
- * mostly this constructor signature is for unit test purpose as you can put customized interceptor here
- * @param config
- * @param interceptor
- */
- public JobRunningSpout(RunningJobCrawlConfig config, JobRunningSpoutCollectorInterceptor interceptor){
- this.config = config;
- this.interceptor = interceptor;
- this.callback = new DefaultRunningJobInputStreamCallback(interceptor);
- this.readWriteLock = new ReentrantReadWriteLock();
- }
-
-
- /**
- * TODO: just copy this part from jobHistorySpout, need to move it to a common place
- * @param context
- * @return
- */
- private int calculatePartitionId(TopologyContext context){
- int thisGlobalTaskId = context.getThisTaskId();
- String componentName = context.getComponentId(thisGlobalTaskId);
- List<Integer> globalTaskIds = context.getComponentTasks(componentName);
- int index = 0;
- for(Integer id : globalTaskIds){
- if(id == thisGlobalTaskId){
- return index;
- }
- index++;
- }
- throw new IllegalStateException();
- }
-
- @SuppressWarnings("rawtypes")
- @Override
- public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
- int partitionId = calculatePartitionId(context);
- // sanity verify 0<=partitionId<=numTotalPartitions-1
- if(partitionId < 0 || partitionId > config.controlConfig.numTotalPartitions){
- throw new IllegalStateException("partitionId should be less than numTotalPartitions with partitionId " +
- partitionId + " and numTotalPartitions " + config.controlConfig.numTotalPartitions);
- }
- Class<? extends JobPartitioner> partitionerCls = config.controlConfig.partitionerCls;
- JobPartitioner partitioner = null;
- try {
- partitioner = partitionerCls.newInstance();
- } catch (Exception e) {
- LOG.error("failing instantiating job partitioner class " + partitionerCls.getCanonicalName());
- throw new IllegalStateException(e);
- }
- JobFilter jobFilter = new JobFilterByPartition(partitioner, config.controlConfig.numTotalPartitions, partitionId);
- interceptor.setSpoutOutputCollector(collector);
- try {
- zkStateManager = new JobRunningZKStateManager(config);
- crawler = new RunningJobCrawlerImpl(config, zkStateManager, callback, jobFilter, readWriteLock);
- } catch (Exception e) {
- LOG.error("failing creating crawler driver");
- throw new IllegalStateException(e);
- }
- }
-
- @Override
- public void nextTuple() {
- try{
- crawler.crawl();
- }catch(Exception ex){
- LOG.error("fail crawling running job and continue ...", ex);
- }
- try{
- Thread.sleep(DEFAULT_WAIT_SECONDS_BETWEEN_ROUNDS *1000);
- }catch(Exception x){
- }
- }
-
- /**
- * empty because framework will take care of output fields declaration
- */
- @Override
- public void declareOutputFields(OutputFieldsDeclarer declarer) {
-
- }
-
- /**
- * add to processedJob
- */
- @Override
- public void ack(Object msgId) {
- RunningJobMessageId messageId = (RunningJobMessageId) msgId;
- ResourceType type = messageId.type;
- LOG.info("Ack on messageId: " + messageId.toString());
- switch(type) {
- case JOB_CONFIGURATION:
- case JOB_COMPLETE_INFO:
- /** lock this for making processed/processing job list unchanged during crawler calculating last round running job list **/
- try {
- readWriteLock.readLock().lock();
- zkStateManager.addProcessedJob(type, messageId.jobID);
- // Here username & timestamp is meaningless, set to null
- crawler.removeFromProcessingList(type, new JobContext(messageId.jobID, null, null));
- }
- finally {
- try {readWriteLock.readLock().unlock(); LOG.info("Read lock released");}
- catch (Throwable t) { LOG.error("Fail to release Read lock", t);}
- }
- break;
- default:
- break;
- }
- }
-
- /**
- * job is not fully processed
- */
- @Override
- public void fail(Object msgId) {
- RunningJobMessageId messageId = (RunningJobMessageId) msgId;
- ResourceType type = messageId.type;
- // Here timestamp is meaningless, set to null
- if (type.equals(ResourceType.JOB_COMPLETE_INFO) || type.equals(ResourceType.JOB_CONFIGURATION)) {
- try {
- readWriteLock.readLock().lock();
- // Here username in not used, set to null
- crawler.removeFromProcessingList(type, new JobContext(messageId.jobID, null, null));
- }
- finally {
- try {readWriteLock.readLock().unlock(); LOG.info("Read lock released");}
- catch (Throwable t) { LOG.error("Fail to release Read lock", t);}
- }
- }
- }
-
- @Override
- public void deactivate() {
-
- }
-
- @Override
- public void close() {
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/afe86834/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/storm/JobRunningSpoutCollectorInterceptor.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/storm/JobRunningSpoutCollectorInterceptor.java b/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/storm/JobRunningSpoutCollectorInterceptor.java
deleted file mode 100644
index 3f87a38..0000000
--- a/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/storm/JobRunningSpoutCollectorInterceptor.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package eagle.jobrunning.storm;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import backtype.storm.spout.SpoutOutputCollector;
-
-import eagle.dataproc.core.EagleOutputCollector;
-import eagle.dataproc.core.ValuesArray;
-import eagle.jobrunning.callback.RunningJobMessageId;
-
-public class JobRunningSpoutCollectorInterceptor implements EagleOutputCollector{
-
- private static final long serialVersionUID = 1L;
- private SpoutOutputCollector collector;
-
- public void setSpoutOutputCollector(SpoutOutputCollector collector){
- this.collector = collector;
- }
-
- @Override
- public void collect(ValuesArray t) {
- // the first value is fixed as messageId
- RunningJobMessageId messageId = (RunningJobMessageId) t.get(0);
- List<Object> list = new ArrayList<Object>();
- for (int i = 1; i < t.size(); i++) {
- list.add(t.get(i));
- }
- collector.emit(list, messageId);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/afe86834/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/url/JobCompleteCounterServiceURLBuilderImpl.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/url/JobCompleteCounterServiceURLBuilderImpl.java b/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/url/JobCompleteCounterServiceURLBuilderImpl.java
deleted file mode 100644
index 3c926ca..0000000
--- a/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/url/JobCompleteCounterServiceURLBuilderImpl.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package eagle.jobrunning.url;
-
-import eagle.jobrunning.common.JobConstants;
-
-public class JobCompleteCounterServiceURLBuilderImpl implements ServiceURLBuilder {
-
- public String build(String ... parameters) {
- // parameters[0] = historyBaseUrl, parameters[1] = jobID
- // {historyUrl}/jobhistory/jobcounters/job_xxxxxxxxxxxxx_xxxxx?anonymous=true
- return parameters[0] + "jobhistory/jobcounters/" + parameters[1]
- + "?" + JobConstants.ANONYMOUS_PARAMETER;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/afe86834/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/url/JobCompleteDetailServiceURLBuilderImpl.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/url/JobCompleteDetailServiceURLBuilderImpl.java b/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/url/JobCompleteDetailServiceURLBuilderImpl.java
deleted file mode 100644
index ded824a..0000000
--- a/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/url/JobCompleteDetailServiceURLBuilderImpl.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package eagle.jobrunning.url;
-
-import eagle.jobrunning.common.JobConstants;
-import eagle.jobrunning.util.JobUtils;
-
-public class JobCompleteDetailServiceURLBuilderImpl implements ServiceURLBuilder {
-
- public String build(String ... parameters) {
- // parameters[0] = baseUrl, parameters[1] = jobID
- // {baseUrl}/ws/v1/cluster/apps/job_xxxxxxxxxxxxx_xxxxx?anonymous=true
- return parameters[0] + JobConstants.V2_COMPLETE_APPS_URL
- + JobUtils.getAppIDByJobID(parameters[1])
- + "?" + JobConstants.ANONYMOUS_PARAMETER;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/afe86834/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/url/JobCompletedConfigServiceURLBuilderImpl.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/url/JobCompletedConfigServiceURLBuilderImpl.java b/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/url/JobCompletedConfigServiceURLBuilderImpl.java
deleted file mode 100644
index 81ba4fc..0000000
--- a/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/url/JobCompletedConfigServiceURLBuilderImpl.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package eagle.jobrunning.url;
-
-
-public class JobCompletedConfigServiceURLBuilderImpl implements ServiceURLBuilder {
-
- public String build(String ... parameters) {
- // parameters[0] = baseUrl, parameters[1] = jobID
- // {historyUrl}/jobhistory/conf/job_xxxxxxxxxxxxx_xxxxxx
- return parameters[0] + "jobhistory/conf/" + parameters[1];
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/afe86834/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/url/JobCountersServiceURLBuilderImpl.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/url/JobCountersServiceURLBuilderImpl.java b/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/url/JobCountersServiceURLBuilderImpl.java
deleted file mode 100644
index 5f84016..0000000
--- a/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/url/JobCountersServiceURLBuilderImpl.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package eagle.jobrunning.url;
-
-import eagle.jobrunning.common.JobConstants;
-import eagle.jobrunning.util.JobUtils;
-
-public class JobCountersServiceURLBuilderImpl implements ServiceURLBuilder {
-
- public String build(String ... parameters) {
- // parameter[0] = baseUrl, parameter[1] = appID;
- // {rmUrl}/proxy/application_xxxxxxxxxxxxx_xxxxxx/ws/v1/mapreduce/jobs/job_xxxxxxxxxxxxx_xxxxxx/counters?anonymous=true"
- return parameters[0] + JobConstants.V2_PROXY_PREFIX_URL +
- parameters[1] + JobConstants.V2_MR_APPMASTER_PREFIX +
- JobUtils.getJobIDByAppID(parameters[1]) + JobConstants.V2_MR_COUNTERS_URL +
- "?" + JobConstants.ANONYMOUS_PARAMETER;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/afe86834/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/url/JobDetailServiceURLBuilderImpl.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/url/JobDetailServiceURLBuilderImpl.java b/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/url/JobDetailServiceURLBuilderImpl.java
deleted file mode 100644
index 608b667..0000000
--- a/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/url/JobDetailServiceURLBuilderImpl.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package eagle.jobrunning.url;
-
-import eagle.jobrunning.common.JobConstants;
-
-public class JobDetailServiceURLBuilderImpl implements ServiceURLBuilder {
-
- public String build(String... parameters) {
- // parameter[0] = baseUrl , parameter[1] = appID
- // {rmUrl}/proxy/application_xxx/ws/v1/mapreduce/jobs?anonymous=true
- return parameters[0] + JobConstants.V2_PROXY_PREFIX_URL + parameters[1] + JobConstants.V2_APP_DETAIL_URL + "?" + JobConstants.ANONYMOUS_PARAMETER;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/afe86834/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/url/JobListServiceURLBuilderImpl.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/url/JobListServiceURLBuilderImpl.java b/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/url/JobListServiceURLBuilderImpl.java
deleted file mode 100644
index 09fec32..0000000
--- a/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/url/JobListServiceURLBuilderImpl.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package eagle.jobrunning.url;
-
-import eagle.jobrunning.common.JobConstants;
-import eagle.jobrunning.common.JobConstants.JobState;
-
-public class JobListServiceURLBuilderImpl implements ServiceURLBuilder {
-
- public String build(String ... parameters) {
- // {rmUrl}/ws/v1/cluster/apps?state=RUNNING
- String jobState = parameters[1];
- if (jobState.equals(JobState.RUNNING.name())) {
- return parameters[0] + JobConstants.V2_APPS_RUNNING_URL + "&" + JobConstants.ANONYMOUS_PARAMETER;
- }
- else if (jobState.equals(JobState.COMPLETED.name())) {
- return parameters[0] + JobConstants.V2_APPS_COMPLETED_URL + "&" + JobConstants.ANONYMOUS_PARAMETER;
- }
- else if (jobState.equals(JobState.ALL.name())) {
- return parameters[0] + JobConstants.V2_APPS_URL + "&" + JobConstants.ANONYMOUS_PARAMETER;
- }
- return null;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/afe86834/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/url/JobRunningConfigServiceURLBuilderImpl.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/url/JobRunningConfigServiceURLBuilderImpl.java b/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/url/JobRunningConfigServiceURLBuilderImpl.java
deleted file mode 100644
index 10997be..0000000
--- a/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/url/JobRunningConfigServiceURLBuilderImpl.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package eagle.jobrunning.url;
-
-import eagle.jobrunning.common.JobConstants;
-import eagle.jobrunning.util.JobUtils;
-
-public class JobRunningConfigServiceURLBuilderImpl implements ServiceURLBuilder {
-
- public String build(String ... parameters) {
- // parameters[0] = baseUrl, parameters[1] = jobID
- // {baseUrl}/proxy/application_xxxxxxxxxxxxx_xxxxx/ws/v1/mapreduce/jobs/job_xxxxxxxxxxxxx_xxxxx/conf
- String urlString = parameters[0] + JobConstants.V2_PROXY_PREFIX_URL
- + JobUtils.getAppIDByJobID(parameters[1]) + JobConstants.V2_MR_APPMASTER_PREFIX
- + parameters[1] + JobConstants.V2_CONF_URL
- + "?" + JobConstants.ANONYMOUS_PARAMETER;
- return urlString;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/afe86834/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/url/JobStatusServiceURLBuilderImpl.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/url/JobStatusServiceURLBuilderImpl.java b/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/url/JobStatusServiceURLBuilderImpl.java
deleted file mode 100644
index 08aa787..0000000
--- a/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/url/JobStatusServiceURLBuilderImpl.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package eagle.jobrunning.url;
-
-import eagle.jobrunning.common.JobConstants;
-
-public class JobStatusServiceURLBuilderImpl implements ServiceURLBuilder {
-
- public String build(String ... parameters) {
- // parameters[0] = rmUrl, parameters[1] = appID
- // {rmUrl}/ws/v1/cluster/apps/application_xxxxxxxxxxxxx_xxxxx?anonymous=true
- return parameters[0] + "ws/v1/cluster/apps/" + parameters[1]
- + "?" + JobConstants.ANONYMOUS_PARAMETER;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/afe86834/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/url/ServiceURLBuilder.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/url/ServiceURLBuilder.java b/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/url/ServiceURLBuilder.java
deleted file mode 100644
index 2e532e8..0000000
--- a/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/url/ServiceURLBuilder.java
+++ /dev/null
@@ -1,21 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package eagle.jobrunning.url;
-
-public interface ServiceURLBuilder {
- String build(String ... parameters);
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/afe86834/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/util/InputStreamUtils.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/util/InputStreamUtils.java b/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/util/InputStreamUtils.java
deleted file mode 100644
index 448f491..0000000
--- a/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/util/InputStreamUtils.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package eagle.jobrunning.util;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.URL;
-import java.net.URLConnection;
-import java.util.zip.GZIPInputStream;
-
-import eagle.jobrunning.common.JobConstants.CompressionType;
-
-public class InputStreamUtils {
-
- private static final int CONNECTION_TIMEOUT = 10 * 1000;
- private static final int READ_TIMEOUT = 5 * 60 * 1000;
- private static final String GZIP_HTTP_HEADER = "Accept-Encoding";
- private static final String GZIP_COMPRESSION = "gzip";
-
- private static InputStream openGZIPInputStream(URL url, int timeout) throws IOException {
- final URLConnection connection = url.openConnection();
- connection.setConnectTimeout(CONNECTION_TIMEOUT);
- connection.setReadTimeout(timeout);
- connection.addRequestProperty(GZIP_HTTP_HEADER, GZIP_COMPRESSION);
- return new GZIPInputStream(connection.getInputStream());
- }
-
- private static InputStream openInputStream(URL url, int timeout) throws IOException {
- URLConnection connection = url.openConnection();
- connection.setConnectTimeout(timeout);
- return connection.getInputStream();
- }
-
- public static InputStream getInputStream(String urlString, CompressionType compressionType, int timeout) throws Exception {
- final URL url = URLConnectionUtils.getUrl(urlString);
- if (compressionType.equals(CompressionType.GZIP)) {
- return openGZIPInputStream(url, timeout);
- }
- else { // CompressionType.NONE
- return openInputStream(url, timeout);
- }
- }
-
- public static InputStream getInputStream(String urlString, CompressionType compressionType) throws Exception {
- return getInputStream(urlString, compressionType, READ_TIMEOUT);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/afe86834/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/util/JobUtils.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/util/JobUtils.java b/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/util/JobUtils.java
deleted file mode 100644
index b647287..0000000
--- a/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/util/JobUtils.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package eagle.jobrunning.util;
-
-import eagle.jobrunning.common.JobConstants;
-
-public class JobUtils {
-
- public static String checkAndAddLastSlash(String urlBase) {
- if (!urlBase.endsWith("/")) {
- return urlBase + "/";
- }
- return urlBase;
- }
-
- public static String getJobIDByAppID(String appID) {
- if (appID.startsWith(JobConstants.APPLICATION_PREFIX)) {
- return appID.replace(JobConstants.APPLICATION_PREFIX, JobConstants.JOB_PREFIX);
- }
- return null;
- }
-
- public static String getAppIDByJobID(String jobID) {
- if (jobID.startsWith(JobConstants.JOB_PREFIX)) {
- return jobID.replace(JobConstants.JOB_PREFIX, JobConstants.APPLICATION_PREFIX);
- }
- return null;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/afe86834/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/util/URLConnectionUtils.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/util/URLConnectionUtils.java b/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/util/URLConnectionUtils.java
deleted file mode 100644
index 0069911..0000000
--- a/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/util/URLConnectionUtils.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package eagle.jobrunning.util;
-
-import java.io.IOException;
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.net.URLConnection;
-import java.security.KeyManagementException;
-import java.security.NoSuchAlgorithmException;
-import java.security.cert.CertificateException;
-
-import javax.net.ssl.HostnameVerifier;
-import javax.net.ssl.HttpsURLConnection;
-import javax.net.ssl.SSLContext;
-import javax.net.ssl.SSLSession;
-import javax.net.ssl.TrustManager;
-import javax.net.ssl.X509TrustManager;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public final class URLConnectionUtils {
- //TODO: change some public method to private
- private static final Logger LOG = LoggerFactory.getLogger(URLConnectionUtils.class);
-
- public static URLConnection getConnection(String url) throws Exception {
- if (url.startsWith("https://")) {
- return getHTTPSConnection(url);
- } else if (url.startsWith("http://")) {
- return getHTTPConnection(url);
- }
- throw new Exception("Invalid input argument url: " + url);
- }
-
- public static URLConnection getHTTPConnection(String urlString) throws Exception {
- final URL url = new URL(urlString);
- return url.openConnection();
- }
-
- public static URL getUrl(String urlString) throws Exception {
- if(urlString.toLowerCase().contains("https")){
- return getHTTPSUrl(urlString);
- }else if (urlString.toLowerCase().contains("http")) {
- return getURL(urlString);
- }
- throw new Exception("Invalid input argument url: " + urlString);
- }
-
- public static URL getURL(String urlString) throws MalformedURLException {
- return new URL(urlString);
- }
-
- public static URL getHTTPSUrl(String urlString) throws MalformedURLException, NoSuchAlgorithmException, KeyManagementException {
- // Create a trust manager that does not validate certificate chains
- final TrustManager[] trustAllCerts = new TrustManager[] {new TrustAllX509TrustManager()};
- // Install the all-trusting trust manager
- final SSLContext sc = SSLContext.getInstance("SSL");
- sc.init(null, trustAllCerts, new java.security.SecureRandom());
- HttpsURLConnection.setDefaultSSLSocketFactory(sc.getSocketFactory());
- // Create all-trusting host name verifier
- final HostnameVerifier allHostsValid = new HostnameVerifier() {
- public boolean verify(String hostname, SSLSession session) {
- return true;
- }
- };
- HttpsURLConnection.setDefaultHostnameVerifier(allHostsValid);
- return new URL(urlString);
- }
-
- public static URLConnection getHTTPSConnection(String urlString) throws IOException, KeyManagementException, NoSuchAlgorithmException {
- final URL url = getHTTPSUrl(urlString);
- return url.openConnection();
- }
-
- public static class TrustAllX509TrustManager implements X509TrustManager {
- @Override
- public void checkClientTrusted(
- java.security.cert.X509Certificate[] chain, String authType)
- throws CertificateException {
- }
-
- @Override
- public void checkServerTrusted(
- java.security.cert.X509Certificate[] chain, String authType)
- throws CertificateException {
- }
-
- @Override
- public java.security.cert.X509Certificate[] getAcceptedIssuers() {
- return null;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/afe86834/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/yarn/model/App.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/yarn/model/App.java b/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/yarn/model/App.java
deleted file mode 100644
index ad8af9c..0000000
--- a/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/yarn/model/App.java
+++ /dev/null
@@ -1,145 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package eagle.jobrunning.yarn.model;
-
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL)
-@JsonIgnoreProperties(ignoreUnknown = true)
-public class App {
- private String id;
- private String user;
- private String name;
- private String queue;
- private String state;
- private String finalStatus;
- private double progress;
- private String trackingUI;
- private String trackingUrl;
- private String diagnostics;
- private String clusterId;
- private String applicationType;
- private long startedTime;
- private long finishedTime;
- private long elapsedTime;
- private String amContainerLogs;
- private String amHostHttpAddress;
-
- public String getId() {
- return id;
- }
- public void setId(String id) {
- this.id = id;
- }
- public String getUser() {
- return user;
- }
- public void setUser(String user) {
- this.user = user;
- }
- public String getName() {
- return name;
- }
- public void setName(String name) {
- this.name = name;
- }
- public String getQueue() {
- return queue;
- }
- public void setQueue(String queue) {
- this.queue = queue;
- }
- public String getState() {
- return state;
- }
- public void setState(String state) {
- this.state = state;
- }
- public String getFinalStatus() {
- return finalStatus;
- }
- public void setFinalStatus(String finalStatus) {
- this.finalStatus = finalStatus;
- }
- public double getProgress() {
- return progress;
- }
- public void setProgress(double progress) {
- this.progress = progress;
- }
- public String getTrackingUI() {
- return trackingUI;
- }
- public void setTrackingUI(String trackingUI) {
- this.trackingUI = trackingUI;
- }
- public String getTrackingUrl() {
- return trackingUrl;
- }
- public void setTrackingUrl(String trackingUrl) {
- this.trackingUrl = trackingUrl;
- }
- public String getDiagnostics() {
- return diagnostics;
- }
- public void setDiagnostics(String diagnostics) {
- this.diagnostics = diagnostics;
- }
- public String getClusterId() {
- return clusterId;
- }
- public void setClusterId(String clusterId) {
- this.clusterId = clusterId;
- }
- public String getApplicationType() {
- return applicationType;
- }
- public void setApplicationType(String applicationType) {
- this.applicationType = applicationType;
- }
- public long getStartedTime() {
- return startedTime;
- }
- public void setStartedTime(long startedTime) {
- this.startedTime = startedTime;
- }
- public long getFinishedTime() {
- return finishedTime;
- }
- public void setFinishedTime(long finishedTime) {
- this.finishedTime = finishedTime;
- }
- public long getElapsedTime() {
- return elapsedTime;
- }
- public void setElapsedTime(long elapsedTime) {
- this.elapsedTime = elapsedTime;
- }
- public String getAmContainerLogs() {
- return amContainerLogs;
- }
- public void setAmContainerLogs(String amContainerLogs) {
- this.amContainerLogs = amContainerLogs;
- }
- public String getAmHostHttpAddress() {
- return amHostHttpAddress;
- }
- public void setAmHostHttpAddress(String amHostHttpAddress) {
- this.amHostHttpAddress = amHostHttpAddress;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/afe86834/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/yarn/model/AppInfo.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/yarn/model/AppInfo.java b/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/yarn/model/AppInfo.java
deleted file mode 100644
index ef0b6cb..0000000
--- a/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/yarn/model/AppInfo.java
+++ /dev/null
@@ -1,146 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package eagle.jobrunning.yarn.model;
-
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL)
-@JsonIgnoreProperties(ignoreUnknown = true)
-public class AppInfo {
- String id;
- String user;
- String name;
- String queue;
- String state;
- String finalStatus;
- double progress;
- String trackingUI;
- String trackingUrl;
- String diagnostics;
- String clusterId;
- String applicationType;
- long startedTime;
- long finishedTime;
- long elapsedTime;
- String amContainerLogs;
- String amHostHttpAddress;
-
- public String getId() {
- return id;
- }
- public void setId(String id) {
- this.id = id;
- }
- public String getUser() {
- return user;
- }
- public void setUser(String user) {
- this.user = user;
- }
- public String getName() {
- return name;
- }
- public void setName(String name) {
- this.name = name;
- }
- public String getQueue() {
- return queue;
- }
- public void setQueue(String queue) {
- this.queue = queue;
- }
- public String getState() {
- return state;
- }
- public void setState(String state) {
- this.state = state;
- }
- public String getFinalStatus() {
- return finalStatus;
- }
- public void setFinalStatus(String finalStatus) {
- this.finalStatus = finalStatus;
- }
- public double getProgress() {
- return progress;
- }
- public void setProgress(double progress) {
- this.progress = progress;
- }
- public String getTrackingUI() {
- return trackingUI;
- }
- public void setTrackingUI(String trackingUI) {
- this.trackingUI = trackingUI;
- }
- public String getTrackingUrl() {
- return trackingUrl;
- }
- public void setTrackingUrl(String trackingUrl) {
- this.trackingUrl = trackingUrl;
- }
- public String getDiagnostics() {
- return diagnostics;
- }
- public void setDiagnostics(String diagnostics) {
- this.diagnostics = diagnostics;
- }
- public String getClusterId() {
- return clusterId;
- }
- public void setClusterId(String clusterId) {
- this.clusterId = clusterId;
- }
- public String getApplicationType() {
- return applicationType;
- }
- public void setApplicationType(String applicationType) {
- this.applicationType = applicationType;
- }
- public long getStartedTime() {
- return startedTime;
- }
- public void setStartedTime(long startedTime) {
- this.startedTime = startedTime;
- }
- public long getFinishedTime() {
- return finishedTime;
- }
- public void setFinishedTime(long finishedTime) {
- this.finishedTime = finishedTime;
- }
- public long getElapsedTime() {
- return elapsedTime;
- }
- public void setElapsedTime(long elapsedTime) {
- this.elapsedTime = elapsedTime;
- }
- public String getAmContainerLogs() {
- return amContainerLogs;
- }
- public void setAmContainerLogs(String amContainerLogs) {
- this.amContainerLogs = amContainerLogs;
- }
- public String getAmHostHttpAddress() {
- return amHostHttpAddress;
- }
- public void setAmHostHttpAddress(String amHostHttpAddress) {
- this.amHostHttpAddress = amHostHttpAddress;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/afe86834/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/yarn/model/AppWrapper.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/yarn/model/AppWrapper.java b/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/yarn/model/AppWrapper.java
deleted file mode 100644
index 898bb6f..0000000
--- a/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/yarn/model/AppWrapper.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package eagle.jobrunning.yarn.model;
-
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL)
-@JsonIgnoreProperties(ignoreUnknown = true)
-public class AppWrapper {
-
- private AppInfo app;
-
- public AppInfo getApp() {
- return app;
- }
-
- public void setApp(AppInfo app) {
- this.app = app;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/afe86834/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/yarn/model/Applications.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/yarn/model/Applications.java b/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/yarn/model/Applications.java
deleted file mode 100644
index ae97f64..0000000
--- a/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/yarn/model/Applications.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package eagle.jobrunning.yarn.model;
-
-import java.util.List;
-
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL)
-@JsonIgnoreProperties(ignoreUnknown = true)
-public class Applications {
-
- private List<AppInfo> app;
-
- public List<AppInfo> getApp() {
- return app;
- }
-
- public void setApp(List<AppInfo> app) {
- this.app = app;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/afe86834/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/yarn/model/AppsWrapper.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/yarn/model/AppsWrapper.java b/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/yarn/model/AppsWrapper.java
deleted file mode 100644
index 143797c..0000000
--- a/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/yarn/model/AppsWrapper.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package eagle.jobrunning.yarn.model;
-
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL)
-@JsonIgnoreProperties(ignoreUnknown = true)
-public class AppsWrapper {
-
- private Applications apps;
-
- public Applications getApps() {
- return apps;
- }
-
- public void setApps(Applications apps) {
- this.apps = apps;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/afe86834/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/yarn/model/Counter.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/yarn/model/Counter.java b/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/yarn/model/Counter.java
deleted file mode 100644
index 9c0bda3..0000000
--- a/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/yarn/model/Counter.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package eagle.jobrunning.yarn.model;
-
-public class Counter {
-
- private String name;
- private long totalCounterValue;
- private long mapCounterValue;
- private long reduceCounterValue;
-
- public String getName() {
- return name;
- }
- public void setName(String name) {
- this.name = name;
- }
- public long getTotalCounterValue() {
- return totalCounterValue;
- }
- public void setTotalCounterValue(long totalCounterValue) {
- this.totalCounterValue = totalCounterValue;
- }
- public long getMapCounterValue() {
- return mapCounterValue;
- }
- public void setMapCounterValue(long mapCounterValue) {
- this.mapCounterValue = mapCounterValue;
- }
- public long getReduceCounterValue() {
- return reduceCounterValue;
- }
- public void setReduceCounterValue(long reduceCounterValue) {
- this.reduceCounterValue = reduceCounterValue;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/afe86834/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/yarn/model/CounterGroup.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/yarn/model/CounterGroup.java b/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/yarn/model/CounterGroup.java
deleted file mode 100644
index 5d38315..0000000
--- a/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/yarn/model/CounterGroup.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package eagle.jobrunning.yarn.model;
-
-import java.util.List;
-
-public class CounterGroup {
-
- private String counterGroupName;
- private List<Counter> counter;
-
- public String getCounterGroupName() {
- return counterGroupName;
- }
- public void setCounterGroupName(String counterGroupName) {
- this.counterGroupName = counterGroupName;
- }
- public List<Counter> getCounter() {
- return counter;
- }
- public void setCounter(List<Counter> counter) {
- this.counter = counter;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/afe86834/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/yarn/model/JobCompleteWrapper.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/yarn/model/JobCompleteWrapper.java b/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/yarn/model/JobCompleteWrapper.java
deleted file mode 100644
index ea7f555..0000000
--- a/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/yarn/model/JobCompleteWrapper.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package eagle.jobrunning.yarn.model;
-
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL)
-@JsonIgnoreProperties(ignoreUnknown = true)
-public class JobCompleteWrapper {
- private App app;
-
- public App getApp() {
- return app;
- }
-
- public void setApp(App app) {
- this.app = app;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/afe86834/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/yarn/model/JobCounters.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/yarn/model/JobCounters.java b/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/yarn/model/JobCounters.java
deleted file mode 100644
index 00a1684..0000000
--- a/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/yarn/model/JobCounters.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package eagle.jobrunning.yarn.model;
-
-import java.util.List;
-
-public class JobCounters {
-
- private String id;
- private List<CounterGroup> counterGroup;
-
- public String getId() {
- return id;
- }
- public void setId(String id) {
- this.id = id;
- }
- public List<CounterGroup> getCounterGroup() {
- return counterGroup;
- }
- public void setCounterGroup(List<CounterGroup> counterGroup) {
- this.counterGroup = counterGroup;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/afe86834/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/yarn/model/JobCountersWrapper.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/yarn/model/JobCountersWrapper.java b/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/yarn/model/JobCountersWrapper.java
deleted file mode 100644
index 6f37fdd..0000000
--- a/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/yarn/model/JobCountersWrapper.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package eagle.jobrunning.yarn.model;
-
-
-public class JobCountersWrapper {
-
- private JobCounters jobCounters;
-
- public JobCounters getJobCounters() {
- return jobCounters;
- }
-
- public void setJobCounters(JobCounters jobCounters) {
- this.jobCounters = jobCounters;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/afe86834/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/yarn/model/JobDetailInfo.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/yarn/model/JobDetailInfo.java b/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/yarn/model/JobDetailInfo.java
deleted file mode 100644
index 593ad3f..0000000
--- a/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/yarn/model/JobDetailInfo.java
+++ /dev/null
@@ -1,241 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package eagle.jobrunning.yarn.model;
-
-public class JobDetailInfo {
-
- private long startTime;
- private long finishTime;
- private long elapsedTime;
- private String id;
- private String name;
- private String user;
- private String state;
- private int mapsTotal;
- private int mapsCompleted;
- private int reducesTotal;
- private int reducesCompleted;
- private double mapProgress;
- private double reduceProgress;
- private int mapsPending;
- private int mapsRunning;
- private int reducesPending;
- private int reducesRunning;
- private boolean uberized;
- private String diagnostics;
- private int newReduceAttempts;
- private int runningReduceAttempts;
- private int failedReduceAttempts;
- private int killedReduceAttempts;
- private int successfulReduceAttempts;
- private int newMapAttempts;
- private int runningMapAttempts;
- private int failedMapAttempts;
- private int killedMapAttempts;
- private int successfulMapAttempts;
- private String queue;
- private eagle.jobrunning.counter.JobCounters jobcounter;
-
- public long getStartTime() {
- return startTime;
- }
- public void setStartTime(long startTime) {
- this.startTime = startTime;
- }
- public long getFinishTime() {
- return finishTime;
- }
- public void setFinishTime(long finishTime) {
- this.finishTime = finishTime;
- }
- public long getElapsedTime() {
- return elapsedTime;
- }
- public void setElapsedTime(long elapsedTime) {
- this.elapsedTime = elapsedTime;
- }
- public String getId() {
- return id;
- }
- public void setId(String id) {
- this.id = id;
- }
- public String getName() {
- return name;
- }
- public void setName(String name) {
- this.name = name;
- }
- public String getUser() {
- return user;
- }
- public void setUser(String user) {
- this.user = user;
- }
- public String getState() {
- return state;
- }
- public void setState(String state) {
- this.state = state;
- }
- public int getMapsTotal() {
- return mapsTotal;
- }
- public void setMapsTotal(int mapsTotal) {
- this.mapsTotal = mapsTotal;
- }
- public int getMapsCompleted() {
- return mapsCompleted;
- }
- public void setMapsCompleted(int mapsCompleted) {
- this.mapsCompleted = mapsCompleted;
- }
- public int getReducesTotal() {
- return reducesTotal;
- }
- public void setReducesTotal(int reducesTotal) {
- this.reducesTotal = reducesTotal;
- }
- public int getReducesCompleted() {
- return reducesCompleted;
- }
- public void setReducesCompleted(int reducesCompleted) {
- this.reducesCompleted = reducesCompleted;
- }
- public double getMapProgress() {
- return mapProgress;
- }
- public void setMapProgress(double mapProgress) {
- this.mapProgress = mapProgress;
- }
- public double getReduceProgress() {
- return reduceProgress;
- }
- public void setReduceProgress(double reduceProgress) {
- this.reduceProgress = reduceProgress;
- }
- public int getMapsPending() {
- return mapsPending;
- }
- public void setMapsPending(int mapsPending) {
- this.mapsPending = mapsPending;
- }
- public int getMapsRunning() {
- return mapsRunning;
- }
- public void setMapsRunning(int mapsRunning) {
- this.mapsRunning = mapsRunning;
- }
- public int getReducesPending() {
- return reducesPending;
- }
- public void setReducesPending(int reducesPending) {
- this.reducesPending = reducesPending;
- }
- public int getReducesRunning() {
- return reducesRunning;
- }
- public void setReducesRunning(int reducesRunning) {
- this.reducesRunning = reducesRunning;
- }
- public boolean isUberized() {
- return uberized;
- }
- public void setUberized(boolean uberized) {
- this.uberized = uberized;
- }
- public String getDiagnostics() {
- return diagnostics;
- }
- public void setDiagnostics(String diagnostics) {
- this.diagnostics = diagnostics;
- }
- public int getNewReduceAttempts() {
- return newReduceAttempts;
- }
- public void setNewReduceAttempts(int newReduceAttempts) {
- this.newReduceAttempts = newReduceAttempts;
- }
- public int getRunningReduceAttempts() {
- return runningReduceAttempts;
- }
- public void setRunningReduceAttempts(int runningReduceAttempts) {
- this.runningReduceAttempts = runningReduceAttempts;
- }
- public int getFailedReduceAttempts() {
- return failedReduceAttempts;
- }
- public void setFailedReduceAttempts(int failedReduceAttempts) {
- this.failedReduceAttempts = failedReduceAttempts;
- }
- public int getKilledReduceAttempts() {
- return killedReduceAttempts;
- }
- public void setKilledReduceAttempts(int killedReduceAttempts) {
- this.killedReduceAttempts = killedReduceAttempts;
- }
- public int getSuccessfulReduceAttempts() {
- return successfulReduceAttempts;
- }
- public void setSuccessfulReduceAttempts(int successfulReduceAttempts) {
- this.successfulReduceAttempts = successfulReduceAttempts;
- }
- public int getNewMapAttempts() {
- return newMapAttempts;
- }
- public void setNewMapAttempts(int newMapAttempts) {
- this.newMapAttempts = newMapAttempts;
- }
- public int getRunningMapAttempts() {
- return runningMapAttempts;
- }
- public void setRunningMapAttempts(int runningMapAttempts) {
- this.runningMapAttempts = runningMapAttempts;
- }
- public int getFailedMapAttempts() {
- return failedMapAttempts;
- }
- public void setFailedMapAttempts(int failedMapAttempts) {
- this.failedMapAttempts = failedMapAttempts;
- }
- public int getKilledMapAttempts() {
- return killedMapAttempts;
- }
- public void setKilledMapAttempts(int killedMapAttempts) {
- this.killedMapAttempts = killedMapAttempts;
- }
- public int getSuccessfulMapAttempts() {
- return successfulMapAttempts;
- }
- public void setSuccessfulMapAttempts(int successfulMapAttempts) {
- this.successfulMapAttempts = successfulMapAttempts;
- }
- public String getQueue() {
- return queue;
- }
- public void setQueue(String queue) {
- this.queue = queue;
- }
- public eagle.jobrunning.counter.JobCounters getJobcounter() {
- return jobcounter;
- }
- public void setJobcounter(eagle.jobrunning.counter.JobCounters jobcounter) {
- this.jobcounter = jobcounter;
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/afe86834/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/yarn/model/Jobs.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/yarn/model/Jobs.java b/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/yarn/model/Jobs.java
deleted file mode 100644
index 3b0ad9c..0000000
--- a/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/yarn/model/Jobs.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package eagle.jobrunning.yarn.model;
-
-import java.util.List;
-
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL)
-@JsonIgnoreProperties(ignoreUnknown = true)
-public class Jobs {
-
- private List<JobDetailInfo> job;
-
- public List<JobDetailInfo> getJob() {
- return job;
- }
-
- public void setJob(List<JobDetailInfo> job) {
- this.job = job;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/afe86834/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/yarn/model/JobsWrapper.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/yarn/model/JobsWrapper.java b/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/yarn/model/JobsWrapper.java
deleted file mode 100644
index 7c262fa..0000000
--- a/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/yarn/model/JobsWrapper.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package eagle.jobrunning.yarn.model;
-
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-
-@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL)
-@JsonIgnoreProperties(ignoreUnknown = true)
-public class JobsWrapper {
-
- private Jobs jobs;
-
- public Jobs getJobs() {
- return jobs;
- }
-
- public void setJobs(Jobs jobs) {
- this.jobs = jobs;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/afe86834/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/zkres/JobRunningZKStateLCM.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/zkres/JobRunningZKStateLCM.java b/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/zkres/JobRunningZKStateLCM.java
deleted file mode 100644
index b0cc199..0000000
--- a/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/zkres/JobRunningZKStateLCM.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package eagle.jobrunning.zkres;
-
-import java.util.List;
-
-import eagle.jobrunning.common.JobConstants.ResourceType;
-
-public interface JobRunningZKStateLCM {
-
- List<String> readProcessedJobs(ResourceType type);
-
- void addProcessedJob(ResourceType type, String jobID);
-
- // date format e.g. "20150901"
- void truncateJobBefore(ResourceType type, String date);
-
- void truncateProcessedJob(ResourceType type, String jobID);
-
- void truncateEverything();
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/afe86834/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/zkres/JobRunningZKStateManager.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/zkres/JobRunningZKStateManager.java b/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/zkres/JobRunningZKStateManager.java
deleted file mode 100644
index 3d5901a..0000000
--- a/eagle-core/eagle-data-process/eagle-storm-jobrunning-spout/src/main/java/eagle/jobrunning/zkres/JobRunningZKStateManager.java
+++ /dev/null
@@ -1,210 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package eagle.jobrunning.zkres;
-
-import java.nio.charset.StandardCharsets;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.List;
-
-import eagle.common.config.EagleConfigFactory;
-import org.apache.curator.framework.CuratorFramework;
-import org.apache.curator.framework.CuratorFrameworkFactory;
-import org.apache.curator.framework.recipes.locks.InterProcessMutex;
-import org.apache.curator.retry.RetryNTimes;
-import org.apache.zookeeper.CreateMode;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import eagle.jobrunning.common.JobConstants.ResourceType;
-import eagle.jobrunning.config.RunningJobCrawlConfig;
-import eagle.common.DateTimeUtil;
-
-public class JobRunningZKStateManager implements JobRunningZKStateLCM{
- public static final Logger LOG = LoggerFactory.getLogger(JobRunningZKStateManager.class);
- private String zkRoot;
- private CuratorFramework _curator;
-
- public static final String DATE_FORMAT_PATTERN = "yyyyMMdd";
-
- private CuratorFramework newCurator(RunningJobCrawlConfig config) throws Exception {
- return CuratorFrameworkFactory.newClient(
- config.zkStateConfig.zkQuorum,
- config.zkStateConfig.zkSessionTimeoutMs,
- 15000,
- new RetryNTimes(config.zkStateConfig.zkRetryTimes, config.zkStateConfig.zkRetryInterval)
- );
- }
-
- public JobRunningZKStateManager(RunningJobCrawlConfig config) {
- this.zkRoot = config.zkStateConfig.zkRoot;
- try {
- _curator = newCurator(config);
- _curator.start();
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
-
- public void close() {
- _curator.close();
- _curator = null;
- }
-
- public long getTimestampFromDate(String dateStr) throws ParseException {
- SimpleDateFormat sdf = new SimpleDateFormat(DATE_FORMAT_PATTERN);
- sdf.setTimeZone(EagleConfigFactory.load().getTimeZone());
- Date d = sdf.parse(dateStr);
- return d.getTime();
- }
-
- @Override
- public List<String> readProcessedJobs(ResourceType type) {
- String path = zkRoot + "/" + type.name() + "/jobs";
- InterProcessMutex lock = new InterProcessMutex(_curator, path);
- try {
- lock.acquire();
- if (_curator.checkExists().forPath(path) != null) {
- LOG.info("Got processed job list from zk, type: " + type.name());
- return _curator.getChildren().forPath(path);
- } else {
- LOG.info("Currently processed job list is empty, type: " + type.name());
- return new ArrayList<String>();
- }
- } catch (Exception e) {
- LOG.error("fail read processed jobs", e);
- throw new RuntimeException(e);
- }
- finally {
- try{
- lock.release();
- }catch(Exception e){
- LOG.error("fail releasing lock", e);
- throw new RuntimeException(e);
- }
- }
- }
-
- @Override
- public void addProcessedJob(ResourceType type, String jobID) {
- String path = zkRoot + "/" + type.name() + "/jobs/" + jobID;
- try {
- //we record date for cleanup, e.g. cleanup job's znodes whose created date < 20150801
- String date = DateTimeUtil.format(System.currentTimeMillis(), DATE_FORMAT_PATTERN);
- LOG.info("add processed job, jobID: " + jobID + ", type: " + type + ", date: " + date);
- if (_curator.checkExists().forPath(path) == null) {
- _curator.create()
- .creatingParentsIfNeeded()
- .withMode(CreateMode.PERSISTENT)
- .forPath(path, date.getBytes(StandardCharsets.UTF_8));
- }
- else {
- LOG.warn("Job already exist in zk, skip the job: " + jobID + " , type: " + type);
- }
- } catch (Exception e) {
- LOG.error("fail adding processed jobs", e);
- throw new RuntimeException(e);
- }
- }
-
- @Override
- public void truncateJobBefore(ResourceType type, String date) {
- String path = zkRoot + "/" + type.name() + "/jobs";
- InterProcessMutex lock = new InterProcessMutex(_curator, path);
- try {
- lock.acquire();
- long thresholdTime = getTimestampFromDate(date);
- if (_curator.checkExists().forPath(path) != null) {
- LOG.info("Going to delete processed job before " + date + ", type: " + type);
- List<String> jobIDList = _curator.getChildren().forPath(path);
- for(String jobID : jobIDList) {
- if (!jobID.startsWith("job_")) continue; // skip lock node
- String jobPath = path + "/" + jobID;
- long createTime = getTimestampFromDate(new String(_curator.getData().forPath(jobPath), StandardCharsets.UTF_8));
- if (createTime < thresholdTime) {
- LOG.info("Going to truncate job: " + jobPath);
- _curator.delete().deletingChildrenIfNeeded().forPath(jobPath);
- }
- }
- }
- else {
- LOG.info("Currently processed job list is empty, type: " + type.name());
- }
- } catch (Exception e) {
- LOG.error("fail deleting processed jobs", e);
- throw new RuntimeException(e);
- }
- finally {
- try{
- lock.release();
- }catch(Exception e){
- LOG.error("fail releasing lock", e);
- throw new RuntimeException(e);
- }
- }
- }
-
- @Override
- public void truncateProcessedJob(ResourceType type, String jobID) {
- LOG.info("trying to truncate all data for job " + jobID);
- String path = zkRoot + "/" + type.name() + "/jobs/" + jobID;
- InterProcessMutex lock = new InterProcessMutex(_curator, path);
- try {
- lock.acquire();
- if (_curator.checkExists().forPath(path) != null) {
- _curator.delete().deletingChildrenIfNeeded().forPath(path);
- LOG.info("really truncated all data for jobID: " + jobID);
- }
- } catch (Exception e) {
- LOG.error("fail truncating processed jobs", e);
- throw new RuntimeException(e);
- }
- finally {
- try{
- lock.release();
- }catch(Exception e){
- LOG.error("fail releasing lock", e);
- throw new RuntimeException(e);
- }
- }
- }
-
- @Override
- public void truncateEverything() {
- String path = zkRoot;
- InterProcessMutex lock = new InterProcessMutex(_curator, path);
- try{
- lock.acquire();
- if(_curator.checkExists().forPath(path) != null){
- _curator.delete().deletingChildrenIfNeeded().forPath(path);
- }
- }catch(Exception ex){
- LOG.error("fail truncating verything", ex);
- throw new RuntimeException(ex);
- }
- finally {
- try{
- lock.release();
- }catch(Exception e){
- LOG.error("fail releasing lock", e);
- throw new RuntimeException(e);
- }
- }
- }
-}