You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by de...@apache.org on 2013/04/25 18:55:10 UTC
svn commit: r1475853 - in
/uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/json:
./ jp/ jp/FileInfo.java jp/JobProcessCollection.java jp/JobProcessData.java
Author: degenaro
Date: Thu Apr 25 16:55:10 2013
New Revision: 1475853
URL: http://svn.apache.org/r1475853
Log:
UIMA-2825 DUCC webserver (WS) Job Details Processes tab revisions
Added:
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/json/
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/json/jp/
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/json/jp/FileInfo.java
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/json/jp/JobProcessCollection.java
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/json/jp/JobProcessData.java
Added: uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/json/jp/FileInfo.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/json/jp/FileInfo.java?rev=1475853&view=auto
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/json/jp/FileInfo.java (added)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/json/jp/FileInfo.java Thu Apr 25 16:55:10 2013
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+*/
+package org.apache.uima.ducc.transport.json.jp;
+
+public class FileInfo {
+
+ public String name;
+}
Added: uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/json/jp/JobProcessCollection.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/json/jp/JobProcessCollection.java?rev=1475853&view=auto
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/json/jp/JobProcessCollection.java (added)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/json/jp/JobProcessCollection.java Thu Apr 25 16:55:10 2013
@@ -0,0 +1,493 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+*/
+package org.apache.uima.ducc.transport.json.jp;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.lang.reflect.Type;
+import java.util.ArrayList;
+import java.util.concurrent.ConcurrentSkipListMap;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.GZIPOutputStream;
+
+import org.apache.uima.ducc.common.utils.IOHelper;
+import org.apache.uima.ducc.common.utils.id.DuccId;
+import org.apache.uima.ducc.transport.event.common.IDuccProcess;
+import org.apache.uima.ducc.transport.event.common.IDuccProcessMap;
+import org.apache.uima.ducc.transport.event.common.IDuccProcessWorkItems;
+import org.apache.uima.ducc.transport.event.common.IDuccWorkJob;
+import org.apache.uima.ducc.transport.event.common.TimeWindow;
+
+import com.google.gson.Gson;
+import com.google.gson.reflect.TypeToken;
+
+public class JobProcessCollection {
+
+ public static final String job_processes_data_json_gz = "job-processes-data.json.gz";
+ public static final String encoding = "UTF-8";
+
+ private String directory = null;
+ private String fileName = null;
+
+ private Gson gson = new Gson();
+
+ public JobProcessCollection(IDuccWorkJob job) {
+ String fileDirectory = job.getUserLogsDir()+job.getDuccId().getFriendly()+File.separator;
+ initialize(fileDirectory);
+ }
+
+ public JobProcessCollection(String fileDirectory) {
+ initialize(fileDirectory);
+ }
+
+ private static long getId(IDuccWorkJob job, IDuccProcess process) {
+ long retVal = 0;
+ try {
+ retVal = Long.parseLong(process.getDuccId().toString().trim());
+ }
+ catch(Exception e) {
+ }
+ return retVal;
+ }
+
+ private static String getLogDirectory(IDuccWorkJob job, IDuccProcess process) {
+ String retVal = null;
+ try {
+ retVal = job.getUserLogsDir()+job.getDuccId().getFriendly()+File.separator;
+ }
+ catch(Exception e) {
+ }
+ return retVal;
+ }
+
+ private static ArrayList<FileInfo> getLogFileList(IDuccWorkJob job, IDuccProcess process) {
+ ArrayList<FileInfo> retVal = new ArrayList<FileInfo>();
+ try {
+ FileInfo fileInfo = new FileInfo();
+ fileInfo.name = "jd.out.log";
+ }
+ catch(Exception e) {
+ }
+ return retVal;
+ }
+
+ private static String getHostName(IDuccWorkJob job, IDuccProcess process) {
+ String retVal = null;
+ try {
+ retVal = process.getNodeIdentity().getName();
+ }
+ catch(Exception e) {
+ }
+ return retVal;
+ }
+
+ private static long getHostPid(IDuccWorkJob job, IDuccProcess process) {
+ long retVal = 0;
+ try {
+ retVal = Long.parseLong(process.getPID());
+ }
+ catch(Exception e) {
+ }
+ return retVal;
+ }
+
+ private static String getSchedulerState(IDuccWorkJob job, IDuccProcess process) {
+ String retVal = null;
+ try {
+ retVal = process.getResourceState().toString();
+ }
+ catch(Exception e) {
+ }
+ return retVal;
+ }
+
+ private static String getSchedulerReason(IDuccWorkJob job, IDuccProcess process) {
+ String retVal = null;
+ try {
+ retVal = process.getProcessDeallocationType().toString();
+ }
+ catch(Exception e) {
+ }
+ return retVal;
+ }
+
+ private static String getAgentState(IDuccWorkJob job, IDuccProcess process) {
+ String retVal = null;
+ try {
+ retVal = process.getProcessState().toString();
+ }
+ catch(Exception e) {
+ }
+ return retVal;
+ }
+
+ private static String getAgentReason(IDuccWorkJob job, IDuccProcess process) {
+ String retVal = null;
+ try {
+ retVal = process.getReasonForStoppingProcess();
+ }
+ catch(Exception e) {
+ }
+ return retVal;
+ }
+
+ private static long getTimeInitStart(IDuccWorkJob job, IDuccProcess process) {
+ long retVal = 0;
+ try {
+ retVal = ((TimeWindow) process.getTimeWindowInit()).getStartLong();
+ }
+ catch(Exception e) {
+ }
+ return retVal;
+ }
+
+ private static long getTimeInitEnd(IDuccWorkJob job, IDuccProcess process) {
+ long retVal = 0;
+ try {
+ retVal = ((TimeWindow) process.getTimeWindowInit()).getEndLong();
+ }
+ catch(Exception e) {
+ }
+ return retVal;
+ }
+
+ private static long getTimeRunStart(IDuccWorkJob job, IDuccProcess process) {
+ long retVal = 0;
+ try {
+ retVal = ((TimeWindow) process.getTimeWindowRun()).getStartLong();
+ }
+ catch(Exception e) {
+ }
+ return retVal;
+ }
+
+ private static long getTimeRunEnd(IDuccWorkJob job, IDuccProcess process) {
+ long retVal = 0;
+ try {
+ retVal = ((TimeWindow) process.getTimeWindowRun()).getEndLong();
+ }
+ catch(Exception e) {
+ }
+ return retVal;
+ }
+
+ private static long getTimeGC(IDuccWorkJob job, IDuccProcess process) {
+ long retVal = 0;
+ try {
+ retVal = process.getGarbageCollectionStats().getCollectionTime();
+ }
+ catch(Exception e) {
+ }
+ return retVal;
+ }
+
+ private static long getPgIn(IDuccWorkJob job, IDuccProcess process) {
+ long retVal = 0;
+ try {
+ retVal = process.getMajorFaults();
+ }
+ catch(Exception e) {
+ }
+ return retVal;
+ }
+
+ private static long getSwap(IDuccWorkJob job, IDuccProcess process) {
+ long retVal = 0;
+ try {
+ retVal = process.getSwapUsage();
+ }
+ catch(Exception e) {
+ }
+ return retVal;
+ }
+
+ private static long getSwapMax(IDuccWorkJob job, IDuccProcess process) {
+ long retVal = 0;
+ try {
+ retVal = process.getSwapUsageMax();
+ }
+ catch(Exception e) {
+ }
+ return retVal;
+ }
+
+ private static long getRss(IDuccWorkJob job, IDuccProcess process) {
+ long retVal = 0;
+ try {
+ retVal = process.getResidentMemory();
+ }
+ catch(Exception e) {
+ }
+ return retVal;
+ }
+
+ private static long getRssMax(IDuccWorkJob job, IDuccProcess process) {
+ long retVal = 0;
+ try {
+ retVal = process.getResidentMemoryMax();
+ }
+ catch(Exception e) {
+ }
+ return retVal;
+ }
+
+ private static long getWiTimeAvg(IDuccWorkJob job, IDuccProcess process) {
+ long retVal = 0;
+ try {
+ IDuccProcessWorkItems pwi = process.getProcessWorkItems();
+ retVal = pwi.getSecsAvg();
+ }
+ catch(Exception e) {
+ }
+ return retVal;
+ }
+
+ private static long getWiTimeMax(IDuccWorkJob job, IDuccProcess process) {
+ long retVal = 0;
+ try {
+ IDuccProcessWorkItems pwi = process.getProcessWorkItems();
+ retVal = pwi.getSecsMax();
+ }
+ catch(Exception e) {
+ }
+ return retVal;
+ }
+
+ private static long getWiTimeMin(IDuccWorkJob job, IDuccProcess process) {
+ long retVal = 0;
+ try {
+ IDuccProcessWorkItems pwi = process.getProcessWorkItems();
+ retVal = pwi.getSecsMin();
+ }
+ catch(Exception e) {
+ }
+ return retVal;
+ }
+
+ private static long getWiDone(IDuccWorkJob job, IDuccProcess process) {
+ long retVal = 0;
+ try {
+ IDuccProcessWorkItems pwi = process.getProcessWorkItems();
+ retVal = pwi.getCountDone();
+ }
+ catch(Exception e) {
+ }
+ return retVal;
+ }
+
+ private static long getWiError(IDuccWorkJob job, IDuccProcess process) {
+ long retVal = 0;
+ try {
+ IDuccProcessWorkItems pwi = process.getProcessWorkItems();
+ retVal = pwi.getCountError();
+ }
+ catch(Exception e) {
+ }
+ return retVal;
+ }
+
+ private static long getWiRetry(IDuccWorkJob job, IDuccProcess process) {
+ long retVal = 0;
+ try {
+ IDuccProcessWorkItems pwi = process.getProcessWorkItems();
+ retVal = pwi.getCountRetry();
+ }
+ catch(Exception e) {
+ }
+ return retVal;
+ }
+
+ private static long getWiPreempt(IDuccWorkJob job, IDuccProcess process) {
+ long retVal = 0;
+ try {
+ IDuccProcessWorkItems pwi = process.getProcessWorkItems();
+ retVal = pwi.getCountPreempt();
+ }
+ catch(Exception e) {
+ }
+ return retVal;
+ }
+
+ private static String getJConsole(IDuccWorkJob job, IDuccProcess process) {
+ String retVal = null;
+ try {
+ retVal = process.getProcessJmxUrl();
+ }
+ catch(Exception e) {
+ }
+ return retVal;
+ }
+
+ private void transform(ConcurrentSkipListMap<Long, JobProcessData> map, IDuccWorkJob job, IDuccProcessMap processMap) {
+ if(job != null) {
+ if(processMap != null) {
+ for(DuccId duccId : processMap.keySet()) {
+ IDuccProcess process = processMap.get(duccId);
+ JobProcessData jpd = new JobProcessData();
+ jpd.id = getId(job, process);
+ jpd.logDirectory = getLogDirectory(job, process);
+ jpd.logFileList = getLogFileList(job, process);
+ jpd.hostName = getHostName(job, process);
+ jpd.hostPid = getHostPid(job, process);
+ jpd.schedulerState = getSchedulerState(job, process);
+ jpd.schedulerReason = getSchedulerReason(job, process);
+ jpd.agentState = getAgentState(job, process);
+ jpd.agentReason = getAgentReason(job, process);
+ jpd.timeInitStart = getTimeInitStart(job, process);
+ jpd.timeInitEnd = getTimeInitEnd(job, process);
+ jpd.timeRunStart = getTimeRunStart(job, process);
+ jpd.timeRunEnd = getTimeRunEnd(job, process);
+ jpd.timeGC = getTimeGC(job, process);
+ jpd.pageIn = getPgIn(job, process);
+ jpd.swap = getSwap(job, process);
+ jpd.swapMax = getSwapMax(job, process);
+ jpd.rss = getRss(job, process);
+ jpd.rssMax = getRssMax(job, process);
+ jpd.wiTimeAvg = getWiTimeAvg(job, process);
+ jpd.wiTimeMax = getWiTimeMax(job, process);
+ jpd.wiTimeMin = getWiTimeMin(job, process);
+ jpd.wiDone = getWiDone(job, process);
+ jpd.wiError = getWiError(job, process);
+ jpd.wiRetry = getWiRetry(job, process);
+ jpd.wiPreempt = getWiPreempt(job, process);
+ jpd.jConsole = getJConsole(job, process);
+ Long key = new Long(jpd.id);
+ JobProcessData value = jpd;
+ map.put(key, value);
+ }
+ }
+ }
+ }
+
+ public ConcurrentSkipListMap<Long, JobProcessData> transform(IDuccWorkJob job) {
+ ConcurrentSkipListMap<Long, JobProcessData> map = new ConcurrentSkipListMap<Long, JobProcessData>();
+ if(job != null) {
+ IDuccProcessMap jpMap = job.getProcessMap();
+ transform(map, job, jpMap);
+ IDuccProcessMap jdMap = job.getDriver().getProcessMap();
+ transform(map, job, jdMap);
+ }
+ return map;
+ }
+
+ public void initialize(String fileDirectory) {
+ if(fileDirectory == null) {
+ throw new NullPointerException();
+ }
+ directory = fileDirectory;
+ fileName = IOHelper.marryDir2File(directory,job_processes_data_json_gz);
+ }
+
+ public String getDirectory() {
+ return directory;
+ }
+
+ public String getQualifiedFileName() {
+ return fileName;
+ }
+
+ public void exportData(ConcurrentSkipListMap<Long, JobProcessData> map) throws IOException {
+ if(map == null) {
+ throw new NullPointerException();
+ }
+ BufferedWriter writer = null;
+ OutputStreamWriter osr = null;
+ GZIPOutputStream gos = null;
+ FileOutputStream fos = null;
+ File file = null;
+ try {
+ String json = gson.toJson(map);
+ file = new File(fileName);
+ fos = new FileOutputStream(file);
+ gos = new GZIPOutputStream(fos);
+ osr = new OutputStreamWriter(gos, encoding);
+ writer = new BufferedWriter(osr);
+ writer.write(json);
+ }
+ finally {
+ if(writer != null) {
+ writer.close();
+ }
+ }
+ }
+
+ private int getSize() throws IOException {
+ int size = 0;
+ BufferedReader reader = null;
+ InputStreamReader isr = null;
+ GZIPInputStream gis = null;
+ FileInputStream fis = null;
+ File file = null;
+ int rc = 0;
+ try {
+ file = new File(fileName);
+ fis = new FileInputStream(file);
+ gis = new GZIPInputStream(fis);
+ isr = new InputStreamReader(gis, encoding);
+ reader = new BufferedReader(isr);
+ while(rc >= 0) {
+ rc = reader.read();
+ size++;
+ }
+ }
+ finally {
+ if(reader != null) {
+ reader.close();
+ }
+ }
+ return size-1;
+ }
+
+ public ConcurrentSkipListMap<Long, JobProcessData> importData() throws IOException {
+ ConcurrentSkipListMap<Long, JobProcessData> map = new ConcurrentSkipListMap<Long, JobProcessData>();
+ BufferedReader reader = null;
+ InputStreamReader isr = null;
+ GZIPInputStream gis = null;
+ FileInputStream fis = null;
+ File file = null;
+ char[] cbuf = new char[getSize()];
+ try {
+ file = new File(getQualifiedFileName());
+ fis = new FileInputStream(file);
+ gis = new GZIPInputStream(fis);
+ isr = new InputStreamReader(gis, encoding);
+ reader = new BufferedReader(isr);
+ reader.read(cbuf);
+ String json = new String(cbuf);
+ Type typeOfMap = new TypeToken<ConcurrentSkipListMap<Long,JobProcessData>>() { }.getType();
+ map = gson.fromJson(json, typeOfMap);
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ }
+ finally {
+ if(reader != null) {
+ reader.close();
+ }
+ }
+ return map;
+ }
+
+}
Added: uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/json/jp/JobProcessData.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/json/jp/JobProcessData.java?rev=1475853&view=auto
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/json/jp/JobProcessData.java (added)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/json/jp/JobProcessData.java Thu Apr 25 16:55:10 2013
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+*/
+package org.apache.uima.ducc.transport.json.jp;
+
+import java.util.List;
+
+public class JobProcessData {
+
+ public class LogFile {
+ String name = null;
+ long size = 0;
+ }
+
+ public long id = 0;
+ public String logDirectory = null;
+ public List<FileInfo> logFileList = null;
+ public String hostName = null;
+ public long hostPid = 0;
+ public String schedulerState = null;
+ public String schedulerReason = null;
+ public String agentState = null;
+ public String agentReason = null;
+ public long timeInitStart = 0;
+ public long timeInitEnd = 0;
+ public long timeRunStart = 0;
+ public long timeRunEnd = 0;
+ public long timeGC = 0;
+ public long pageIn = 0;
+ public long swap = 0;
+ public long swapMax = 0;
+ public long rss = 0;
+ public long rssMax = 0;
+ public long wiTimeAvg = 0;
+ public long wiTimeMax = 0;
+ public long wiTimeMin = 0;
+ public long wiDone = 0;
+ public long wiError = 0;
+ public long wiRetry = 0;
+ public long wiPreempt = 0;
+ public String jConsole = null;
+}