You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by ch...@apache.org on 2015/12/10 21:13:27 UTC
svn commit: r1719195 - in /uima/sandbox/uima-ducc/trunk: src/main/admin/
uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/
uima-ducc-database/src/main/java/org/apache/uima/ducc/database/
Author: challngr
Date: Thu Dec 10 20:13:26 2015
New Revision: 1719195
URL: http://svn.apache.org/viewvc?rev=1719195&view=rev
Log:
UIMA-4577 Tweaks so database classes can be used in CLI utilities. Implement rm_qoccupancy using
the DB as proof of concept.
Added:
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmQLoad.java
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmShareState.java
Modified:
uima/sandbox/uima-ducc/trunk/src/main/admin/rm_qoccupancy
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/IRmPersistence.java
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/NullRmStatePersistence.java
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbHandle.java
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbManager.java
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmNodeState.java
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmStatePersistence.java
Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/rm_qoccupancy
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/rm_qoccupancy?rev=1719195&r1=1719194&r2=1719195&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/admin/rm_qoccupancy (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/rm_qoccupancy Thu Dec 10 20:13:26 2015
@@ -21,59 +21,95 @@
import os
import sys
+import glob
import subprocess
from ducc_util import DuccUtil
class DuccRmQOccupancy(DuccUtil):
- def format_shares(self, sh):
- for s in sh:
- if ( s['blacklisted'] ):
- pass
- else:
- print '%19s J[%8d] S[%8d] O[%d] II[%8d] IR[%8d] E[%5s] P[%5s] F[%5s] I[%5s]' % ('', s['jobid'], s['shareid'], s['order'], s['investment-init'], s['investment-run'],
- s['evicted'], s['purged'], s['fixed'], s['initialized'])
-
-
- def format_machines(self, lines):
-
- print("%20s %11s %6s %6s %15s %10s %5s %6s" % ("Node", "Blacklisted", "Online", "Status", "Nodepool", "Memory", "Order", "Free"))
-
- for m in lines:
- print "%20s %11s %6s %6s %15s %10s %5s %6s" % (m['name'], m['blacklisted'], m['online'], m['status'], m['nodepool'], m['memory'], m['order'], m['shares-free'])
- if ( len(m['shares']) != 0 ) :
- self.format_shares(m['shares'])
- print ''
+ def format(self, nodes, shares):
+ print("%20s %11s %6s %6s %15s %10s %6s %6s %6s %8s %7s %10s %8s" % ("Node", "Blacklisted", "Online", "Status", "Nodepool", "Memory", "Order", "Free", "In-Use", "Np-InUse", "Quantum", "Reservable", "Classes"))
+ print("%20s %11s %6s %6s %15s %10s %6s %6s %6s %8s %7s %10s %8s" % ("----", "-----------", "------", "------", "--------", "------", "-----", "----", "------", "--------", "-------", "----------", "-------"))
+ for n in nodes:
+ if (n['responsive']):
+ status = 'up'
+ else:
+ status = 'down'
+ print "%20s %11s %6s %6s %15s %10s %3s(Q) %6s %6s %8s %7s %10s %8s" % (n['name'], n['blacklisted'], n['online'], status, n['nodepool'],
+ n['memory'], n['share_order'], n['shares_left'], n['assignments'],
+ n['np_assignments'], n['quantum'], n['reservable'], n['classes'])
+ if ( shares.has_key(n['name']) ):
+ for s in shares[n['name']]:
+ fmt = '%19s ' + s['jobtype'] +'[%8d] S[%8d] O[%d] II[%8d] IR[%8d] E[%5s] P[%5s] F[%5s] S[%10s]'
+ state = s['state']
+ if ( state == 'null' ):
+ state = "Assigned"
+ print fmt % ('', s['job_id'], s['ducc_dbid'], s['share_order'], s['init_time'], s['investment'], s['evicted'], s['purged'], s['fixed'], state)
- def format(self, lines):
- qoccupancy = eval(lines)
- self.format_machines(qoccupancy)
+ print ''
+ # Given DUCC_HOME, a directory, and part of the name of a jar, find the actual name of the jar which will
+ # likeley be versioned
+ def resolve_jar(self, DH, dirname, basename):
+ partial = DH + '/' + dirname + '/'+ basename + '*'
+ ret = glob.glob(partial)
+ return ret[0]
+
def main(self, argv):
if ( argv == '-h' or argv == '-?' or argv == '--help' ):
print 'rm_qload queries and formats the current state of the RM scheduling tables. It takes no parameters.'
sys.exit(1);
- DUCC_JVM_OPTS = ' -Dducc.deploy.configuration=' + self.DUCC_HOME + "/resources/ducc.properties "
+ DH = self.DUCC_HOME
+ CP = [self.resolve_jar(DH, '/lib/uima-ducc', 'uima-ducc-database'),
+ self.resolve_jar(DH, '/lib/uima-ducc', 'uima-ducc-common'),
+ DH + '/lib/cassandra/*',
+ DH + '/lib/apache-log4j/*',
+ DH + '/lib/guava/*',
+ self.resolve_jar(DH, '/cassandra-server/lib', 'slf4j-api'),
+ self.resolve_jar(DH, '/apache-uima/apache-activemq/lib', 'slf4j-log4j12'),
+ ]
+ os.environ['CLASSPATH'] = ':'.join(CP)
+
+ DUCC_JVM_OPTS = ''
DUCC_JVM_OPTS = DUCC_JVM_OPTS + ' -DDUCC_HOME=' + self.DUCC_HOME
- DUCC_JVM_OPTS = DUCC_JVM_OPTS + ' -Dducc.head=' + self.ducc_properties.get('ducc.head')
+ DUCC_JVM_OPTS = DUCC_JVM_OPTS + ' -Dducc.rm.persistence.impl=org.apache.uima.ducc.database.RmStatePersistence'
- CMD = [self.java(), DUCC_JVM_OPTS, 'org.apache.uima.ducc.common.main.DuccRmAdmin', '--qoccupancy']
+ dbn = self.ducc_properties.get('ducc.database.host')
+ CMD = [self.java(), DUCC_JVM_OPTS, 'org.apache.uima.ducc.database.RmShareState', dbn]
+ CMD = ' '.join(CMD)
+ lines = ''
+ proc = subprocess.Popen(CMD, bufsize=0, stdout=subprocess.PIPE, shell=True)
+ for line in proc.stdout:
+ lines = lines + line
+
+ shares = eval(lines)
+
+ shares_by_machine = {}
+ for share in shares:
+ k = share['node']
+ if ( shares_by_machine.has_key(k) ):
+ share_list = shares_by_machine[k]
+ else:
+ share_list = []
+ shares_by_machine[k] = share_list
+ share_list.append(share)
+
+ CMD = [self.java(), DUCC_JVM_OPTS, 'org.apache.uima.ducc.database.RmNodeState', dbn]
CMD = ' '.join(CMD)
lines = ''
proc = subprocess.Popen(CMD, bufsize=0, stdout=subprocess.PIPE, shell=True)
for line in proc.stdout:
lines = lines + line
- if 'not yet initialized' in lines:
- print lines
- return
-
- self.format(lines)
+ nodes = eval(lines)
+ nodes = sorted(nodes, key=lambda n: n["name"])
+
+ self.format(nodes, shares_by_machine)
return
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/IRmPersistence.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/IRmPersistence.java?rev=1719195&r1=1719194&r2=1719195&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/IRmPersistence.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/IRmPersistence.java Thu Dec 10 20:13:26 2015
@@ -19,6 +19,7 @@
package org.apache.uima.ducc.common.persistence.rm;
+import java.util.List;
import java.util.Map;
import java.util.Properties;
@@ -167,7 +168,7 @@ public interface IRmPersistence
/**
* Fetch all machines in the database.
*
- * @return A map of properties objects containing full details about the machines,
+ * @return A map of map objects containing full details about the machines,
* keyed on machine name. If there are no machines found in the db,
* an empty map is returned.
*
@@ -177,6 +178,31 @@ public interface IRmPersistence
public Map<String, Map<String, Object>> getAllMachines() throws Exception;
/**
+ * Fetch all shares in the database.
+ *
+ * @return A map of map objects containing full details about the shares.
+ * The key is node:jobid:shareid concatenated as a string, as it takes all of these
+ * to uniquely identify a share. If there are no shares found in the db,
+ * an empty map is returned.
+ *
+ * @throws Exception. Anything that goes wrong throws. Usually the
+ * throw will originate in the DB because of some DB issue.
+ */
+ public Map<String, Map<String, Object>> getAllShares() throws Exception;
+
+ /**
+ * Fetch all the load records. These are relatively short, one per job/reservation/etc
+ * in the database, summarizing the resources they have vs the resources they want in
+ * a perfect world.
+ *
+ * @return A list of map objects containing load information.
+ *
+ * @throws Exception. Anything that goes wrong throws. Usually the
+ * throw will originate in the DB because of some DB issue.
+ */
+ public List<Map<String, Object>> getLoad() throws Exception;
+
+ /**
* A new job arrives (or is recovered after restart).
*/
public void addJob(IDbJob j) throws Exception;
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/NullRmStatePersistence.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/NullRmStatePersistence.java?rev=1719195&r1=1719194&r2=1719195&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/NullRmStatePersistence.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/NullRmStatePersistence.java Thu Dec 10 20:13:26 2015
@@ -18,7 +18,9 @@
*/
package org.apache.uima.ducc.common.persistence.rm;
+import java.util.ArrayList;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import java.util.Properties;
@@ -48,6 +50,8 @@ public class NullRmStatePersistence impl
public void updateShare(String node, DuccId shareid, DuccId jobid, long investment, String state, long init_time, long pid) {}
public Properties getMachine(String id) { return null; }
public Map<String, Map<String, Object>> getAllMachines() { return new HashMap<String, Map<String, Object>>(); }
+ public Map<String, Map<String, Object>> getAllShares() { return new HashMap<String, Map<String, Object>>(); }
+ public List<Map<String, Object>> getLoad() { return new ArrayList<Map<String, Object>>(); }
public void addJob(IDbJob j ) {}
public void deleteJob(IDbJob j ) {}
public void updateDemand(IDbJob j) {}
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbHandle.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbHandle.java?rev=1719195&r1=1719194&r2=1719195&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbHandle.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbHandle.java Thu Dec 10 20:13:26 2015
@@ -50,7 +50,7 @@ public class DbHandle
String methodName = "execute";
long now = System.currentTimeMillis();
ResultSet ret = manager.execute(sql);
- logger.info(methodName, null, "Time to execute", System.currentTimeMillis() - now);
+ if ( manager.noisy ) logger.info(methodName, null, "Time to execute", System.currentTimeMillis() - now);
return ret;
}
@@ -76,7 +76,7 @@ public class DbHandle
BoundStatement bound = boundStatement.bind(fields);
return execute(bound);
} finally {
- logger.info(methodName, null, "Time to execute prepared statement:", ps.getQueryString(), System.currentTimeMillis() - now);
+ if ( manager.noisy ) logger.info(methodName, null, "Time to execute prepared statement:", ps.getQueryString(), System.currentTimeMillis() - now);
}
}
@@ -253,7 +253,6 @@ public class DbHandle
PreparedStatement prepare(String cql)
{
String methodName = "prepare";
- logger.info(methodName, null, "Preparing", cql);
return manager.prepare(cql);
}
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbManager.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbManager.java?rev=1719195&r1=1719194&r2=1719195&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbManager.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbManager.java Thu Dec 10 20:13:26 2015
@@ -44,9 +44,11 @@ import com.datastax.driver.core.policies
public class DbManager
{
static final String URL_PROPERTY = "ducc.database.url";
+ static final String NOISE_PROPERTY = "ducc.database.noisy";
private static String db_id = null;
private static String db_pw = null;
+ boolean noisy = true;
String dburl;
DuccLogger logger;
@@ -59,6 +61,8 @@ public class DbManager
{
this.dburl = dburl;
this.logger = logger;
+
+ if ( System.getProperty(NOISE_PROPERTY) != null ) noisy = false;
}
boolean checkForDatabase()
@@ -97,7 +101,6 @@ public class DbManager
// will throw sometims, so we can assume
// we're allowed to continue if control passes down.
- logger.info(methodName, null, "Attach to", dburl, "as", db_id, db_pw);
PlainTextAuthProvider auth = new PlainTextAuthProvider(db_id, db_pw); // throws if no good
ReconnectionPolicy rp = new ConstantReconnectionPolicy(10000); // if we lose connection, keep trying every 10 seconds
@@ -108,17 +111,18 @@ public class DbManager
.build();
Metadata metadata = cluster.getMetadata();
- logger.info(methodName, null, "Connected to cluster:", metadata.getClusterName());
-
- for ( Host host : metadata.getAllHosts() ) {
- logger.info(methodName, null, "Datatacenter:", host.getDatacenter(), "Host:", host.getAddress(), "Rack:", host.getRack());
- }
+ if ( noisy) {
+ logger.info(methodName, null, "Connected to cluster:", metadata.getClusterName());
+ for ( Host host : metadata.getAllHosts() ) {
+ logger.info(methodName, null, "Datatacenter:", host.getDatacenter(), "Host:", host.getAddress(), "Rack:", host.getRack());
+ }
+ }
}
public synchronized void shutdown()
{
String methodName = "closeDatabase";
- logger.info(methodName, null, "Closing the database.");
+ if ( noisy ) logger.info(methodName, null, "Closing the database.");
if ( cluster != null ) cluster.close();
cluster = null;
session = null;
@@ -148,9 +152,9 @@ public class DbManager
{
String methodName = "execute";
if ( logger.isDebug() ) {
- logger.info(methodName, null, "EXECUTE CQL:", cql);
+ if ( noisy ) logger.info(methodName, null, "EXECUTE CQL:", cql);
} else {
- logger.info(methodName, null, "EXECUTE CQL:", truncateText(cql));
+ if ( noisy ) logger.info(methodName, null, "EXECUTE CQL:", truncateText(cql));
}
return session.execute(cql);
}
@@ -163,7 +167,7 @@ public class DbManager
ResultSet execute(SimpleStatement s)
{
String methodName = "execute";
- logger.info(methodName, null, "EXECUTE STATEMENT:", truncateText(s.getQueryString()));
+ if ( noisy ) logger.info(methodName, null, "EXECUTE STATEMENT:", truncateText(s.getQueryString()));
return session.execute(s);
}
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmNodeState.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmNodeState.java?rev=1719195&r1=1719194&r2=1719195&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmNodeState.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmNodeState.java Thu Dec 10 20:13:26 2015
@@ -21,11 +21,14 @@ package org.apache.uima.ducc.database;
import java.util.Map;
+import org.apache.uima.ducc.common.persistence.IDbProperty;
import org.apache.uima.ducc.common.persistence.rm.IRmPersistence;
+import org.apache.uima.ducc.common.persistence.rm.IRmPersistence.RmNodes;
import org.apache.uima.ducc.common.persistence.rm.NullRmStatePersistence;
import org.apache.uima.ducc.common.persistence.rm.RmPersistenceFactory;
import org.apache.uima.ducc.common.utils.DuccLogger;
+
public class RmNodeState
{
DuccLogger logger = DuccLogger.getLogger(RmNodeState.class, "State");
@@ -35,33 +38,20 @@ public class RmNodeState
{
this.dburl = dburl;
}
-
+
void run()
throws Exception
{
+ System.setProperty(DbManager.NOISE_PROPERTY, "false");
IRmPersistence persistence = RmPersistenceFactory.getInstance(this.getClass().getName(), "RM");
if ( persistence instanceof NullRmStatePersistence ) {
System.out.println("Cannot get viable RM Persistance isntance.");
return;
}
-
+
try {
- for ( int i = 0; i < 10; i++ ) {
- Map<String, Map<String, Object>> state = persistence.getAllMachines();
- for ( String node : state.keySet() ) {
- StringBuffer buf = new StringBuffer(node);
- buf.append(": ");
- Map<String, Object> st = state.get(node);
- for ( String k : st.keySet() ) {
- buf.append(k);
- buf.append("[");
- buf.append(st.get(k).toString());
- buf.append("] ");
- }
- System.out.println(buf.toString());
- }
- Thread.sleep(2000);
- }
+ Map<String, Map<String, Object>> state = persistence.getAllMachines();
+ System.out.println(toJson(state));
} catch ( Exception e ) {
e.printStackTrace();
} finally {
@@ -70,6 +60,45 @@ public class RmNodeState
}
}
+ String toJson(Map<String, Map<String, Object>> nodes)
+ {
+ StringBuffer buf = new StringBuffer("[");
+ for ( Map<String, Object> vals : nodes.values() ) {
+ buf.append("{");
+ for ( IDbProperty p : RmNodes.values() ) {
+ if ( p.isMeta() ) continue;
+ if ( p.isPrivate() ) continue;
+ buf.append("'");
+ buf.append(p.pname());
+ buf.append("'");
+ buf.append(":");
+ switch(p.type()) {
+ case String:
+ buf.append("'"); // must quote strings
+ buf.append(vals.get(p.columnName()));
+ buf.append("'");
+ break;
+ case Boolean:
+ boolean bv = (boolean) vals.get(p.columnName());
+ buf.append(bv ? "True" : "False"); // must pythonify the booleans
+ break;
+ case Integer:
+ case Long:
+ case Double
+: buf.append(vals.get(p.columnName()).toString());
+ break;
+ default:
+ // RmNodes doesn't use other types
+ break;
+ }
+ buf.append(",");
+ }
+ buf.append("},\n");
+ }
+ buf.append("]");
+ return buf.toString();
+ }
+
public static void main(String[] args)
{
if ( args.length != 1 ) {
Added: uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmQLoad.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmQLoad.java?rev=1719195&view=auto
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmQLoad.java (added)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmQLoad.java Thu Dec 10 20:13:26 2015
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+*/
+
+package org.apache.uima.ducc.database;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.uima.ducc.common.persistence.IDbProperty;
+import org.apache.uima.ducc.common.persistence.rm.IRmPersistence;
+import org.apache.uima.ducc.common.persistence.rm.IRmPersistence.RmLoad;
+import org.apache.uima.ducc.common.persistence.rm.NullRmStatePersistence;
+import org.apache.uima.ducc.common.persistence.rm.RmPersistenceFactory;
+import org.apache.uima.ducc.common.utils.DuccLogger;
+
+
+public class RmQLoad
+{
+ DuccLogger logger = DuccLogger.getLogger(RmQLoad.class, "State");
+ String dburl = null;
+
+ RmQLoad(String dburl)
+ {
+ this.dburl = dburl;
+ }
+
+ void run()
+ throws Exception
+ {
+ System.setProperty(DbManager.NOISE_PROPERTY, "false");
+ IRmPersistence persistence = RmPersistenceFactory.getInstance(this.getClass().getName(), "RM");
+ if ( persistence instanceof NullRmStatePersistence ) {
+ System.out.println("Cannot get viable RM Persistance isntance.");
+ return;
+ }
+
+ try {
+ List<Map<String, Object>> state = persistence.getLoad();
+ System.out.println(toJson(state));
+ } catch ( Exception e ) {
+ e.printStackTrace();
+ } finally {
+ // In "real life" you don't need to, and shouldn't, close the persistence until the process is ready to exit.
+ persistence.close();
+ }
+ }
+
+ String toJson(List<Map<String, Object>> nodes)
+ {
+ StringBuffer buf = new StringBuffer("[");
+ for ( Map<String, Object> vals : nodes ) {
+ buf.append("{");
+ for ( IDbProperty p : RmLoad.values() ) {
+ if ( p.isMeta() ) continue;
+ if ( p.isPrivate() ) continue;
+ buf.append("'");
+ buf.append(p.pname());
+ buf.append("'");
+ buf.append(":");
+ switch(p.type()) {
+ case String:
+ case UUID:
+ buf.append("'"); // must quote strings
+ buf.append(vals.get(p.columnName()));
+ buf.append("'");
+ break;
+ case Boolean:
+ boolean bv = (boolean) vals.get(p.columnName());
+ buf.append(bv ? "True" : "False"); // must pythonify the booleans
+ break;
+ case Integer:
+ case Long:
+ case Double:
+ buf.append(vals.get(p.columnName()).toString());
+ break;
+ default:
+ // RmNodes doesn't use other types
+ break;
+ }
+ buf.append(",");
+ }
+ buf.append("},\n");
+ }
+ buf.append("]");
+ return buf.toString();
+ }
+
+ public static void main(String[] args)
+ {
+ if ( args.length != 1 ) {
+ System.out.println("Usage: RmQLoad <dburl>");
+ System.exit(1);
+ }
+ System.setProperty(DbManager.URL_PROPERTY, args[0]);
+
+ RmQLoad rns = new RmQLoad(args[0]);
+ try {
+ rns.run();
+ } catch ( Exception e ) {
+ e.printStackTrace();
+ }
+ }
+}
Added: uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmShareState.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmShareState.java?rev=1719195&view=auto
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmShareState.java (added)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmShareState.java Thu Dec 10 20:13:26 2015
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+*/
+
+package org.apache.uima.ducc.database;
+
+import java.util.Map;
+
+import org.apache.uima.ducc.common.persistence.IDbProperty;
+import org.apache.uima.ducc.common.persistence.rm.IRmPersistence;
+import org.apache.uima.ducc.common.persistence.rm.IRmPersistence.RmShares;
+import org.apache.uima.ducc.common.persistence.rm.NullRmStatePersistence;
+import org.apache.uima.ducc.common.persistence.rm.RmPersistenceFactory;
+import org.apache.uima.ducc.common.utils.DuccLogger;
+
+
+public class RmShareState
+{
+ DuccLogger logger = DuccLogger.getLogger(RmShareState.class, "State");
+ String dburl = null;
+
+ RmShareState(String dburl)
+ {
+ this.dburl = dburl;
+ }
+
+ void run()
+ throws Exception
+ {
+ System.setProperty(DbManager.NOISE_PROPERTY, "false");
+ IRmPersistence persistence = RmPersistenceFactory.getInstance(this.getClass().getName(), "RM");
+ if ( persistence instanceof NullRmStatePersistence ) {
+ System.out.println("Cannot get viable RM Persistance isntance.");
+ return;
+ }
+
+ try {
+ Map<String, Map<String, Object>> state = persistence.getAllShares();
+ System.out.println(toJson(state));
+ } catch ( Exception e ) {
+ e.printStackTrace();
+ } finally {
+ // In "real life" you don't need to, and shouldn't, close the persistence until the process is ready to exit.
+ persistence.close();
+ }
+ }
+
+ String toJson(Map<String, Map<String, Object>> nodes)
+ {
+ StringBuffer buf = new StringBuffer("[");
+ for ( Map<String, Object> vals : nodes.values() ) {
+ buf.append("{");
+ for ( IDbProperty p : RmShares.values() ) {
+ if ( p.isMeta() ) continue;
+ if ( p.isPrivate() ) continue;
+ buf.append("'");
+ buf.append(p.pname());
+ buf.append("'");
+ buf.append(":");
+ switch(p.type()) {
+ case String:
+ case UUID:
+ buf.append("'"); // must quote strings
+ buf.append(vals.get(p.columnName()));
+ buf.append("'");
+ break;
+ case Boolean:
+ boolean bv = (boolean) vals.get(p.columnName());
+ buf.append(bv ? "True" : "False"); // must pythonify the booleans
+ break;
+ case Integer:
+ case Long:
+ case Double:
+ buf.append(vals.get(p.columnName()).toString());
+ break;
+ default:
+ // RmNodes doesn't use other types
+ break;
+ }
+ buf.append(",");
+ }
+ buf.append("},\n");
+ }
+ buf.append("]");
+ return buf.toString();
+ }
+
+ public static void main(String[] args)
+ {
+ if ( args.length != 1 ) {
+ System.out.println("Usage: RmShareState <dburl>");
+ System.exit(1);
+ }
+ System.setProperty(DbManager.URL_PROPERTY, args[0]);
+
+ RmShareState rns = new RmShareState(args[0]);
+ try {
+ rns.run();
+ } catch ( Exception e ) {
+ e.printStackTrace();
+ }
+ }
+}
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmStatePersistence.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmStatePersistence.java?rev=1719195&r1=1719194&r2=1719195&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmStatePersistence.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmStatePersistence.java Thu Dec 10 20:13:26 2015
@@ -203,7 +203,7 @@ public class RmStatePersistence
} catch ( Exception e ) {
logger.error(methodName, null, "Problem setting properties", e);
} finally {
- logger.info(methodName, null, "Total time to update properties on", System.currentTimeMillis() - now);
+ logger.debug(methodName, null, "Total time to update properties on", System.currentTimeMillis() - now);
}
@@ -285,7 +285,39 @@ public class RmStatePersistence
ret.put((String)mach.get(RmNodes.Name.pname()), mach);
}
return ret;
- }
+ }
+
+ public Map<String, Map<String, Object>> getAllShares()
+ throws Exception
+ {
+ //String methodName = "getAllShares";
+ Map<String, Map<String, Object>> ret = new HashMap<String, Map<String, Object>>();
+ String cql = "SELECT * FROM " + RM_SHARE_TABLE;
+ DbHandle h = dbManager.open();
+ ResultSet rs = h.execute(cql);
+ for ( Row r : rs ) {
+
+ Map<String, Object> share = DbUtil.getProperties(RmShares.values(), r);
+ String key = share.get(RmShares.Node.pname()) + ":" + share.get(RmShares.DuccDbid.pname()) + ":" + share.get(RmShares.JobId.pname());
+ ret.put(key, share);
+ }
+ return ret;
+ }
+
+ public List<Map<String, Object>> getLoad()
+ throws Exception
+ {
+ //String methodName = "getLoad";
+ List<Map<String, Object>> ret = new ArrayList<Map<String, Object>>();
+ String cql = "SELECT * FROM " + RM_LOAD_TABLE;
+ DbHandle h = dbManager.open();
+ ResultSet rs = h.execute(cql);
+ for ( Row r : rs ) {
+ Map<String, Object> job = DbUtil.getProperties(RmLoad.values(), r);
+ ret.add(job);
+ }
+ return ret;
+ }
public void addJob(IDbJob j)
throws Exception