You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by de...@apache.org on 2014/11/05 23:24:04 UTC
svn commit: r1636988 - in /uima/sandbox/uima-ducc/trunk/uima-ducc-user/src:
main/java/org/apache/uima/ducc/user/jd/
main/java/org/apache/uima/ducc/user/jd/iface/
test/java/org/apache/uima/ducc/user/jd/test/
Author: degenaro
Date: Wed Nov 5 22:24:04 2014
New Revision: 1636988
URL: http://svn.apache.org/r1636988
Log: (empty)
Added:
uima/sandbox/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/user/jd/iface/IJdUserErrorHandler.java (with props)
uima/sandbox/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/user/jd/iface/JdUserErrorHandler.java (with props)
Removed:
uima/sandbox/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/user/jd/iface/IJdUserErrorHandlerDirective.java
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/user/jd/CollectionReaderManager.java
uima/sandbox/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/user/jd/iface/JdUserCollectionReader.java
uima/sandbox/uima-ducc/trunk/uima-ducc-user/src/test/java/org/apache/uima/ducc/user/jd/test/TestSuite.java
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/user/jd/CollectionReaderManager.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/user/jd/CollectionReaderManager.java?rev=1636988&r1=1636987&r2=1636988&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/user/jd/CollectionReaderManager.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/user/jd/CollectionReaderManager.java Wed Nov 5 22:24:04 2014
@@ -25,6 +25,9 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.concurrent.atomic.AtomicInteger;
+import javax.xml.parsers.FactoryConfigurationError;
+import javax.xml.parsers.ParserConfigurationException;
+
import org.apache.uima.UIMAFramework;
import org.apache.uima.aae.UimaSerializer;
import org.apache.uima.cas.CAS;
@@ -46,6 +49,7 @@ import org.apache.uima.util.InvalidXMLEx
import org.apache.uima.util.Progress;
import org.apache.uima.util.XMLInputSource;
import org.apache.uima.util.XMLParser;
+import org.xml.sax.SAXException;
public class CollectionReaderManager {
@@ -147,7 +151,7 @@ public class CollectionReaderManager {
if(cr.hasNext()) {
CAS cas = cm.getEmptyCas();
cr.getNext(cas);
- String serializedCas = uimaSerializer.serializeCasToXmi(cas, xmiSerializationSharedData);
+ String serializedCas = serialize(cas);
String documentText = cas.getDocumentText();
retVal = new JdUserMetaCas(seqNo.incrementAndGet(), serializedCas, documentText);
cm.recycle(cas);
@@ -156,6 +160,23 @@ public class CollectionReaderManager {
return retVal;
}
+ public String serialize(CAS cas) throws Exception {
+ String serializedCas = uimaSerializer.serializeCasToXmi(cas, xmiSerializationSharedData);
+ return serializedCas;
+ }
+
+ public CAS deserialize(String serializedCas) throws ResourceInitializationException, FactoryConfigurationError, ParserConfigurationException, SAXException, IOException {
+ CAS cas = cm.getEmptyCas();
+ boolean lenient = true;
+ int mergePoint = -1;
+ uimaSerializer.deserializeCasFromXmi(serializedCas, cas, xmiSerializationSharedData, lenient, mergePoint);
+ return cas;
+ }
+
+ public void recycle(CAS cas) {
+ cm.recycle(cas);
+ }
+
private void setCrXml(String value) {
crXml = value;
}
Added: uima/sandbox/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/user/jd/iface/IJdUserErrorHandler.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/user/jd/iface/IJdUserErrorHandler.java?rev=1636988&view=auto
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/user/jd/iface/IJdUserErrorHandler.java (added)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/user/jd/iface/IJdUserErrorHandler.java Wed Nov 5 22:24:04 2014
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+*/
+package org.apache.uima.ducc.user.jd.iface;
+
+import java.util.Properties;
+
+import org.apache.uima.cas.CAS;
+
+public interface IJdUserErrorHandler {
+
+ public enum InitializeKey {
+ killJobLimit, // the number of work items to fail before the killing job
+ killWorkItemLimit, // the number of times a work item can be retried due to failure
+ killProcessFlag // whether or not the process of a failing work item should be killed
+ };
+
+ public void initialize(Properties properties);
+
+ public enum HandleKey {
+ killJobFlag, // request job kill
+ killJobReason, // reason for job kill (text)
+ killWorkItemFlag, // request work item kill
+ killWorkItemReason, // reason for work item kill (text)
+ killProcessFlag, // request process kill
+ killProcessReason, // reason for process kill (text)
+ };
+
+ public Properties handle(CAS cas, Exception e);
+}
Propchange: uima/sandbox/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/user/jd/iface/IJdUserErrorHandler.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/user/jd/iface/JdUserCollectionReader.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/user/jd/iface/JdUserCollectionReader.java?rev=1636988&r1=1636987&r2=1636988&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/user/jd/iface/JdUserCollectionReader.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/user/jd/iface/JdUserCollectionReader.java Wed Nov 5 22:24:04 2014
@@ -31,6 +31,10 @@ public class JdUserCollectionReader {
crm = new CollectionReaderManager(crXml, crCfg);
}
+ public CollectionReaderManager getCollectionReaderManager() {
+ return crm;
+ }
+
public JdUserMetaCas getJdUserMetaCas() throws JdUserException {
JdUserMetaCas retVal = crm.getJdUserMetaCas();
return retVal;
Added: uima/sandbox/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/user/jd/iface/JdUserErrorHandler.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/user/jd/iface/JdUserErrorHandler.java?rev=1636988&view=auto
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/user/jd/iface/JdUserErrorHandler.java (added)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/user/jd/iface/JdUserErrorHandler.java Wed Nov 5 22:24:04 2014
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+*/
+package org.apache.uima.ducc.user.jd.iface;
+
+import java.util.Properties;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.uima.cas.CAS;
+
+public class JdUserErrorHandler implements IJdUserErrorHandler {
+
+ private AtomicInteger exceptionLimitPerJob = new AtomicInteger(15);
+ private AtomicInteger exceptionLimitPerWorkItem = new AtomicInteger(0);
+
+ private AtomicInteger exceptionCounter = new AtomicInteger();
+
+ private ConcurrentHashMap<String,AtomicInteger> map = new ConcurrentHashMap<String,AtomicInteger>();
+
+ public JdUserErrorHandler() {
+ }
+
+ public JdUserErrorHandler(Properties properties) {
+ initialize(properties);
+ }
+
+ @Override
+ public void initialize(Properties properties) {
+ if(properties != null) {
+ initializeLimitPerJob(properties);
+ initializeLimitPerWorkItem(properties);
+ }
+ }
+
+ private void initializeLimitPerJob(Properties properties) {
+ try {
+ String key = InitializeKey.killJobLimit.name();
+ if(properties.containsKey(key)) {
+ String value = properties.getProperty(key);
+ int limit = Integer.parseInt(value);
+ if(limit > 0) {
+ exceptionLimitPerJob = new AtomicInteger(limit);
+ }
+ }
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ private void initializeLimitPerWorkItem(Properties properties) {
+ try {
+ String key = InitializeKey.killWorkItemLimit.name();
+ if(properties.containsKey(key)) {
+ String value = properties.getProperty(key);
+ int limit = Integer.parseInt(value);
+ if(limit > 0) {
+ exceptionLimitPerWorkItem = new AtomicInteger(limit);
+ }
+ }
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ @Override
+ public Properties handle(CAS cas, Exception exception) {
+ Properties properties = new Properties();
+ exceptionCounter.incrementAndGet();
+ handleKillJob(properties, cas, exception);
+ handleKillWorkItem(properties, cas, exception);
+ return properties;
+ }
+
+ private void killJob(Properties properties, String reason) {
+ String key;
+ String value;
+ key = HandleKey.killJobFlag.name();
+ value = Boolean.TRUE.toString();
+ properties.put(key, value);
+ key = HandleKey.killJobReason.name();
+ value = reason;
+ properties.put(key, value);
+ }
+
+ private void killProcess(Properties properties, String reason) {
+ String key;
+ String value;
+ key = HandleKey.killProcessFlag.name();
+ value = Boolean.TRUE.toString();
+ properties.put(key, value);
+ key = HandleKey.killProcessReason.name();
+ value = reason;
+ properties.put(key, value);
+ }
+
+ private void killWorkItem(Properties properties, String reason) {
+ String key;
+ String value;
+ key = HandleKey.killWorkItemFlag.name();
+ value = Boolean.TRUE.toString();
+ properties.put(key, value);
+ key = HandleKey.killWorkItemReason.name();
+ value = reason;
+ properties.put(key, value);
+ }
+
+ private void handleKillJob(Properties properties, CAS cas, Exception exception) {
+ try {
+ int counter = exceptionCounter.get();
+ int limit = exceptionLimitPerJob.get();
+ if(counter > limit) {
+ String reasonKJ = "errors="+counter+" "+"limit="+limit;
+ killJob(properties, reasonKJ);
+ }
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ private void handleKillWorkItem(Properties properties, CAS cas, Exception exception) {
+ try {
+ if(cas == null) {
+ String reasonKJ = "cas=null";
+ killJob(properties, reasonKJ);
+ String reasonKP = "kill process (if possible!)";
+ killProcess(properties, reasonKP);
+ }
+ else if(exception == null){
+ String reasonKJ = "exception=null";
+ killJob(properties, reasonKJ);
+ String reasonKP = "kill process (if possible!)";
+ killProcess(properties, reasonKP);
+ }
+ else {
+ String mapKey = cas.getDocumentText();
+ if(!map.containsKey(mapKey)) {
+ map.putIfAbsent(mapKey, new AtomicInteger(0));
+ }
+ AtomicInteger mapValue = map.get(mapKey);
+ int counter = mapValue.incrementAndGet();
+ int limit = exceptionLimitPerWorkItem.get();
+ if(counter > limit) {
+ String reasonKW = "errors="+counter+" "+"limit="+limit;
+ killWorkItem(properties, reasonKW);
+ }
+ String reasonKP = "kill process (if possible!)";
+ killProcess(properties, reasonKP);
+ }
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ }
+ }
+}
Propchange: uima/sandbox/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/user/jd/iface/JdUserErrorHandler.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-user/src/test/java/org/apache/uima/ducc/user/jd/test/TestSuite.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-user/src/test/java/org/apache/uima/ducc/user/jd/test/TestSuite.java?rev=1636988&r1=1636987&r2=1636988&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-user/src/test/java/org/apache/uima/ducc/user/jd/test/TestSuite.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-user/src/test/java/org/apache/uima/ducc/user/jd/test/TestSuite.java Wed Nov 5 22:24:04 2014
@@ -23,8 +23,14 @@ import static org.junit.Assert.fail;
import java.io.File;
import java.net.URL;
+import java.util.Properties;
+import org.apache.uima.cas.CAS;
+import org.apache.uima.ducc.user.jd.CollectionReaderManager;
+import org.apache.uima.ducc.user.jd.iface.IJdUserErrorHandler;
+import org.apache.uima.ducc.user.jd.iface.IJdUserErrorHandler.HandleKey;
import org.apache.uima.ducc.user.jd.iface.JdUserCollectionReader;
+import org.apache.uima.ducc.user.jd.iface.JdUserErrorHandler;
import org.apache.uima.ducc.user.jd.iface.JdUserException;
import org.apache.uima.ducc.user.jd.iface.JdUserMetaCas;
import org.junit.After;
@@ -170,4 +176,109 @@ public class TestSuite {
}
}
+ @Test
+ public void test06() {
+ try {
+ IJdUserErrorHandler eh = new JdUserErrorHandler();
+ CAS cas = null;
+ Exception e = null;
+ Properties properties = eh.handle(cas, e);
+ String key;
+ String value;
+ key = HandleKey.killJobFlag.name();
+ value = properties.getProperty(key);
+ assertTrue(value.equalsIgnoreCase("true"));
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ fail("Exception");
+ }
+ }
+
+ @Test
+ public void test07() {
+ try {
+ URL url = this.getClass().getResource("/CR100.xml");
+ File file = new File(url.getFile());
+ String crXml = file.getAbsolutePath();
+ debug(crXml);
+ String crCfg = null;
+ JdUserCollectionReader jdUserCollectionReader = new JdUserCollectionReader(crXml, crCfg);
+ int total = jdUserCollectionReader.getTotal();
+ assertTrue(total == 100);
+ JdUserMetaCas jdUserMetaCas = null;
+ jdUserMetaCas = jdUserCollectionReader.getJdUserMetaCas();
+ assertTrue(jdUserMetaCas != null);
+ String serializedCas = jdUserMetaCas.getSerializedCas();
+ assertTrue(serializedCas != null);
+ CollectionReaderManager crm = jdUserCollectionReader.getCollectionReaderManager();
+ CAS cas = crm.deserialize(serializedCas);
+ JdUserErrorHandler eh = new JdUserErrorHandler();
+ Exception exception = null;
+ Properties properties = eh.handle(cas, exception);
+ String key;
+ String value;
+ key = HandleKey.killJobFlag.name();
+ value = properties.getProperty(key);
+ assertTrue(value.equalsIgnoreCase("true"));
+ key = HandleKey.killJobReason.name();
+ value = properties.getProperty(key);
+ debug(key+": "+value);
+ key = HandleKey.killProcessFlag.name();
+ value = properties.getProperty(key);
+ assertTrue(value.equalsIgnoreCase("true"));
+ key = HandleKey.killProcessReason.name();
+ value = properties.getProperty(key);
+ debug(key+": "+value);
+ crm.recycle(cas);
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ fail("Exception");
+ }
+ }
+
+ @Test
+ public void test08() {
+ try {
+ URL url = this.getClass().getResource("/CR100.xml");
+ File file = new File(url.getFile());
+ String crXml = file.getAbsolutePath();
+ debug(crXml);
+ String crCfg = null;
+ JdUserCollectionReader jdUserCollectionReader = new JdUserCollectionReader(crXml, crCfg);
+ int total = jdUserCollectionReader.getTotal();
+ assertTrue(total == 100);
+ JdUserMetaCas jdUserMetaCas = null;
+ jdUserMetaCas = jdUserCollectionReader.getJdUserMetaCas();
+ assertTrue(jdUserMetaCas != null);
+ String serializedCas = jdUserMetaCas.getSerializedCas();
+ assertTrue(serializedCas != null);
+ CollectionReaderManager crm = jdUserCollectionReader.getCollectionReaderManager();
+ CAS cas = crm.deserialize(serializedCas);
+ JdUserErrorHandler eh = new JdUserErrorHandler();
+ Exception exception = new RuntimeException();
+ Properties properties = eh.handle(cas, exception);
+ String key;
+ String value;
+ key = HandleKey.killWorkItemFlag.name();
+ value = properties.getProperty(key);
+ assertTrue(value.equalsIgnoreCase("true"));
+ key = HandleKey.killWorkItemReason.name();
+ value = properties.getProperty(key);
+ debug(key+": "+value);
+ key = HandleKey.killProcessFlag.name();
+ value = properties.getProperty(key);
+ assertTrue(value.equalsIgnoreCase("true"));
+ key = HandleKey.killProcessReason.name();
+ value = properties.getProperty(key);
+ debug(key+": "+value);
+ assertTrue(!properties.containsKey(HandleKey.killJobFlag));
+ crm.recycle(cas);
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ fail("Exception");
+ }
+ }
}