You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by am...@apache.org on 2016/03/26 21:27:32 UTC
[01/13] incubator-asterixdb git commit: Improve Error Handling in
Local Directory Feeds
Repository: incubator-asterixdb
Updated Branches:
refs/heads/master cf39bc999 -> 121e1d9ab
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java
----------------------------------------------------------------------
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java b/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java
index 5eba66c..a629cdf 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java
+++ b/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java
@@ -432,7 +432,7 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
switch (feedDataSource.getSourceFeedType()) {
case PRIMARY:
switch (feedDataSource.getLocation()) {
- case SOURCE_FEED_COMPUTE_STAGE:
+ case COMPUTE:
if (feedDataSource.getFeed().getFeedId().equals(feedDataSource.getSourceFeedId())) {
locationArray = feedDataSource.getLocations();
} else {
@@ -453,9 +453,12 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
}
}
break;
- case SOURCE_FEED_INTAKE_STAGE:
+ case INTAKE:
locationArray = feedDataSource.getLocations();
break;
+ default:
+ throw new AsterixException(
+ "Can't subscibe to a FeedRuntime with type: " + feedDataSource.getLocation());
}
break;
case SECONDARY:
@@ -467,14 +470,17 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
if (activity.getDataverseName().equals(feedDataSource.getSourceFeedId().getDataverse())
&& activity.getFeedName().equals(feedDataSource.getSourceFeedId().getFeedName())) {
switch (feedDataSource.getLocation()) {
- case SOURCE_FEED_INTAKE_STAGE:
+ case INTAKE:
locations = activity.getFeedActivityDetails()
.get(FeedActivityDetails.COLLECT_LOCATIONS);
break;
- case SOURCE_FEED_COMPUTE_STAGE:
+ case COMPUTE:
locations = activity.getFeedActivityDetails()
.get(FeedActivityDetails.COMPUTE_LOCATIONS);
break;
+ default:
+ throw new AsterixException(
+ "Can't subscibe to a FeedRuntime with type: " + feedDataSource.getLocation());
}
break;
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FeedDataSource.java
----------------------------------------------------------------------
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FeedDataSource.java b/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FeedDataSource.java
index 9ce800d..bc70707 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FeedDataSource.java
+++ b/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FeedDataSource.java
@@ -21,7 +21,7 @@ package org.apache.asterix.metadata.declared;
import java.util.List;
import org.apache.asterix.external.feed.api.IFeed;
-import org.apache.asterix.external.feed.api.IFeedLifecycleListener.ConnectionLocation;
+import org.apache.asterix.external.feed.api.IFeedRuntime.FeedRuntimeType;
import org.apache.asterix.external.feed.management.FeedId;
import org.apache.asterix.metadata.entities.Feed;
import org.apache.asterix.om.types.IAType;
@@ -36,7 +36,7 @@ public class FeedDataSource extends AqlDataSource {
private final Feed feed;
private final FeedId sourceFeedId;
private final IFeed.FeedType sourceFeedType;
- private final ConnectionLocation location;
+ private final FeedRuntimeType location;
private final String targetDataset;
private final String[] locations;
private final int computeCardinality;
@@ -46,7 +46,7 @@ public class FeedDataSource extends AqlDataSource {
public FeedDataSource(Feed feed, AqlSourceId id, String targetDataset, IAType itemType, IAType metaType,
List<IAType> pkTypes, List<List<String>> partitioningKeys,
List<ScalarFunctionCallExpression> keyAccessExpression, FeedId sourceFeedId, IFeed.FeedType sourceFeedType,
- ConnectionLocation location, String[] locations, INodeDomain domain) throws AlgebricksException {
+ FeedRuntimeType location, String[] locations, INodeDomain domain) throws AlgebricksException {
super(id, itemType, metaType, AqlDataSourceType.FEED, domain);
this.feed = feed;
this.targetDataset = targetDataset;
@@ -77,7 +77,7 @@ public class FeedDataSource extends AqlDataSource {
return sourceFeedId;
}
- public ConnectionLocation getLocation() {
+ public FeedRuntimeType getLocation() {
return location;
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/serde/AStringSerializerDeserializer.java
----------------------------------------------------------------------
diff --git a/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/serde/AStringSerializerDeserializer.java b/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/serde/AStringSerializerDeserializer.java
index 8e2da0b..be86c48 100644
--- a/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/serde/AStringSerializerDeserializer.java
+++ b/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/serde/AStringSerializerDeserializer.java
@@ -25,7 +25,9 @@ import java.io.IOException;
import org.apache.asterix.om.base.AString;
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.util.string.UTF8StringReader;
import org.apache.hyracks.util.string.UTF8StringUtil;
+import org.apache.hyracks.util.string.UTF8StringWriter;
/**
* NOTE: this class will have objection creations in each serialize/deserialize
@@ -37,14 +39,23 @@ public class AStringSerializerDeserializer implements ISerializerDeserializer<AS
private static final long serialVersionUID = 1L;
public static final AStringSerializerDeserializer INSTANCE = new AStringSerializerDeserializer();
+ private final UTF8StringWriter utf8StringWriter;
+ private final UTF8StringReader utf8StringReader;
private AStringSerializerDeserializer() {
+ this.utf8StringWriter = null;
+ this.utf8StringReader = null;
+ }
+
+ public AStringSerializerDeserializer(UTF8StringWriter utf8StringWriter, UTF8StringReader utf8StringReader) {
+ this.utf8StringWriter = utf8StringWriter;
+ this.utf8StringReader = utf8StringReader;
}
@Override
public AString deserialize(DataInput in) throws HyracksDataException {
try {
- return new AString(UTF8StringUtil.readUTF8(in));
+ return new AString(UTF8StringUtil.readUTF8(in, utf8StringReader));
} catch (IOException e) {
throw new HyracksDataException(e);
}
@@ -53,7 +64,7 @@ public class AStringSerializerDeserializer implements ISerializerDeserializer<AS
@Override
public void serialize(AString instance, DataOutput out) throws HyracksDataException {
try {
- UTF8StringUtil.writeUTF8(instance.getStringValue(), out);
+ UTF8StringUtil.writeUTF8(instance.getStringValue(), out, utf8StringWriter);
} catch (IOException e) {
throw new HyracksDataException(e);
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-om/src/main/java/org/apache/asterix/formats/nontagged/AqlSerializerDeserializerProvider.java
----------------------------------------------------------------------
diff --git a/asterix-om/src/main/java/org/apache/asterix/formats/nontagged/AqlSerializerDeserializerProvider.java b/asterix-om/src/main/java/org/apache/asterix/formats/nontagged/AqlSerializerDeserializerProvider.java
index 99a6d2f..5b41535 100644
--- a/asterix-om/src/main/java/org/apache/asterix/formats/nontagged/AqlSerializerDeserializerProvider.java
+++ b/asterix-om/src/main/java/org/apache/asterix/formats/nontagged/AqlSerializerDeserializerProvider.java
@@ -64,6 +64,8 @@ import org.apache.hyracks.algebricks.data.ISerializerDeserializerProvider;
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.dataflow.common.data.marshalling.ShortSerializerDeserializer;
+import org.apache.hyracks.util.string.UTF8StringReader;
+import org.apache.hyracks.util.string.UTF8StringWriter;
public class AqlSerializerDeserializerProvider implements ISerializerDeserializerProvider, Serializable {
@@ -73,6 +75,13 @@ public class AqlSerializerDeserializerProvider implements ISerializerDeserialize
private AqlSerializerDeserializerProvider() {
}
+ // Can't be shared among threads <Stateful>
+ @SuppressWarnings("rawtypes")
+ public ISerializerDeserializer getAStringSerializerDeserializer() {
+ return addTag(new AStringSerializerDeserializer(new UTF8StringWriter(), new UTF8StringReader()),
+ ATypeTag.STRING);
+ }
+
@SuppressWarnings("rawtypes")
@Override
public ISerializerDeserializer getSerializerDeserializer(Object typeInfo) {
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-om/src/test/java/org/apache/asterix/dataflow/data/nontagged/serde/SimpleSerializerDeserializerTest.java
----------------------------------------------------------------------
diff --git a/asterix-om/src/test/java/org/apache/asterix/dataflow/data/nontagged/serde/SimpleSerializerDeserializerTest.java b/asterix-om/src/test/java/org/apache/asterix/dataflow/data/nontagged/serde/SimpleSerializerDeserializerTest.java
index eabd747..8a5270f 100644
--- a/asterix-om/src/test/java/org/apache/asterix/dataflow/data/nontagged/serde/SimpleSerializerDeserializerTest.java
+++ b/asterix-om/src/test/java/org/apache/asterix/dataflow/data/nontagged/serde/SimpleSerializerDeserializerTest.java
@@ -39,7 +39,8 @@ public class SimpleSerializerDeserializerTest {
String className = cl.getName();
if (className.endsWith("ARecordSerializerDeserializer")
|| className.endsWith("AUnorderedListSerializerDeserializer")
- || className.endsWith("AOrderedListSerializerDeserializer")) {
+ || className.endsWith("AOrderedListSerializerDeserializer")
+ || className.endsWith("AStringSerializerDeserializer")) {
// Serializer/Deserializer for complex types can have (immutable) states.
continue;
}
[04/13] incubator-asterixdb git commit: Improve Error Handling in
Local Directory Feeds
Posted by am...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/Value.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/Value.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/Value.java
index f0466b2..427e2a2 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/Value.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/Value.java
@@ -18,6 +18,7 @@
*/
package org.apache.asterix.external.classad;
+import org.apache.asterix.external.classad.object.pool.ClassAdObjectPool;
import org.apache.asterix.om.base.AMutableDouble;
import org.apache.asterix.om.base.AMutableInt32;
import org.apache.asterix.om.base.AMutableInt64;
@@ -30,9 +31,9 @@ public class Value {
private long longVal;
private double doubleVal;
private boolean boolVal;
- private ClassAdTime timeVal = new ClassAdTime();
- private ClassAd classadVal = new ClassAd();
- private ExprList listVal = new ExprList();
+ private final ClassAdTime timeVal;
+ private final ClassAd classadVal;
+ private final ExprList listVal;
private String stringVal;
/// Value types
@@ -164,7 +165,7 @@ public class Value {
}
}
- public boolean isClassAdValue(ClassAd ad) {
+ public boolean isClassAdValue(ClassAd ad) throws HyracksDataException {
if (valueType == ValueType.CLASSAD_VALUE) {
ad.setValue(classadVal);
return true;
@@ -265,49 +266,11 @@ public class Value {
1024.0 * 1024.0 * 1024.0 * 1024.0 // Terra
};
- public Value() {
+ public Value(ClassAdObjectPool objectPool) {
valueType = ValueType.UNDEFINED_VALUE;
- }
-
- public Value(Value value) throws HyracksDataException {
- valueType = value.valueType;
- switch (value.valueType) {
- case ABSOLUTE_TIME_VALUE:
- timeVal = new ClassAdTime(value.timeVal);
- break;
- case BOOLEAN_VALUE:
- this.boolVal = value.boolVal;
- break;
- case CLASSAD_VALUE:
- this.classadVal = new ClassAd(value.classadVal);
- break;
- case ERROR_VALUE:
- break;
- case INTEGER_VALUE:
- this.longVal = value.longVal;
- break;
- case LIST_VALUE:
- this.listVal = new ExprList(value.listVal);
- break;
- case NULL_VALUE:
- break;
- case REAL_VALUE:
- this.doubleVal = value.doubleVal;
- break;
- case RELATIVE_TIME_VALUE:
- this.timeVal = new ClassAdTime(value.timeVal);
- break;
- case SLIST_VALUE:
- this.listVal = new ExprList(value.listVal);
- break;
- case STRING_VALUE:
- this.stringVal = value.stringVal;
- break;
- case UNDEFINED_VALUE:
- break;
- default:
- break;
- }
+ this.timeVal = new ClassAdTime();
+ this.listVal = new ExprList(objectPool);
+ this.classadVal = new ClassAd(objectPool);
}
public void setValue(Value value) throws HyracksDataException {
@@ -351,14 +314,15 @@ public class Value {
}
}
- public void assign(Value value) throws HyracksDataException {
- if (this != value) {
- setValue(value);
- }
+ private void clear() {
+ valueType = ValueType.UNDEFINED_VALUE;
}
- public void clear() {
+ public void reset() {
valueType = ValueType.UNDEFINED_VALUE;
+ listVal.reset();
+ timeVal.reset();
+ classadVal.reset();
}
public void setRealValue(double r) {
@@ -409,7 +373,7 @@ public class Value {
listVal.setValue(expList);
}
- public void setClassAdValue(ClassAd ad) {
+ public void setClassAdValue(ClassAd ad) throws HyracksDataException {
clear();
valueType = ValueType.CLASSAD_VALUE;
classadVal.setValue(ad);
@@ -483,7 +447,7 @@ public class Value {
@Override
public String toString() {
- ClassAdUnParser unparser = new PrettyPrint();
+ ClassAdUnParser unparser = new PrettyPrint(new ClassAdObjectPool());
AMutableCharArrayString unparsed_text = new AMutableCharArrayString();
switch (valueType) {
case ABSOLUTE_TIME_VALUE:
@@ -521,14 +485,15 @@ public class Value {
return null;
}
- public static boolean convertValueToRealValue(Value value, Value realValue) throws HyracksDataException {
+ public static boolean convertValueToRealValue(Value value, Value realValue, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
boolean could_convert;
- AMutableCharArrayString buf = new AMutableCharArrayString();
+ AMutableCharArrayString buf = objectPool.strPool.get();
int endIndex;
char end;
- AMutableInt64 ivalue = new AMutableInt64(0);
- ClassAdTime atvalue = new ClassAdTime();
- MutableBoolean bvalue = new MutableBoolean();
+ AMutableInt64 ivalue = objectPool.int64Pool.get();
+ ClassAdTime atvalue = objectPool.classAdTimePool.get();
+ MutableBoolean bvalue = objectPool.boolPool.get();
double rvalue;
NumberFactor nf = NumberFactor.NO_FACTOR;
@@ -604,7 +569,7 @@ public class Value {
break;
case REAL_VALUE:
- realValue.copyFrom(value);
+ realValue.setValue(value);
could_convert = true;
break;
@@ -627,14 +592,15 @@ public class Value {
return could_convert;
}
- public static boolean convertValueToIntegerValue(Value value, Value integerValue) throws HyracksDataException {
+ public static boolean convertValueToIntegerValue(Value value, Value integerValue, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
boolean could_convert;
- AMutableCharArrayString buf = new AMutableCharArrayString();
+ AMutableCharArrayString buf = objectPool.strPool.get();
char end;
- AMutableInt64 ivalue = new AMutableInt64(0);
- AMutableDouble rtvalue = new AMutableDouble(0);
- ClassAdTime atvalue = new ClassAdTime();
- MutableBoolean bvalue = new MutableBoolean();
+ AMutableInt64 ivalue = objectPool.int64Pool.get();
+ AMutableDouble rtvalue = objectPool.doublePool.get();
+ ClassAdTime atvalue = objectPool.classAdTimePool.get();
+ MutableBoolean bvalue = objectPool.boolPool.get();
NumberFactor nf;
switch (value.getType()) {
@@ -700,7 +666,7 @@ public class Value {
break;
case INTEGER_VALUE:
- integerValue.copyFrom(value);
+ integerValue.setValue(value);
could_convert = true;
break;
@@ -729,51 +695,12 @@ public class Value {
return could_convert;
}
- public void copyFrom(Value val) throws HyracksDataException {
- clear();
- valueType = val.valueType;
- switch (val.valueType) {
- case STRING_VALUE:
- stringVal = val.stringVal;
- return;
-
- case BOOLEAN_VALUE:
- boolVal = val.boolVal;
- return;
-
- case INTEGER_VALUE:
- longVal = val.longVal;
- return;
-
- case REAL_VALUE:
-
- doubleVal = val.doubleVal;
- return;
- case UNDEFINED_VALUE:
- case ERROR_VALUE:
- return;
- case LIST_VALUE:
- case SLIST_VALUE:
- listVal.copyFrom(val.listVal);
- return;
- case CLASSAD_VALUE:
- classadVal.copyFrom(val.classadVal);
- return;
-
- case RELATIVE_TIME_VALUE:
- case ABSOLUTE_TIME_VALUE:
- timeVal.setValue(val.timeVal);
- return;
- default:
- setUndefinedValue();
- }
- }
-
- public static boolean convertValueToStringValue(Value value, Value stringValue) throws HyracksDataException {
+ public static boolean convertValueToStringValue(Value value, Value stringValue, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
boolean could_convert = false;
- ClassAdTime atvalue = new ClassAdTime();
- AMutableCharArrayString string_representation = new AMutableCharArrayString();
- ClassAdUnParser unparser = new PrettyPrint();
+ ClassAdTime atvalue = objectPool.classAdTimePool.get();
+ AMutableCharArrayString string_representation = objectPool.strPool.get();
+ ClassAdUnParser unparser = objectPool.prettyPrintPool.get();
switch (value.getType()) {
case UNDEFINED_VALUE:
@@ -787,7 +714,7 @@ public class Value {
break;
case STRING_VALUE:
- stringValue.copyFrom(value);
+ stringValue.setValue(value);
could_convert = true;
break;
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/AMutableCharArrayStringPool.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/AMutableCharArrayStringPool.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/AMutableCharArrayStringPool.java
new file mode 100644
index 0000000..b434522
--- /dev/null
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/AMutableCharArrayStringPool.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.classad.object.pool;
+
+import org.apache.asterix.external.classad.AMutableCharArrayString;
+
+public class AMutableCharArrayStringPool extends Pool<AMutableCharArrayString> {
+
+ @Override
+ public AMutableCharArrayString newInstance() {
+ return new AMutableCharArrayString();
+ }
+
+ @Override
+ protected void reset(AMutableCharArrayString str) {
+ str.reset();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/AttributeReferencePool.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/AttributeReferencePool.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/AttributeReferencePool.java
index d7bcfcc..9eb2c0c 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/AttributeReferencePool.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/AttributeReferencePool.java
@@ -22,9 +22,16 @@ import org.apache.asterix.external.classad.AttributeReference;
public class AttributeReferencePool extends Pool<AttributeReference> {
+ private final ClassAdObjectPool objectPool;
+
+ public AttributeReferencePool(ClassAdObjectPool objectPool) {
+ this.objectPool = objectPool;
+
+ }
+
@Override
public AttributeReference newInstance() {
- return new AttributeReference();
+ return new AttributeReference(objectPool);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/CaseInsensitiveStringPool.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/CaseInsensitiveStringPool.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/CaseInsensitiveStringPool.java
index 7e88961..f8a70f1 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/CaseInsensitiveStringPool.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/CaseInsensitiveStringPool.java
@@ -18,32 +18,10 @@
*/
package org.apache.asterix.external.classad.object.pool;
-import java.util.Stack;
-
import org.apache.asterix.external.classad.CaseInsensitiveString;
public class CaseInsensitiveStringPool extends Pool<CaseInsensitiveString> {
- protected Stack<CaseInsensitiveString> stock = new Stack<CaseInsensitiveString>();
-
- @Override
- public CaseInsensitiveString get() {
- if (!stock.isEmpty()) {
- return stock.pop();
- } else {
- return newInstance();
-
- }
- }
-
- @Override
- public void reset() {
- }
-
- public void put(CaseInsensitiveString aString) {
- stock.push(aString);
- }
-
@Override
public CaseInsensitiveString newInstance() {
return new CaseInsensitiveString();
@@ -51,5 +29,6 @@ public class CaseInsensitiveStringPool extends Pool<CaseInsensitiveString> {
@Override
protected void reset(CaseInsensitiveString obj) {
+ obj.set(null);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ClassAdObjectPool.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ClassAdObjectPool.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ClassAdObjectPool.java
new file mode 100644
index 0000000..c6e265f
--- /dev/null
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ClassAdObjectPool.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.classad.object.pool;
+
+import org.apache.asterix.external.classad.CaseInsensitiveString;
+import org.apache.asterix.external.classad.ExprTree;
+
+public class ClassAdObjectPool {
+ public final ExprHolderPool mutableExprPool;
+ public final TokenValuePool tokenValuePool;
+ public final ClassAdPool classAdPool;
+ public final ExprListPool exprListPool;
+ public final ValuePool valuePool;
+ public final LiteralPool literalPool;
+ public final BitSetPool bitSetPool;
+ public final OperationPool operationPool;
+ public final AttributeReferencePool attrRefPool;
+ public final ClassAdParserPool classAdParserPool;
+ public final AMutableCharArrayStringPool strPool;
+ public final FunctionCallPool funcPool;
+ public final MutableBooleanPool boolPool;
+ public final CaseInsensitiveStringPool caseInsensitiveStringPool;
+ public final StringArrayListPool stringArrayListPool;
+ public final Int32Pool int32Pool;
+ public final Int64Pool int64Pool;
+ public final ClassAdTimePool classAdTimePool;
+ public final DoublePool doublePool;
+ public final EvalStatePool evalStatePool;
+ public final HashMapPool<CaseInsensitiveString, ExprTree> strToExprPool;
+ public final PrettyPrintPool prettyPrintPool;
+ public final TreeSetPool<String> strSetPool;
+ public final MutableNumberFactorPool numFactorPool;
+
+ public ClassAdObjectPool() {
+ int32Pool = new Int32Pool();
+ int64Pool = new Int64Pool();
+ mutableExprPool = new ExprHolderPool(this);
+ tokenValuePool = new TokenValuePool();
+ classAdPool = new ClassAdPool(this);
+ exprListPool = new ExprListPool(this);
+ valuePool = new ValuePool(this);
+ literalPool = new LiteralPool(this);
+ bitSetPool = new BitSetPool();
+ operationPool = new OperationPool(this);
+ attrRefPool = new AttributeReferencePool(this);
+ strPool = new AMutableCharArrayStringPool();
+ funcPool = new FunctionCallPool(this);
+ boolPool = new MutableBooleanPool();
+ caseInsensitiveStringPool = new CaseInsensitiveStringPool();
+ stringArrayListPool = new StringArrayListPool();
+ classAdParserPool = new ClassAdParserPool(this);
+ classAdTimePool = new ClassAdTimePool();
+ doublePool = new DoublePool();
+ evalStatePool = new EvalStatePool(this);
+ strToExprPool = new HashMapPool<CaseInsensitiveString, ExprTree>();
+ prettyPrintPool = new PrettyPrintPool(this);
+ strSetPool = new TreeSetPool<String>();
+ numFactorPool = new MutableNumberFactorPool();
+ }
+
+ public void reset() {
+ mutableExprPool.reset();
+ tokenValuePool.reset();
+ classAdPool.reset();
+ exprListPool.reset();
+ valuePool.reset();
+ literalPool.reset();
+ bitSetPool.reset();
+ operationPool.reset();
+ attrRefPool.reset();
+ strPool.reset();
+ classAdParserPool.reset();
+ funcPool.reset();
+ boolPool.reset();
+ caseInsensitiveStringPool.reset();
+ stringArrayListPool.reset();
+ int32Pool.reset();
+ int64Pool.reset();
+ classAdTimePool.reset();
+ doublePool.reset();
+ evalStatePool.reset();
+ strToExprPool.reset();
+ prettyPrintPool.reset();
+ strSetPool.reset();
+ numFactorPool.reset();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ClassAdParserPool.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ClassAdParserPool.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ClassAdParserPool.java
new file mode 100644
index 0000000..4f85e7c
--- /dev/null
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ClassAdParserPool.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.classad.object.pool;
+
+import org.apache.asterix.external.library.ClassAdParser;
+
+public class ClassAdParserPool extends Pool<ClassAdParser> {
+ private final ClassAdObjectPool objectPool;
+
+ public ClassAdParserPool(ClassAdObjectPool objectPool) {
+ this.objectPool = objectPool;
+ }
+
+ @Override
+ public ClassAdParser newInstance() {
+ return new ClassAdParser(objectPool);
+ }
+
+ @Override
+ protected void reset(ClassAdParser obj) {
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ClassAdPool.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ClassAdPool.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ClassAdPool.java
index 40b8eb3..039e414 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ClassAdPool.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ClassAdPool.java
@@ -22,9 +22,15 @@ import org.apache.asterix.external.classad.ClassAd;
public class ClassAdPool extends Pool<ClassAd> {
+ private final ClassAdObjectPool objectPool;
+
+ public ClassAdPool(ClassAdObjectPool objectPool) {
+ this.objectPool = objectPool;
+ }
+
@Override
public ClassAd newInstance() {
- return new ClassAd(false, false);
+ return new ClassAd(objectPool);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ClassAdTimePool.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ClassAdTimePool.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ClassAdTimePool.java
new file mode 100644
index 0000000..06a2c2f
--- /dev/null
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ClassAdTimePool.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.classad.object.pool;
+
+import org.apache.asterix.external.classad.ClassAdTime;
+
+public class ClassAdTimePool extends Pool<ClassAdTime> {
+
+ @Override
+ public ClassAdTime newInstance() {
+ return new ClassAdTime();
+ }
+
+ @Override
+ protected void reset(ClassAdTime obj) {
+ obj.setCurrentAbsolute();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/DoublePool.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/DoublePool.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/DoublePool.java
new file mode 100644
index 0000000..52c1d26
--- /dev/null
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/DoublePool.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.classad.object.pool;
+
+import org.apache.asterix.om.base.AMutableDouble;
+
+public class DoublePool extends Pool<AMutableDouble> {
+
+ @Override
+ public AMutableDouble newInstance() {
+ return new AMutableDouble(0);
+ }
+
+ @Override
+ protected void reset(AMutableDouble obj) {
+ obj.setValue(0);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/EvalStatePool.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/EvalStatePool.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/EvalStatePool.java
new file mode 100644
index 0000000..a047cc9
--- /dev/null
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/EvalStatePool.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.classad.object.pool;
+
+import org.apache.asterix.external.classad.EvalState;
+
+public class EvalStatePool extends Pool<EvalState> {
+
+ public final ClassAdObjectPool objectPool;
+
+ public EvalStatePool(ClassAdObjectPool objectPool) {
+ this.objectPool = objectPool;
+ }
+
+ @Override
+ public EvalState newInstance() {
+ return new EvalState(objectPool);
+ }
+
+ @Override
+ protected void reset(EvalState obj) {
+ obj.reset();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ExprHolderPool.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ExprHolderPool.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ExprHolderPool.java
index b34e863..156f012 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ExprHolderPool.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ExprHolderPool.java
@@ -21,9 +21,16 @@ package org.apache.asterix.external.classad.object.pool;
import org.apache.asterix.external.classad.ExprTreeHolder;
public class ExprHolderPool extends Pool<ExprTreeHolder> {
+
+ private final ClassAdObjectPool objectPool;
+
+ public ExprHolderPool(ClassAdObjectPool objectPool) {
+ this.objectPool = objectPool;
+ }
+
@Override
public ExprTreeHolder newInstance() {
- return new ExprTreeHolder();
+ return new ExprTreeHolder(objectPool);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ExprListPool.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ExprListPool.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ExprListPool.java
index 1f2958a..66f5329 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ExprListPool.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ExprListPool.java
@@ -22,9 +22,15 @@ import org.apache.asterix.external.classad.ExprList;
public class ExprListPool extends Pool<ExprList> {
+ private final ClassAdObjectPool objectPool;
+
+ public ExprListPool(ClassAdObjectPool objectPool) {
+ this.objectPool = objectPool;
+ }
+
@Override
public ExprList newInstance() {
- return new ExprList();
+ return new ExprList(objectPool);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/FunctionCallPool.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/FunctionCallPool.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/FunctionCallPool.java
new file mode 100644
index 0000000..9281ed7
--- /dev/null
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/FunctionCallPool.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.classad.object.pool;
+
+import org.apache.asterix.external.classad.FunctionCall;
+
+public class FunctionCallPool extends Pool<FunctionCall> {
+
+ private final ClassAdObjectPool objectPool;
+
+ public FunctionCallPool(ClassAdObjectPool objectPool) {
+ this.objectPool = objectPool;
+ }
+
+ @Override
+ public FunctionCall newInstance() {
+ return new FunctionCall(objectPool);
+ }
+
+ @Override
+ protected void reset(FunctionCall obj) {
+ obj.reset();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/HashMapPool.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/HashMapPool.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/HashMapPool.java
new file mode 100644
index 0000000..b0365f3
--- /dev/null
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/HashMapPool.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.classad.object.pool;
+
+import java.util.HashMap;
+
+public class HashMapPool<K, V> extends Pool<HashMap<K, V>> {
+
+ @Override
+ public HashMap<K, V> newInstance() {
+ return new HashMap<K, V>();
+ }
+
+ @Override
+ protected void reset(HashMap<K, V> obj) {
+ obj.clear();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/Int32Pool.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/Int32Pool.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/Int32Pool.java
new file mode 100644
index 0000000..504f147
--- /dev/null
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/Int32Pool.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.classad.object.pool;
+
+import org.apache.asterix.om.base.AMutableInt32;
+
+public class Int32Pool extends Pool<AMutableInt32> {
+
+ @Override
+ public AMutableInt32 newInstance() {
+ return new AMutableInt32(0);
+ }
+
+ @Override
+ protected void reset(AMutableInt32 obj) {
+ obj.setValue(0);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/Int64Pool.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/Int64Pool.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/Int64Pool.java
new file mode 100644
index 0000000..f86b57a
--- /dev/null
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/Int64Pool.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.classad.object.pool;
+
+import org.apache.asterix.om.base.AMutableInt64;
+
+public class Int64Pool extends Pool<AMutableInt64> {
+
+ @Override
+ public AMutableInt64 newInstance() {
+ return new AMutableInt64(0);
+ }
+
+ @Override
+ protected void reset(AMutableInt64 obj) {
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/LiteralPool.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/LiteralPool.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/LiteralPool.java
index 23d8dae..2a85a4a 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/LiteralPool.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/LiteralPool.java
@@ -21,9 +21,15 @@ package org.apache.asterix.external.classad.object.pool;
import org.apache.asterix.external.classad.Literal;
public class LiteralPool extends Pool<Literal> {
+ private final ClassAdObjectPool objectPool;
+
+ public LiteralPool(ClassAdObjectPool objectPool) {
+ this.objectPool = objectPool;
+ }
+
@Override
public Literal newInstance() {
- return new Literal();
+ return new Literal(objectPool);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/MutableBooleanPool.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/MutableBooleanPool.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/MutableBooleanPool.java
new file mode 100644
index 0000000..8713272
--- /dev/null
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/MutableBooleanPool.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.classad.object.pool;
+
+import org.apache.commons.lang3.mutable.MutableBoolean;
+
+public class MutableBooleanPool extends Pool<MutableBoolean> {
+
+ @Override
+ public MutableBoolean newInstance() {
+ return new MutableBoolean();
+ }
+
+ @Override
+ protected void reset(MutableBoolean obj) {
+ obj.setValue(false);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/MutableNumberFactorPool.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/MutableNumberFactorPool.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/MutableNumberFactorPool.java
new file mode 100644
index 0000000..9d452d5
--- /dev/null
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/MutableNumberFactorPool.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.classad.object.pool;
+
+import org.apache.asterix.external.classad.AMutableNumberFactor;
+import org.apache.asterix.external.classad.Value.NumberFactor;
+
+public class MutableNumberFactorPool extends Pool<AMutableNumberFactor> {
+
+ @Override
+ public AMutableNumberFactor newInstance() {
+ return new AMutableNumberFactor();
+ }
+
+ @Override
+ protected void reset(AMutableNumberFactor obj) {
+ obj.setFactor(NumberFactor.NO_FACTOR);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/OperationPool.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/OperationPool.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/OperationPool.java
index 062d870..1607f6e 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/OperationPool.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/OperationPool.java
@@ -22,9 +22,15 @@ import org.apache.asterix.external.classad.Operation;
public class OperationPool extends Pool<Operation> {
+ private final ClassAdObjectPool objectPool;
+
+ public OperationPool(ClassAdObjectPool objectPool) {
+ this.objectPool = objectPool;
+ }
+
@Override
public Operation newInstance() {
- return new Operation();
+ return new Operation(objectPool);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/PrettyPrintPool.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/PrettyPrintPool.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/PrettyPrintPool.java
new file mode 100644
index 0000000..d38c5d6
--- /dev/null
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/PrettyPrintPool.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.classad.object.pool;
+
+import org.apache.asterix.external.classad.PrettyPrint;
+
+public class PrettyPrintPool extends Pool<PrettyPrint> {
+
+ private final ClassAdObjectPool objectPool;
+
+ public PrettyPrintPool(ClassAdObjectPool objectPool) {
+ this.objectPool = objectPool;
+ }
+
+ @Override
+ public PrettyPrint newInstance() {
+ return new PrettyPrint(objectPool);
+ }
+
+ @Override
+ protected void reset(PrettyPrint obj) {
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/StringArrayListPool.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/StringArrayListPool.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/StringArrayListPool.java
new file mode 100644
index 0000000..0ec687c
--- /dev/null
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/StringArrayListPool.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.classad.object.pool;
+
+import java.util.ArrayList;
+
+public class StringArrayListPool extends Pool<ArrayList<String>> {
+
+ @Override
+ public ArrayList<String> newInstance() {
+ return new ArrayList<String>();
+ }
+
+ @Override
+ protected void reset(ArrayList<String> obj) {
+ obj.clear();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/TreeSetPool.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/TreeSetPool.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/TreeSetPool.java
new file mode 100644
index 0000000..d09c170
--- /dev/null
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/TreeSetPool.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.classad.object.pool;
+
+import java.util.TreeSet;
+
+public class TreeSetPool<T> extends Pool<TreeSet<T>> {
+
+ @Override
+ public TreeSet<T> newInstance() {
+ return new TreeSet<T>();
+ }
+
+ @Override
+ protected void reset(TreeSet<T> obj) {
+ obj.clear();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ValuePool.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ValuePool.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ValuePool.java
index 698e947..9ad4424 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ValuePool.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/object/pool/ValuePool.java
@@ -21,13 +21,20 @@ package org.apache.asterix.external.classad.object.pool;
import org.apache.asterix.external.classad.Value;
public class ValuePool extends Pool<Value> {
+
+ private final ClassAdObjectPool objectPool;
+
+ public ValuePool(ClassAdObjectPool objectPool) {
+ this.objectPool = objectPool;
+ }
+
@Override
public Value newInstance() {
- return new Value();
+ return new Value(objectPool);
}
@Override
protected void reset(Value obj) {
- obj.clear();
+ obj.reset();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdFunctionalTest.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdFunctionalTest.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdFunctionalTest.java
index 09e8001..41aa7c7 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdFunctionalTest.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdFunctionalTest.java
@@ -20,6 +20,8 @@ package org.apache.asterix.external.classad.test;
import java.net.URLDecoder;
+import org.apache.asterix.external.classad.object.pool.ClassAdObjectPool;
+
import junit.framework.Test;
import junit.framework.TestCase;
import junit.framework.TestSuite;
@@ -48,9 +50,10 @@ public class ClassAdFunctionalTest extends TestCase {
public void testApp() {
try {
- String[] args = { "", "-d", "-v", URLDecoder.decode(getClass().getResource("/functional_tests.txt").getPath(), "UTF-8") };
- FunctionalTester.test(args.length, args);
- } catch (Exception e) {
+ String[] args = { "", "-d", "-v",
+ URLDecoder.decode(getClass().getResource("/functional_tests.txt").getPath(), "UTF-8") };
+ FunctionalTester.test(args.length, args, new ClassAdObjectPool());
+ } catch (Throwable e) {
e.printStackTrace();
assertTrue(false);
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdParserTest.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdParserTest.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdParserTest.java
index d65fecc..ab393ba 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdParserTest.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdParserTest.java
@@ -24,12 +24,13 @@ import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Paths;
+import org.apache.asterix.external.classad.ClassAd;
+import org.apache.asterix.external.classad.object.pool.ClassAdObjectPool;
+
import junit.framework.Test;
import junit.framework.TestCase;
import junit.framework.TestSuite;
-import org.apache.asterix.external.classad.ClassAd;
-
public class ClassAdParserTest extends TestCase {
/**
* Create the test case
@@ -54,18 +55,21 @@ public class ClassAdParserTest extends TestCase {
public void test() {
try {
// test here
- ClassAd pAd = new ClassAd();
+ ClassAdObjectPool objectPool = new ClassAdObjectPool();
+ ClassAd pAd = new ClassAd(objectPool);
String szInput;
String[] files = new String[] { "/testdata.txt" };
BufferedReader infile = null;
for (String path : files) {
- infile = Files.newBufferedReader(Paths.get(URLDecoder.decode(getClass().getResource(path).getPath(), "UTF-8")),
+ infile = Files.newBufferedReader(
+ Paths.get(URLDecoder.decode(getClass().getResource(path).getPath(), "UTF-8")),
StandardCharsets.UTF_8);
szInput = infile.readLine();
while (szInput != null) {
if (szInput.trim().length() == 0) {
// ClassAdChain completed
pAd.clear();
+ objectPool.reset();
} else if (!pAd.insert(szInput)) {
// Problem
System.out.println("BARFED ON:" + szInput);
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdToADMTest.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdToADMTest.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdToADMTest.java
index 57b0dcd..d822310 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdToADMTest.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdToADMTest.java
@@ -28,9 +28,12 @@ import org.apache.asterix.external.classad.CharArrayLexerSource;
import org.apache.asterix.external.classad.ClassAd;
import org.apache.asterix.external.classad.ExprTree;
import org.apache.asterix.external.classad.Value;
+import org.apache.asterix.external.classad.object.pool.ClassAdObjectPool;
import org.apache.asterix.external.input.record.reader.stream.SemiStructuredRecordReader;
-import org.apache.asterix.external.input.stream.LocalFileSystemInputStream;
+import org.apache.asterix.external.input.stream.LocalFSInputStream;
import org.apache.asterix.external.library.ClassAdParser;
+import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.dataflow.std.file.FileSplit;
import junit.framework.Test;
import junit.framework.TestCase;
@@ -60,17 +63,20 @@ public class ClassAdToADMTest extends TestCase {
public void test() {
try {
// test here
- ClassAd pAd = new ClassAd();
+ ClassAdObjectPool objectPool = new ClassAdObjectPool();
+ ClassAd pAd = new ClassAd(objectPool);
String[] files = new String[] { "/jobads.txt" };
- ClassAdParser parser = new ClassAdParser(null, false, true, false, null, null, null);
+ ClassAdParser parser = new ClassAdParser(objectPool);
CharArrayLexerSource lexerSource = new CharArrayLexerSource();
for (String path : files) {
- LocalFileSystemInputStream in = new LocalFileSystemInputStream(
- Paths.get(getClass().getResource(path).toURI()), null, false);
+ LocalFSInputStream in = new LocalFSInputStream(
+ new FileSplit[] { new FileSplit("",
+ new FileReference(Paths.get(getClass().getResource(path).toURI()).toFile())) },
+ null, null, 0, null, false);
SemiStructuredRecordReader recordReader = new SemiStructuredRecordReader(in, null, "[", "]");
- Value val = new Value();
+ Value val = new Value(objectPool);
while (recordReader.hasNext()) {
- val.clear();
+ val.reset();
IRawRecord<char[]> record = recordReader.next();
lexerSource.setNewSource(record.get());
parser.setLexerSource(lexerSource);
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdUnitTest.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdUnitTest.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdUnitTest.java
index b9f32f5..32ce485 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdUnitTest.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdUnitTest.java
@@ -18,7 +18,7 @@
*/
package org.apache.asterix.external.classad.test;
-import java.io.IOException;
+import org.apache.asterix.external.classad.object.pool.ClassAdObjectPool;
import junit.framework.Test;
import junit.framework.TestCase;
@@ -51,8 +51,9 @@ public class ClassAdUnitTest extends TestCase {
public void test() {
String[] args = { "", "-d", "-vv", "-all" };
try {
- ClassAdUnitTester.test(args.length, args);
- } catch (IOException e) {
+ ClassAdUnitTester.test(args.length, args, new ClassAdObjectPool());
+ } catch (Throwable e) {
+ e.printStackTrace();
assertTrue(false);
}
assertTrue(true);
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdUnitTester.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdUnitTester.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdUnitTester.java
index cec9299..e3c9bf7 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdUnitTester.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdUnitTester.java
@@ -32,6 +32,7 @@ import org.apache.asterix.external.classad.Literal;
import org.apache.asterix.external.classad.Util;
import org.apache.asterix.external.classad.Value;
import org.apache.asterix.external.classad.Value.ValueType;
+import org.apache.asterix.external.classad.object.pool.ClassAdObjectPool;
import org.apache.asterix.external.library.ClassAdParser;
import org.apache.asterix.om.base.AMutableDouble;
import org.apache.asterix.om.base.AMutableInt32;
@@ -200,7 +201,7 @@ public class ClassAdUnitTester {
*
* @throws IOException
*********************************************************************/
- public static boolean test(int argc, String[] argv) throws IOException {
+ public static boolean test(int argc, String[] argv, ClassAdObjectPool objectPool) throws IOException {
AMutableInt32 numberOfErrors = new AMutableInt32(0);
AMutableInt32 numberOfTests = new AMutableInt32(0);
boolean have_errors;
@@ -213,17 +214,22 @@ public class ClassAdUnitTester {
/* ----- Run tests ----- */
if (parameters.checkAll || parameters.checkParsing) {
- testParsing(parameters, results);
+ testParsing(parameters, results, objectPool);
}
if (parameters.checkAll || parameters.checkClassad) {
- testClassad(parameters, results);
+ try {
+ testClassad(parameters, results, objectPool);
+ } catch (Throwable th) {
+ th.printStackTrace();
+ throw th;
+ }
}
if (parameters.checkAll || parameters.checkExprlist) {
- testExprList(parameters, results);
+ testExprList(parameters, results, objectPool);
}
if (parameters.checkAll || parameters.checkValue) {
- testValue(parameters, results);
+ testValue(parameters, results, objectPool);
}
if (parameters.checkAll || parameters.checkLiteral) {
}
@@ -255,10 +261,11 @@ public class ClassAdUnitTester {
}
public static void test(String name, boolean test, String testLine, Results results) {
- if (test)
+ if (test) {
results.AddSuccessfulTest(name, testLine);
- else
+ } else {
results.AddFailedTest(name, testLine);
+ }
}
public static void test(String name, boolean test, Results results) {
@@ -272,8 +279,9 @@ public class ClassAdUnitTester {
*
* @throws IOException
*********************************************************************/
- public static void testParsing(Parameters parameters, Results results) throws IOException {
- ClassAdParser parser = new ClassAdParser(null, false, true, false, null, null, null);
+ public static void testParsing(Parameters parameters, Results results, ClassAdObjectPool objectPool)
+ throws IOException {
+ ClassAdParser parser = new ClassAdParser(objectPool);
ExprTree tree;
// My goal is to ensure that these expressions don't crash
@@ -312,23 +320,25 @@ public class ClassAdUnitTester {
/*********************************************************************
* Function: test_classad
* Purpose: Test the ClassAd class.
+ * @param objectPool
*
* @throws IOException
*********************************************************************/
- public static void testClassad(Parameters parameters, Results results) throws IOException {
- ClassAdParser parser = new ClassAdParser(null, false, true, false, null, null, null);
+ public static void testClassad(Parameters parameters, Results results, ClassAdObjectPool objectPool)
+ throws IOException {
+ ClassAdParser parser = new ClassAdParser(objectPool);
boolean haveAttribute;
boolean success;
System.out.println("Testing the ClassAd class...");
String input_basic = "[ A = 3; B = 4.0; C = \"babyzilla\"; D = true; E = {1}; F = [ AA = 3; ]; G =\"deleteme\";]";
- ClassAd basic = new ClassAd();
+ ClassAd basic = new ClassAd(objectPool);
AMutableInt64 i = new AMutableInt64(0);
MutableBoolean b = new MutableBoolean();
AMutableDouble r = new AMutableDouble(0);
AMutableCharArrayString s = new AMutableCharArrayString();
- ClassAd c = new ClassAd();
+ ClassAd c = new ClassAd(objectPool);
// ExprList *l;
basic = parser.parseClassAd(input_basic);
@@ -426,8 +436,8 @@ public class ClassAdUnitTester {
/* ----- Test Parsing multiple ClassAds ----- */
String twoClassads = "[ a = 3; ][ b = 4; ]";
- ClassAd classad1 = new ClassAd();
- ClassAd classad2 = new ClassAd();
+ ClassAd classad1 = new ClassAd(objectPool);
+ ClassAd classad2 = new ClassAd(objectPool);
AMutableInt32 offset = new AMutableInt32(0);
parser.parseClassAd(twoClassads, classad1, offset);
@@ -439,7 +449,7 @@ public class ClassAdUnitTester {
/* ----- Test chained ClassAds ----- */
// classad1 and classad2 from above test are used.
- ClassAd classad3 = new ClassAd();
+ ClassAd classad3 = new ClassAd(objectPool);
classad1.chainToAd(classad2);
test("classad1's parent is classad2", classad1.getChainedParentAd().equals(classad2), "Test chained ClassAds 1",
@@ -488,7 +498,8 @@ public class ClassAdUnitTester {
*
* @throws IOException
*********************************************************************/
- public static void testExprList(Parameters parameters, Results results) throws IOException {
+ public static void testExprList(Parameters parameters, Results results, ClassAdObjectPool objectPool)
+ throws IOException {
System.out.println("Testing the ExprList class...");
Literal literal10;
@@ -506,17 +517,17 @@ public class ClassAdUnitTester {
ExprList list2Copy;
/* ----- Setup Literals, the vectors, then ExprLists ----- */
- literal10 = Literal.createReal("1.0");
- literal20 = Literal.createReal("2.0");
- literal21 = Literal.createReal("2.1");
+ literal10 = Literal.createReal("1.0", objectPool);
+ literal20 = Literal.createReal("2.0", objectPool);
+ literal21 = Literal.createReal("2.1", objectPool);
vector1.add(literal10);
vector2.add(literal20);
vector2.add(literal21);
- list0 = new ExprList();
- list1 = new ExprList(vector1);
- list2 = new ExprList(vector2);
+ list0 = new ExprList(objectPool);
+ list1 = new ExprList(vector1, objectPool);
+ list2 = new ExprList(vector2, objectPool);
/* ----- Did the lists get made? ----- */
test("Made list 0", (list0 != null), "Did the lists get made? 0", results);
@@ -558,7 +569,7 @@ public class ClassAdUnitTester {
/* ----- Test adding and deleting from a list ----- */
Literal add;
- add = Literal.createReal("2.2");
+ add = Literal.createReal("2.2", objectPool);
if (list2Copy != null) {
list2Copy.insert(add);
@@ -574,11 +585,11 @@ public class ClassAdUnitTester {
/* ----- Test an ExprList bug that Nate Mueller found ----- */
ClassAd classad;
- ClassAdParser parser = new ClassAdParser(null, false, true, false, null, null, null);
+ ClassAdParser parser = new ClassAdParser(objectPool);
MutableBoolean b = new MutableBoolean();
boolean haveAttribute;
boolean canEvaluate;
- Value value = new Value();
+ Value value = new Value(objectPool);
String listClassadText = "[foo = 3; have_foo = member(foo, {1, 2, 3});]";
classad = parser.parseClassAd(listClassadText);
@@ -598,8 +609,9 @@ public class ClassAdUnitTester {
*
* @throws HyracksDataException
*********************************************************************/
- public static void testValue(Parameters parameters, Results results) throws HyracksDataException {
- Value v = new Value();
+ public static void testValue(Parameters parameters, Results results, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ Value v = new Value(objectPool);
boolean isExpectedType;
System.out.println("Testing the Value class...");
test("New value is undefined", (v.isUndefinedValue()), "test_value 1", results);
@@ -659,25 +671,25 @@ public class ClassAdUnitTester {
test("Relative time is 10", (10 == rt.getRelativeTime()), results);
test("GetType gives RELATIVE_TIME_VALUE", (v.getType() == ValueType.RELATIVE_TIME_VALUE), results);
- ExprList l = new ExprList();
- ExprList ll = new ExprList();
+ ExprList l = new ExprList(objectPool);
+ ExprList ll = new ExprList(objectPool);
v.setListValue(l);
isExpectedType = v.isListValue(ll);
test("Value is list value", isExpectedType, results);
test("List value is correct", l.equals(ll), results);
test("GetType gives LIST_VALUE", (v.getType() == ValueType.LIST_VALUE), results);
- ExprList sl = new ExprList(true);
- ll = new ExprList(true);
+ ExprList sl = new ExprList(true, objectPool);
+ ll = new ExprList(true, objectPool);
v.setListValue(sl);
isExpectedType = v.isListValue(ll);
test("Value is list value", isExpectedType, results);
test("List value is correct", sl.equals(ll), results);
test("GetType gives SLIST_VALUE", (v.getType() == ValueType.SLIST_VALUE), results);
- ClassAd c = new ClassAd();
+ ClassAd c = new ClassAd(objectPool);
c.insertAttr("test_int", 10);
- ClassAd cc = new ClassAd();
+ ClassAd cc = new ClassAd(objectPool);
v.setClassAdValue(c);
isExpectedType = v.isClassAdValue(cc);
test("Value is ClassAd value", isExpectedType, results);
[06/13] incubator-asterixdb git commit: Improve Error Handling in
Local Directory Feeds
Posted by am...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/ClassAd.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/ClassAd.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/ClassAd.java
index df6ae0d..4f27cd8 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/ClassAd.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/ClassAd.java
@@ -27,7 +27,7 @@ import java.util.Map.Entry;
import java.util.TreeSet;
import org.apache.asterix.external.classad.Value.NumberFactor;
-import org.apache.asterix.external.classad.object.pool.CaseInsensitiveStringPool;
+import org.apache.asterix.external.classad.object.pool.ClassAdObjectPool;
import org.apache.asterix.external.library.ClassAdParser;
import org.apache.asterix.om.base.AMutableDouble;
import org.apache.asterix.om.base.AMutableInt32;
@@ -97,7 +97,6 @@ public class ClassAd extends ExprTree {
private static final int CLASSAD_VERSION_PATCH = 0;
private static final String CLASSAD_VERSION = "8.0.0";
public static final ArrayList<String> specialAttrNames = new ArrayList<String>();
- private final CaseInsensitiveStringPool StringPool = new CaseInsensitiveStringPool();
static {
specialAttrNames.add(ATTR_TOPLEVEL);
@@ -106,42 +105,22 @@ public class ClassAd extends ExprTree {
specialAttrNames.add(ATTR_PARENT);
}
- public static final FunctionCall curr_time_expr = FunctionCall.createFunctionCall("time", new ExprList());
+ public static final FunctionCall curr_time_expr = FunctionCall.createFunctionCall("time",
+ new ExprList(new ClassAdObjectPool()), new ClassAdObjectPool());
private ClassAd alternateScope;
- //private boolean doDirtyTracking;
- private Map<CaseInsensitiveString, ExprTree> attrList = new HashMap<CaseInsensitiveString, ExprTree>();
+ private final Map<CaseInsensitiveString, ExprTree> attrList;
private ClassAd chainedParentAd;
private ClassAdParser parser = null;
- private ClassAd newAd = null;
+ private ClassAd newAd;
/*
* Constructors
*/
- public ClassAd() {
- chainedParentAd = null;
- alternateScope = null;
- newAd = new ClassAd(false, false);
- parser = new ClassAdParser(null, false, true, false, null, null, null);
- }
-
- public ClassAd(boolean initializeParser, boolean initializeNewAd) {
- chainedParentAd = null;
- alternateScope = null;
- if (initializeNewAd) {
- newAd = new ClassAd(false, false);
- }
- if (initializeParser) {
- parser = new ClassAdParser(null, false, true, false, null, null, null);
- }
- }
-
- public ClassAd(ClassAd ad) throws HyracksDataException {
- if (ad == null) {
- clear();
- } else {
- copyFrom(ad);
- }
+ public ClassAd(ClassAdObjectPool objectPool) {
+ super(objectPool);
+ parser = new ClassAdParser(this.objectPool);
+ attrList = new HashMap<CaseInsensitiveString, ExprTree>();
}
@Override
@@ -165,10 +144,6 @@ public class ClassAd extends ExprTree {
return attrList;
}
- public void setAttrList(Map<CaseInsensitiveString, ExprTree> attrList) {
- this.attrList = attrList;
- }
-
public void classAdLibraryVersion(AMutableInt32 major, AMutableInt32 minor, AMutableInt32 patch) {
major.setValue(CLASSAD_VERSION_MAJOR);
minor.setValue(CLASSAD_VERSION_MINOR);
@@ -187,21 +162,6 @@ public class ClassAd extends ExprTree {
return curr_time_expr;
}
- //public TreeSet<CaseInsensitiveString> dirtyAttrList = new TreeSet<CaseInsensitiveString>();
-
- /*
- * Reference is an ordered set of Strings <The ordering uses less than ignore case>. Example
- * below
- * TreeSet<String> references = new TreeSet<String>(
- * new Comparator<String>(){
- * public int compare(String o1, String o2) {
- * return o1.compareToIgnoreCase(o2);
- * }
- * });
- *
- * // PortReferences is a Map<ClassAd,OrderedSet<Strings>>
- */
-
public boolean copyFrom(ClassAd ad) throws HyracksDataException {
boolean succeeded = true;
@@ -211,38 +171,26 @@ public class ClassAd extends ExprTree {
clear();
// copy scoping attributes
super.copyFrom(ad);
- if (ad.chainedParentAd != null) {
- if (chainedParentAd == null) {
- chainedParentAd = new ClassAd();
- }
- chainedParentAd.setValue(ad.chainedParentAd);
- }
- if (ad.alternateScope != null) {
- if (alternateScope == null) {
- alternateScope = new ClassAd();
- }
- alternateScope.setValue(ad.alternateScope);
- }
- //this.doDirtyTracking = false;
+ chainedParentAd = ad.chainedParentAd;
+ alternateScope = ad.alternateScope;
for (Entry<CaseInsensitiveString, ExprTree> attr : ad.attrList.entrySet()) {
- ExprTree tree = attr.getValue().copy();
- attrList.put(attr.getKey(), tree);
- // if (ad.doDirtyTracking && ad.IsAttributeDirty(attr.getKey())) {
- // dirtyAttrList.add(attr.getKey());
- //}
+ ExprTree tree = objectPool.mutableExprPool.get();
+ CaseInsensitiveString key = objectPool.caseInsensitiveStringPool.get();
+ tree.copyFrom(attr.getValue());
+ key.set(attr.getKey().get());
+ attrList.put(key, tree);
}
- //doDirtyTracking = ad.doDirtyTracking;
}
return succeeded;
}
public boolean update(ClassAd ad) throws HyracksDataException {
for (Entry<CaseInsensitiveString, ExprTree> attr : ad.attrList.entrySet()) {
- ExprTree tree = attr.getValue().copy();
- attrList.put(attr.getKey(), tree);
- // if (ad.doDirtyTracking && ad.IsAttributeDirty(attr.getKey())) {
- // dirtyAttrList.add(attr.getKey());
- //}
+ ExprTree tree = objectPool.mutableExprPool.get();
+ CaseInsensitiveString key = objectPool.caseInsensitiveStringPool.get();
+ tree.copyFrom(attr.getValue());
+ key.set(attr.getKey().get());
+ attrList.put(key, tree);
}
return true;
}
@@ -306,9 +254,6 @@ public class ClassAd extends ExprTree {
if (alternateScope != null) {
alternateScope.clear();
}
- if (parser != null) {
- parser.reset();
- }
}
public void unchain() {
@@ -317,18 +262,24 @@ public class ClassAd extends ExprTree {
}
}
- public void getComponents(Map<CaseInsensitiveString, ExprTree> attrs) {
+ public void getComponents(Map<CaseInsensitiveString, ExprTree> attrs, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
attrs.clear();
for (Entry<CaseInsensitiveString, ExprTree> attr : this.attrList.entrySet()) {
- attrs.put(attr.getKey(), attr.getValue());
+ ExprTree tree = objectPool.mutableExprPool.get();
+ CaseInsensitiveString key = objectPool.caseInsensitiveStringPool.get();
+ tree.copyFrom(attr.getValue());
+ key.set(attr.getKey().get());
+ attrs.put(key, tree);
}
}
public ClassAd privateGetDeepScope(ExprTree tree) throws HyracksDataException {
- ClassAd scope = new ClassAd();
- Value val = new Value();
- if (tree == null)
+ if (tree == null) {
return (null);
+ }
+ ClassAd scope = objectPool.classAdPool.get();
+ Value val = objectPool.valuePool.get();
tree.setParentScope(this);
if (!tree.publicEvaluate(val) || !val.isClassAdValue(scope)) {
return (null);
@@ -339,9 +290,9 @@ public class ClassAd extends ExprTree {
// --- begin integer attribute insertion ----
public boolean insertAttr(String name, int value, NumberFactor f) throws HyracksDataException {
ExprTree plit;
- Value val = new Value();
+ Value val = objectPool.valuePool.get();
val.setIntegerValue(value);
- plit = Literal.createLiteral(val, f);
+ plit = Literal.createLiteral(val, f, objectPool);
return insert(name, plit);
}
@@ -351,10 +302,10 @@ public class ClassAd extends ExprTree {
public boolean insertAttr(String name, long value, NumberFactor f) throws HyracksDataException {
ExprTree plit;
- Value val = new Value();
+ Value val = objectPool.valuePool.get();
val.setIntegerValue(value);
- plit = Literal.createLiteral(val, f);
+ plit = Literal.createLiteral(val, f, objectPool);
return (insert(name, plit));
}
@@ -365,16 +316,18 @@ public class ClassAd extends ExprTree {
public boolean deepInsertAttr(ExprTree scopeExpr, String name, int value, NumberFactor f)
throws HyracksDataException {
ClassAd ad = privateGetDeepScope(scopeExpr);
- if (ad == null)
+ if (ad == null) {
return (false);
+ }
return (ad.insertAttr(name, value, f));
}
public boolean deepInsertAttr(ExprTree scopeExpr, String name, long value, NumberFactor f)
throws HyracksDataException {
ClassAd ad = privateGetDeepScope(scopeExpr);
- if (ad == null)
+ if (ad == null) {
return (false);
+ }
return (ad.insertAttr(name, value, f));
}
@@ -383,17 +336,18 @@ public class ClassAd extends ExprTree {
// --- begin real attribute insertion ---
public boolean insertAttr(String name, double value, NumberFactor f) throws HyracksDataException {
ExprTree plit;
- Value val = new Value();
+ Value val = objectPool.valuePool.get();
val.setRealValue(value);
- plit = Literal.createLiteral(val, f);
+ plit = Literal.createLiteral(val, f, objectPool);
return (insert(name, plit));
}
public boolean deepInsertAttr(ExprTree scopeExpr, String name, double value, NumberFactor f)
throws HyracksDataException {
ClassAd ad = privateGetDeepScope(scopeExpr);
- if (ad == null)
+ if (ad == null) {
return (false);
+ }
return (ad.insertAttr(name, value, f));
}
@@ -402,16 +356,17 @@ public class ClassAd extends ExprTree {
// --- begin boolean attribute insertion
public boolean insertAttr(String name, boolean value) throws HyracksDataException {
ExprTree plit;
- Value val = new Value();
+ Value val = objectPool.valuePool.get();
val.setBooleanValue(value);
- plit = Literal.createLiteral(val);
+ plit = Literal.createLiteral(val, objectPool);
return (insert(name, plit));
}
public boolean deepInsertAttr(ExprTree scopeExpr, String name, boolean value) throws HyracksDataException {
ClassAd ad = privateGetDeepScope(scopeExpr);
- if (ad == null)
+ if (ad == null) {
return (false);
+ }
return (ad.insertAttr(name, value));
}
@@ -420,33 +375,35 @@ public class ClassAd extends ExprTree {
// --- begin string attribute insertion
public boolean insertAttr(String name, AMutableCharArrayString value) throws HyracksDataException {
ExprTree plit;
- Value val = new Value();
+ Value val = objectPool.valuePool.get();
val.setStringValue(value);
- plit = Literal.createLiteral(val);
+ plit = Literal.createLiteral(val, objectPool);
return (insert(name, plit));
}
public boolean deepInsertAttr(ExprTree scopeExpr, String name, AMutableCharArrayString value)
throws HyracksDataException {
ClassAd ad = privateGetDeepScope(scopeExpr);
- if (ad == null)
+ if (ad == null) {
return (false);
+ }
return (ad.insertAttr(name, value));
}
public boolean insertAttr(String name, String value) throws HyracksDataException {
ExprTree plit;
- Value val = new Value();
+ Value val = objectPool.valuePool.get();
val.setStringValue(value);
- plit = Literal.createLiteral(val);
+ plit = Literal.createLiteral(val, objectPool);
return (insert(name, plit));
}
public boolean deepInsertAttr(ExprTree scopeExpr, String name, String value) throws HyracksDataException {
ClassAd ad = privateGetDeepScope(scopeExpr);
- if (ad == null)
+ if (ad == null) {
return (false);
+ }
return (ad.insertAttr(name, value));
}
@@ -498,7 +455,7 @@ public class ClassAd extends ExprTree {
// cache doesn't already have an entry for this name:value, so add
// it to the cache now.
if (newTree.getKind() != NodeKind.LITERAL_NODE) {
- Literal lit = parser.getLiteral();
+ Literal lit = objectPool.literalPool.get();
lit.getValue().setStringValue(szValue);
bRet = insert(name, lit, false);
} else {
@@ -511,7 +468,8 @@ public class ClassAd extends ExprTree {
}
public boolean insert(String attrName, ExprTree expr) throws HyracksDataException {
- boolean result = insert(attrName, expr.isTreeHolder() ? ((ExprTreeHolder) expr).getInnerTree() : expr, false);
+ ExprTree tree = expr.copy();
+ boolean result = insert(attrName, tree.isTreeHolder() ? ((ExprTreeHolder) tree).getInnerTree() : tree, false);
return result;
}
@@ -522,13 +480,14 @@ public class ClassAd extends ExprTree {
if (attrName.isEmpty() || pRef == null) {
throw new HyracksDataException();
}
- CaseInsensitiveString pstrAttr = StringPool.get();
- pstrAttr.set(attrName);
-
if (tree != null) {
+ CaseInsensitiveString pstrAttr = objectPool.caseInsensitiveStringPool.get();
+ pstrAttr.set(attrName);
+ ExprTreeHolder mutableTree = objectPool.mutableExprPool.get();
+ mutableTree.copyFrom(tree);
// parent of the expression is this classad
tree.setParentScope(this);
- attrList.put(pstrAttr, tree);
+ attrList.put(pstrAttr, mutableTree);
bRet = true;
}
return (bRet);
@@ -536,8 +495,9 @@ public class ClassAd extends ExprTree {
public boolean deepInsert(ExprTree scopeExpr, String name, ExprTree tree) throws HyracksDataException {
ClassAd ad = privateGetDeepScope(scopeExpr);
- if (ad == null)
+ if (ad == null) {
return (false);
+ }
return (ad.insert(name, tree));
}
@@ -545,10 +505,9 @@ public class ClassAd extends ExprTree {
// --- begin lookup methods
public ExprTree lookup(String name) {
- CaseInsensitiveString aString = StringPool.get();
+ CaseInsensitiveString aString = objectPool.caseInsensitiveStringPool.get();
aString.set(name);
ExprTree expr = lookup(aString);
- StringPool.put(aString);
return expr;
}
@@ -569,9 +528,9 @@ public class ClassAd extends ExprTree {
}
}
- public ExprTree lookupInScope(AMutableCharArrayString name, ClassAd finalScope) {
- EvalState state = new EvalState();
- ExprTreeHolder tree = new ExprTreeHolder();
+ public ExprTree lookupInScope(AMutableCharArrayString name, ClassAd finalScope) throws HyracksDataException {
+ EvalState state = objectPool.evalStatePool.get();
+ ExprTreeHolder tree = objectPool.mutableExprPool.get();
int rval;
state.setScopes(this);
rval = lookupInScope(name.toString(), tree, state);
@@ -583,10 +542,10 @@ public class ClassAd extends ExprTree {
return null;
}
- public int lookupInScope(String name, ExprTreeHolder expr, EvalState state) {
+ public int lookupInScope(String name, ExprTreeHolder expr, EvalState state) throws HyracksDataException {
ClassAd current = this;
- ClassAd superScope = new ClassAd();
+ ClassAd superScope = objectPool.classAdPool.get();
expr.setInnerTree(null);
while (expr.getInnerTree() == null && current != null) {
@@ -598,11 +557,17 @@ public class ClassAd extends ExprTree {
if ((expr.getInnerTree() != null)) {
return EvalResult.EVAL_OK.ordinal();
}
-
- if (state.getRootAd().equals(current)) {
- superScope = null;
- } else {
- superScope = current.parentScope;
+ try {
+ if (state.getRootAd() == null) {
+ return (EvalResult.EVAL_UNDEF.ordinal());
+ } else if (state.getRootAd().equals(current)) {
+ superScope = null;
+ } else {
+ superScope = current.parentScope;
+ }
+ } catch (Throwable th) {
+ th.printStackTrace();
+ throw th;
}
if (!getSpecialAttrNames().contains(name)) {
// continue searching from the superScope ...
@@ -638,10 +603,9 @@ public class ClassAd extends ExprTree {
// --- begin deletion methods
public boolean delete(String name) throws HyracksDataException {
- CaseInsensitiveString aString = StringPool.get();
+ CaseInsensitiveString aString = objectPool.caseInsensitiveStringPool.get();
aString.set(name);
boolean success = delete(aString);
- StringPool.put(aString);
return success;
}
@@ -657,10 +621,10 @@ public class ClassAd extends ExprTree {
// behavior copied from old ClassAds. It's also one reason you
// probably don't want to use this feature in the future.
if (chainedParentAd != null && chainedParentAd.lookup(name) != null) {
- Value undefined_value = new Value();
+ Value undefined_value = objectPool.valuePool.get();
undefined_value.setUndefinedValue();
deleted_attribute = true;
- ExprTree plit = Literal.createLiteral(undefined_value);
+ ExprTree plit = Literal.createLiteral(undefined_value, objectPool);
insert(name.get(), plit);
}
return deleted_attribute;
@@ -668,12 +632,12 @@ public class ClassAd extends ExprTree {
public boolean deepDelete(ExprTree scopeExpr, String name) throws HyracksDataException {
ClassAd ad = privateGetDeepScope(scopeExpr);
- if (ad == null)
+ if (ad == null) {
return (false);
- CaseInsensitiveString aString = StringPool.get();
+ }
+ CaseInsensitiveString aString = objectPool.caseInsensitiveStringPool.get();
aString.set(name);;
boolean success = ad.delete(aString);
- StringPool.put(aString);
return success;
}
@@ -694,9 +658,9 @@ public class ClassAd extends ExprTree {
if (tree == null) {
tree = chainedParentAd.lookup(name);
}
- Value undefined_value = new Value();
+ Value undefined_value = objectPool.valuePool.get();
undefined_value.setUndefinedValue();
- ExprTree plit = Literal.createLiteral(undefined_value);
+ ExprTree plit = Literal.createLiteral(undefined_value, objectPool);
//why??
insert(name, plit);
}
@@ -705,8 +669,9 @@ public class ClassAd extends ExprTree {
public ExprTree deepRemove(ExprTree scopeExpr, String name) throws HyracksDataException {
ClassAd ad = privateGetDeepScope(scopeExpr);
- if (ad == null)
+ if (ad == null) {
return (null);
+ }
return (ad.remove(name));
}
@@ -721,7 +686,7 @@ public class ClassAd extends ExprTree {
public void modify(ClassAd mod) throws HyracksDataException {
ClassAd ctx;
ExprTree expr;
- Value val = new Value();
+ Value val = objectPool.valuePool.get();
// Step 0: Determine Context
if ((expr = mod.lookup(Common.ATTR_CONTEXT)) != null) {
@@ -734,7 +699,7 @@ public class ClassAd extends ExprTree {
// Step 1: Process Replace attribute
if ((expr = mod.lookup(Common.ATTR_REPLACE)) != null) {
- ClassAd ad = new ClassAd();
+ ClassAd ad = objectPool.classAdPool.get();
if (expr.publicEvaluate(val) && val.isClassAdValue(ad)) {
ctx.clear();
ctx.update(ad);
@@ -743,7 +708,7 @@ public class ClassAd extends ExprTree {
// Step 2: Process Updates attribute
if ((expr = mod.lookup(Common.ATTR_UPDATES)) != null) {
- ClassAd ad = new ClassAd();
+ ClassAd ad = objectPool.classAdPool.get();
if (expr.publicEvaluate(val) && val.isClassAdValue(ad)) {
ctx.update(ad);
}
@@ -751,8 +716,8 @@ public class ClassAd extends ExprTree {
// Step 3: Process Deletes attribute
if ((expr = mod.lookup(Common.ATTR_DELETES)) != null) {
- ExprList list = new ExprList();
- AMutableCharArrayString attrName = new AMutableCharArrayString();
+ ExprList list = objectPool.exprListPool.get();
+ AMutableCharArrayString attrName = objectPool.strPool.get();
// make a first pass to check that it is a list of strings ...
if (!expr.publicEvaluate(val) || !val.isListValue(list)) {
@@ -774,18 +739,18 @@ public class ClassAd extends ExprTree {
@Override
public ExprTree copy() throws HyracksDataException {
- ClassAd newAd = new ClassAd();
- newAd.parentScope = parentScope;
- newAd.chainedParentAd = chainedParentAd;
+ ClassAd newAd = objectPool.classAdPool.get();
+ newAd.parentScope = (parentScope == null) ? null : (ClassAd) parentScope.copy();
+ newAd.chainedParentAd = chainedParentAd == null ? null : (ClassAd) chainedParentAd.copy();
for (Entry<CaseInsensitiveString, ExprTree> entry : attrList.entrySet()) {
- newAd.insert(entry.getKey().get(), entry.getValue().copy(), false);
+ newAd.insert(entry.getKey().get(), entry.getValue(), false);
}
return newAd;
}
@Override
- public boolean publicEvaluate(EvalState state, Value val) {
+ public boolean publicEvaluate(EvalState state, Value val) throws HyracksDataException {
val.setClassAdValue(this);
return (true);
}
@@ -800,9 +765,9 @@ public class ClassAd extends ExprTree {
@Override
public boolean privateFlatten(EvalState state, Value val, ExprTreeHolder tree, AMutableInt32 i)
throws HyracksDataException {
- ClassAd newAd = new ClassAd();
- Value eval = new Value();
- ExprTreeHolder etree = new ExprTreeHolder();
+ ClassAd newAd = objectPool.classAdPool.get();
+ Value eval = objectPool.valuePool.get();
+ ExprTreeHolder etree = objectPool.mutableExprPool.get();;
ClassAd oldAd;
tree.setInnerTree(null); // Just to be safe... wenger 2003-12-11.
@@ -814,23 +779,27 @@ public class ClassAd extends ExprTree {
// flatten expression
if (!entry.getValue().publicFlatten(state, eval, etree)) {
tree.setInnerTree(null);;
- eval.clear();
+ eval.setUndefinedValue();
state.setCurAd(oldAd);
return false;
}
// if a value was obtained, convert it to a literal
if (etree.getInnerTree() == null) {
- etree.setInnerTree(Literal.createLiteral(eval));
+ etree.setInnerTree(Literal.createLiteral(eval, objectPool));
if (etree.getInnerTree() == null) {
tree.setInnerTree(null);
- eval.clear();
+ eval.setUndefinedValue();
state.setCurAd(oldAd);
return false;
}
}
- newAd.attrList.put(entry.getKey(), etree);
- eval.clear();
+ CaseInsensitiveString key = objectPool.caseInsensitiveStringPool.get();
+ ExprTreeHolder value = objectPool.mutableExprPool.get();
+ key.set(entry.getKey().get());
+ value.copyFrom(etree);
+ newAd.attrList.put(key, value);
+ eval.setUndefinedValue();
}
tree.setInnerTree(newAd);
@@ -839,8 +808,8 @@ public class ClassAd extends ExprTree {
}
public boolean evaluateAttr(String attr, Value val) throws HyracksDataException {
- EvalState state = new EvalState();
- ExprTreeHolder tree = new ExprTreeHolder();
+ EvalState state = objectPool.evalStatePool.get();
+ ExprTreeHolder tree = objectPool.mutableExprPool.get();
state.setScopes(this);
switch (lookupInScope(attr, tree, state)) {
case ExprTree.EVAL_FAIL_Int:
@@ -860,8 +829,8 @@ public class ClassAd extends ExprTree {
public boolean evaluateExpr(String buf, Value result) throws HyracksDataException {
boolean successfully_evaluated;
- ExprTreeHolder tree = new ExprTreeHolder();
- ClassAdParser parser = new ClassAdParser(null, false, true, false, null, null, null);
+ ExprTreeHolder tree = objectPool.mutableExprPool.get();
+ ClassAdParser parser = objectPool.classAdParserPool.get();
try {
if (parser.parseExpression(buf, tree)) {
@@ -876,54 +845,54 @@ public class ClassAd extends ExprTree {
}
public boolean evaluateExpr(ExprTreeHolder tree, Value val) throws HyracksDataException {
- EvalState state = new EvalState();
+ EvalState state = objectPool.evalStatePool.get();
state.setScopes(this);
return (tree.publicEvaluate(state, val));
}
public boolean evaluateExpr(ExprTreeHolder tree, Value val, ExprTreeHolder sig) throws HyracksDataException {
- EvalState state = new EvalState();
+ EvalState state = objectPool.evalStatePool.get();
state.setScopes(this);
return (tree.publicEvaluate(state, val, sig));
}
public boolean evaluateAttrInt(String attr, AMutableInt64 i) throws HyracksDataException {
- Value val = new Value();
+ Value val = objectPool.valuePool.get();
return (evaluateAttr(attr, val) && val.isIntegerValue(i));
}
public boolean evaluateAttrReal(String attr, AMutableDouble r) throws HyracksDataException {
- Value val = new Value();
+ Value val = objectPool.valuePool.get();
return (evaluateAttr(attr, val) && val.isRealValue(r));
}
public boolean evaluateAttrNumber(String attr, AMutableInt64 i) throws HyracksDataException {
- Value val = new Value();
+ Value val = objectPool.valuePool.get();
return (evaluateAttr(attr, val) && val.isNumber(i));
}
public boolean evaluateAttrNumber(String attr, AMutableDouble r) throws HyracksDataException {
- Value val = new Value();
+ Value val = objectPool.valuePool.get();
return (evaluateAttr(attr, val) && val.isNumber(r));
}
public boolean evaluateAttrString(String attr, AMutableCharArrayString buf, int len) throws HyracksDataException {
- Value val = new Value();
+ Value val = objectPool.valuePool.get();
return (evaluateAttr(attr, val) && val.isStringValue(buf, len));
}
public boolean evaluateAttrString(String attr, AMutableCharArrayString buf) throws HyracksDataException {
- Value val = new Value();
+ Value val = objectPool.valuePool.get();
return (evaluateAttr(attr, val) && val.isStringValue(buf));
}
public boolean evaluateAttrBool(String attr, MutableBoolean b) throws HyracksDataException {
- Value val = new Value();
+ Value val = objectPool.valuePool.get();
return (evaluateAttr(attr, val) && val.isBooleanValue(b));
}
public boolean evaluateAttrBoolEquiv(String attr, MutableBoolean b) throws HyracksDataException {
- Value val = new Value();
+ Value val = objectPool.valuePool.get();
return (evaluateAttr(attr, val) && val.isBooleanValueEquiv(b));
}
@@ -942,7 +911,7 @@ public class ClassAd extends ExprTree {
public boolean getExternalReferences(ExprTree tree, TreeSet<String> refs, boolean fullNames)
throws HyracksDataException {
- EvalState state = new EvalState();
+ EvalState state = objectPool.evalStatePool.get();
// Treat this ad as the root of the tree for reference tracking.
// If an attribute is only present in a parent scope of this ad,
// then we want to treat it as an external reference.
@@ -962,12 +931,12 @@ public class ClassAd extends ExprTree {
return (true);
case ATTRREF_NODE: {
- ClassAd start = new ClassAd();
- ExprTreeHolder tree = new ExprTreeHolder();
- ExprTreeHolder result = new ExprTreeHolder();
- AMutableCharArrayString attr = new AMutableCharArrayString();
- Value val = new Value();
- MutableBoolean abs = new MutableBoolean();
+ ClassAd start = objectPool.classAdPool.get();
+ ExprTreeHolder tree = objectPool.mutableExprPool.get();
+ ExprTreeHolder result = objectPool.mutableExprPool.get();
+ AMutableCharArrayString attr = objectPool.strPool.get();
+ Value val = objectPool.valuePool.get();
+ MutableBoolean abs = objectPool.boolPool.get();
((AttributeReference) expr).getComponents(tree, attr, abs);
// establish starting point for attribute search
@@ -984,9 +953,9 @@ public class ClassAd extends ExprTree {
// are in the tree part
if (val.isUndefinedValue()) {
if (fullNames) {
- AMutableCharArrayString fullName = new AMutableCharArrayString();
+ AMutableCharArrayString fullName = objectPool.strPool.get();
if (tree.getInnerTree() != null) {
- ClassAdUnParser unparser = new PrettyPrint();
+ ClassAdUnParser unparser = objectPool.prettyPrintPool.get();
unparser.unparse(fullName, tree);
fullName.appendChar('.');
}
@@ -1041,10 +1010,10 @@ public class ClassAd extends ExprTree {
}
case OP_NODE: {
// recurse on subtrees
- AMutableInt32 opKind = new AMutableInt32(0);
- ExprTreeHolder t1 = new ExprTreeHolder();
- ExprTreeHolder t2 = new ExprTreeHolder();
- ExprTreeHolder t3 = new ExprTreeHolder();
+ AMutableInt32 opKind = objectPool.int32Pool.get();
+ ExprTreeHolder t1 = objectPool.mutableExprPool.get();
+ ExprTreeHolder t2 = objectPool.mutableExprPool.get();
+ ExprTreeHolder t3 = objectPool.mutableExprPool.get();
((Operation) expr).getComponents(opKind, t1, t2, t3);
if (t1.getInnerTree() != null && !privateGetExternalReferences(t1, ad, state, refs, fullNames)) {
@@ -1060,8 +1029,8 @@ public class ClassAd extends ExprTree {
}
case FN_CALL_NODE: {
// recurse on subtrees
- AMutableCharArrayString fnName = new AMutableCharArrayString();
- ExprList args = new ExprList();
+ AMutableCharArrayString fnName = objectPool.strPool.get();
+ ExprList args = objectPool.exprListPool.get();
((FunctionCall) expr).getComponents(fnName, args);
for (ExprTree tree : args.getExprList()) {
if (!privateGetExternalReferences(tree, ad, state, refs, fullNames)) {
@@ -1072,8 +1041,8 @@ public class ClassAd extends ExprTree {
}
case CLASSAD_NODE: {
// recurse on subtrees
- Map<CaseInsensitiveString, ExprTree> attrs = new HashMap<CaseInsensitiveString, ExprTree>();
- ((ClassAd) expr).getComponents(attrs);
+ Map<CaseInsensitiveString, ExprTree> attrs = objectPool.strToExprPool.get();
+ ((ClassAd) expr).getComponents(attrs, objectPool);
for (Entry<CaseInsensitiveString, ExprTree> entry : attrs.entrySet()) {
if (state.getDepthRemaining() <= 0) {
return false;
@@ -1089,7 +1058,7 @@ public class ClassAd extends ExprTree {
}
case EXPR_LIST_NODE: {
// recurse on subtrees
- ExprList exprs = new ExprList();
+ ExprList exprs = objectPool.exprListPool.get();
((ExprList) expr).getComponents(exprs);
for (ExprTree exprTree : exprs.getExprList()) {
@@ -1115,7 +1084,7 @@ public class ClassAd extends ExprTree {
// PortReferences is a Map<ClassAd,TreeSet<Strings>>
public boolean getExternalReferences(ExprTree tree, Map<ClassAd, TreeSet<String>> refs)
throws HyracksDataException {
- EvalState state = new EvalState();
+ EvalState state = objectPool.evalStatePool.get();
// Treat this ad as the root of the tree for reference tracking.
// If an attribute is only present in a parent scope of this ad,
// then we want to treat it as an external reference.
@@ -1133,12 +1102,12 @@ public class ClassAd extends ExprTree {
return (true);
case ATTRREF_NODE: {
- ClassAd start = new ClassAd();
- ExprTreeHolder tree = new ExprTreeHolder();
- ExprTreeHolder result = new ExprTreeHolder();
- AMutableCharArrayString attr = new AMutableCharArrayString();
- Value val = new Value();
- MutableBoolean abs = new MutableBoolean();
+ ClassAd start = objectPool.classAdPool.get();
+ ExprTreeHolder tree = objectPool.mutableExprPool.get();
+ ExprTreeHolder result = objectPool.mutableExprPool.get();
+ AMutableCharArrayString attr = objectPool.strPool.get();
+ Value val = objectPool.valuePool.get();
+ MutableBoolean abs = objectPool.boolPool.get();
((AttributeReference) expr).getComponents(tree, attr, abs);
// establish starting point for attribute search
@@ -1148,8 +1117,9 @@ public class ClassAd extends ExprTree {
return false; // NAC
} // NAC
} else {
- if (!tree.publicEvaluate(state, val))
+ if (!tree.publicEvaluate(state, val)) {
return (false);
+ }
// if the tree evals to undefined, the external references
// are in the tree part
if (val.isUndefinedValue()) {
@@ -1157,8 +1127,9 @@ public class ClassAd extends ExprTree {
}
// otherwise, if the tree didn't evaluate to a classad,
// we have a problem
- if (!val.isClassAdValue(start))
+ if (!val.isClassAdValue(start)) {
return (false);
+ }
// make sure that we are starting from a "valid" scope
if (!refs.containsKey(start) && start != this) {
@@ -1169,7 +1140,7 @@ public class ClassAd extends ExprTree {
ClassAd curAd = state.getCurAd();
TreeSet<String> pitr = refs.get(start);
if (pitr == null) {
- pitr = new TreeSet<String>();
+ pitr = objectPool.strSetPool.get();
refs.put(start, pitr);
}
switch (start.lookupInScope(attr.toString(), result, state)) {
@@ -1198,10 +1169,10 @@ public class ClassAd extends ExprTree {
case OP_NODE: {
// recurse on subtrees
- AMutableInt32 opKind = new AMutableInt32(0);
- ExprTreeHolder t1 = new ExprTreeHolder();
- ExprTreeHolder t2 = new ExprTreeHolder();
- ExprTreeHolder t3 = new ExprTreeHolder();
+ AMutableInt32 opKind = objectPool.int32Pool.get();
+ ExprTreeHolder t1 = objectPool.mutableExprPool.get();
+ ExprTreeHolder t2 = objectPool.mutableExprPool.get();
+ ExprTreeHolder t3 = objectPool.mutableExprPool.get();
((Operation) expr).getComponents(opKind, t1, t2, t3);
if (t1.getInnerTree() != null && !privateGetExternalReferences(t1, ad, state, refs)) {
return (false);
@@ -1217,8 +1188,8 @@ public class ClassAd extends ExprTree {
case FN_CALL_NODE: {
// recurse on subtrees
- AMutableCharArrayString fnName = new AMutableCharArrayString();
- ExprList args = new ExprList();
+ AMutableCharArrayString fnName = objectPool.strPool.get();
+ ExprList args = objectPool.exprListPool.get();
((FunctionCall) expr).getComponents(fnName, args);
for (ExprTree exprTree : args.getExprList()) {
@@ -1231,9 +1202,9 @@ public class ClassAd extends ExprTree {
case CLASSAD_NODE: {
// recurse on subtrees
- HashMap<CaseInsensitiveString, ExprTree> attrs = new HashMap<CaseInsensitiveString, ExprTree>();
+ HashMap<CaseInsensitiveString, ExprTree> attrs = objectPool.strToExprPool.get();
- ((ClassAd) expr).getComponents(attrs);
+ ((ClassAd) expr).getComponents(attrs, objectPool);
for (Entry<CaseInsensitiveString, ExprTree> entry : attrs.entrySet()) {
if (!privateGetExternalReferences(entry.getValue(), ad, state, refs)) {
return (false);
@@ -1244,7 +1215,7 @@ public class ClassAd extends ExprTree {
case EXPR_LIST_NODE: {
// recurse on subtrees
- ExprList exprs = new ExprList();
+ ExprList exprs = objectPool.exprListPool.get();
((ExprList) expr).getComponents(exprs);
for (ExprTree exprTree : exprs.getExprList()) {
if (!privateGetExternalReferences(exprTree, ad, state, refs)) {
@@ -1273,7 +1244,7 @@ public class ClassAd extends ExprTree {
*/
public boolean getInternalReferences(ExprTree tree, TreeSet<String> refs, boolean fullNames)
throws HyracksDataException {
- EvalState state = new EvalState();
+ EvalState state = objectPool.evalStatePool.get();
// Treat this ad as the root of the tree for reference tracking.
// If an attribute is only present in a parent scope of this ad,
@@ -1295,12 +1266,12 @@ public class ClassAd extends ExprTree {
}
case ATTRREF_NODE: {
- ClassAd start = new ClassAd();
- ExprTreeHolder tree = new ExprTreeHolder();
- ExprTreeHolder result = new ExprTreeHolder();
- AMutableCharArrayString attr = new AMutableCharArrayString();
- Value val = new Value();
- MutableBoolean abs = new MutableBoolean();
+ ClassAd start = objectPool.classAdPool.get();;
+ ExprTreeHolder tree = objectPool.mutableExprPool.get();
+ ExprTreeHolder result = objectPool.mutableExprPool.get();
+ AMutableCharArrayString attr = objectPool.strPool.get();
+ Value val = objectPool.valuePool.get();
+ MutableBoolean abs = objectPool.boolPool.get();
((AttributeReference) expr).getComponents(tree, attr, abs);
@@ -1393,10 +1364,10 @@ public class ClassAd extends ExprTree {
case OP_NODE: {
//recurse on subtrees
- AMutableInt32 op = new AMutableInt32(0);
- ExprTreeHolder t1 = new ExprTreeHolder();
- ExprTreeHolder t2 = new ExprTreeHolder();
- ExprTreeHolder t3 = new ExprTreeHolder();
+ AMutableInt32 op = objectPool.int32Pool.get();
+ ExprTreeHolder t1 = objectPool.mutableExprPool.get();
+ ExprTreeHolder t2 = objectPool.mutableExprPool.get();
+ ExprTreeHolder t3 = objectPool.mutableExprPool.get();
((Operation) expr).getComponents(op, t1, t2, t3);
if (t1.getInnerTree() != null && !privateGetInternalReferences(t1, ad, state, refs, fullNames)) {
return false;
@@ -1414,8 +1385,8 @@ public class ClassAd extends ExprTree {
case FN_CALL_NODE: {
//recurse on the subtrees!
- AMutableCharArrayString fnName = new AMutableCharArrayString();
- ExprList args = new ExprList();
+ AMutableCharArrayString fnName = objectPool.strPool.get();
+ ExprList args = objectPool.exprListPool.get();
((FunctionCall) expr).getComponents(fnName, args);
for (ExprTree exprTree : args.getExprList()) {
@@ -1429,7 +1400,7 @@ public class ClassAd extends ExprTree {
case CLASSAD_NODE: {
//also recurse on subtrees...
- HashMap<CaseInsensitiveString, ExprTree> attrs = new HashMap<CaseInsensitiveString, ExprTree>();
+ HashMap<CaseInsensitiveString, ExprTree> attrs = objectPool.strToExprPool.get();
// If this ClassAd is only being used here as the scoping
// for an attribute reference, don't recurse into all of
@@ -1438,7 +1409,7 @@ public class ClassAd extends ExprTree {
return true;
}
- ((ClassAd) expr).getComponents(attrs);
+ ((ClassAd) expr).getComponents(attrs, objectPool);
for (Entry<CaseInsensitiveString, ExprTree> entry : attrs.entrySet()) {
if (state.getDepthRemaining() <= 0) {
return false;
@@ -1457,7 +1428,7 @@ public class ClassAd extends ExprTree {
}
case EXPR_LIST_NODE: {
- ExprList exprs = new ExprList();
+ ExprList exprs = objectPool.exprListPool.get();
((ExprList) expr).getComponents(exprs);
for (ExprTree exprTree : exprs.getExprList()) {
@@ -1484,14 +1455,14 @@ public class ClassAd extends ExprTree {
}
public boolean publicFlatten(ExprTree tree, Value val, ExprTreeHolder fexpr) throws HyracksDataException {
- EvalState state = new EvalState();
+ EvalState state = objectPool.evalStatePool.get();
state.setScopes(this);
return (tree.publicFlatten(state, val, fexpr));
}
public boolean flattenAndInline(ExprTree tree, Value val, ExprTreeHolder fexpr) throws HyracksDataException {
- EvalState state = new EvalState();
+ EvalState state = objectPool.evalStatePool.get();
state.setScopes(this);
state.setFlattenAndInline(true);
@@ -1529,12 +1500,8 @@ public class ClassAd extends ExprTree {
return chainedParentAd;
}
- public void setValue(ClassAd value) {
- this.attrList = value.attrList;
- this.alternateScope = value.alternateScope;
- this.chainedParentAd = value.chainedParentAd;
- this.parentScope = value.parentScope;
- this.size = value.size;
+ public void setValue(ClassAd value) throws HyracksDataException {
+ copyFrom(value);
}
@Override
@@ -1566,6 +1533,6 @@ public class ClassAd extends ExprTree {
}
public void createParser() {
- parser = new ClassAdParser(null, false, true, false, null, null, null);
+ parser = objectPool.classAdParserPool.get();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/ClassAdFunc.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/ClassAdFunc.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/ClassAdFunc.java
index 4e77bc0..8a951ad 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/ClassAdFunc.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/ClassAdFunc.java
@@ -18,8 +18,10 @@
*/
package org.apache.asterix.external.classad;
+import org.apache.asterix.external.classad.object.pool.ClassAdObjectPool;
import org.apache.hyracks.api.exceptions.HyracksDataException;
public interface ClassAdFunc {
- public boolean call(String name, ExprList argList, EvalState state, Value val) throws HyracksDataException;
+ public boolean call(String name, ExprList argList, EvalState state, Value val, ClassAdObjectPool objectPool)
+ throws HyracksDataException;
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/ClassAdTime.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/ClassAdTime.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/ClassAdTime.java
index 66c5f56..359511b 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/ClassAdTime.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/ClassAdTime.java
@@ -92,6 +92,7 @@ public class ClassAdTime {
}
public void setRelativeTime(long ms) {
+ this.isAbsolute = false;
timeZoneCalendar.setTimeInMillis(ms);
}
@@ -141,6 +142,17 @@ public class ClassAdTime {
this.timeZoneCalendar.setTimeInMillis(0);
}
+ public void setCurrentAbsolute() {
+ this.isAbsolute = true;
+ this.timeZoneCalendar = Calendar.getInstance();
+ this.timeZoneCalendar.setTimeInMillis(0);
+ }
+
+ public void setTimeZone(String timeZoneId) {
+ this.timeZoneCalendar = Calendar.getInstance(TimeZone.getTimeZone(timeZoneId));
+ this.timeZoneCalendar.setTimeInMillis(0);
+ }
+
public ClassAdTime(String timeZoneId) {
this.isAbsolute = true;
this.timeZoneCalendar = Calendar.getInstance(TimeZone.getTimeZone(timeZoneId));
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/ClassAdUnParser.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/ClassAdUnParser.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/ClassAdUnParser.java
index 4689612..41333c5 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/ClassAdUnParser.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/ClassAdUnParser.java
@@ -18,12 +18,12 @@
*/
package org.apache.asterix.external.classad;
-import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import org.apache.asterix.external.classad.Value.NumberFactor;
+import org.apache.asterix.external.classad.object.pool.ClassAdObjectPool;
import org.apache.asterix.om.base.AMutableDouble;
import org.apache.asterix.om.base.AMutableInt32;
import org.apache.asterix.om.base.AMutableInt64;
@@ -38,8 +38,11 @@ public class ClassAdUnParser {
" >>> ", " () ", " [] ", " ?: " };
protected static char delimiter = '\"';
+ protected final ClassAdObjectPool objectPool;
+
/// Constructor
- public ClassAdUnParser() {
+ public ClassAdUnParser(ClassAdObjectPool objectPool) {
+ this.objectPool = objectPool;
}
// The default delimiter for strings is '\"'
@@ -64,7 +67,7 @@ public class ClassAdUnParser {
break;
case STRING_VALUE: {
- AMutableCharArrayString s = new AMutableCharArrayString();
+ AMutableCharArrayString s = objectPool.strPool.get();
val.isStringValue(s);
buffer.appendChar('"');
for (int i = 0; i < s.getLength(); i++) {
@@ -118,13 +121,13 @@ public class ClassAdUnParser {
return;
}
case INTEGER_VALUE: {
- AMutableInt64 i = new AMutableInt64(0);
+ AMutableInt64 i = objectPool.int64Pool.get();
val.isIntegerValue(i);
buffer.appendString(String.valueOf(i.getLongValue()));
return;
}
case REAL_VALUE: {
- AMutableDouble real = new AMutableDouble(0);
+ AMutableDouble real = objectPool.doublePool.get();
val.isRealValue(real);
if (real.getDoubleValue() == 0.0) {
// It might be positive or negative and it's
@@ -145,7 +148,7 @@ public class ClassAdUnParser {
return;
}
case BOOLEAN_VALUE: {
- MutableBoolean b = new MutableBoolean();
+ MutableBoolean b = objectPool.boolPool.get();
val.isBooleanValue(b);
buffer.appendString(b.booleanValue() ? "true" : "false");
return;
@@ -159,7 +162,7 @@ public class ClassAdUnParser {
return;
}
case ABSOLUTE_TIME_VALUE: {
- ClassAdTime asecs = new ClassAdTime();
+ ClassAdTime asecs = objectPool.classAdTimePool.get();
val.isAbsoluteTimeValue(asecs);
buffer.appendString("absTime(\"");
@@ -168,7 +171,7 @@ public class ClassAdUnParser {
return;
}
case RELATIVE_TIME_VALUE: {
- ClassAdTime rsecs = new ClassAdTime();
+ ClassAdTime rsecs = objectPool.classAdTimePool.get();
val.isRelativeTimeValue(rsecs);
buffer.appendString("relTime(\"");
Util.relTimeToString(rsecs.getRelativeTime(), buffer);
@@ -177,16 +180,16 @@ public class ClassAdUnParser {
return;
}
case CLASSAD_VALUE: {
- ClassAd ad = new ClassAd();
- Map<CaseInsensitiveString, ExprTree> attrs = new HashMap<CaseInsensitiveString, ExprTree>();
+ ClassAd ad = objectPool.classAdPool.get();
+ Map<CaseInsensitiveString, ExprTree> attrs = objectPool.strToExprPool.get();
val.isClassAdValue(ad);
- ad.getComponents(attrs);
+ ad.getComponents(attrs, objectPool);
unparseAux(buffer, attrs);
return;
}
case SLIST_VALUE:
case LIST_VALUE: {
- ExprList el = new ExprList();
+ ExprList el = objectPool.exprListPool.get();
val.isListValue(el);
unparseAux(buffer, el);
return;
@@ -211,49 +214,49 @@ public class ClassAdUnParser {
switch (tree.getKind()) {
case LITERAL_NODE: { // value
- Value val = new Value();
- AMutableNumberFactor factor = new AMutableNumberFactor();
- ((Literal) tree.self()).getComponents(val, factor);
+ Value val = objectPool.valuePool.get();
+ AMutableNumberFactor factor = objectPool.numFactorPool.get();
+ ((Literal) tree.getTree()).getComponents(val, factor);
unparseAux(buffer, val, factor.getFactor());
return;
}
case ATTRREF_NODE: { // string
- ExprTreeHolder expr = new ExprTreeHolder(); //needs initialization
- AMutableCharArrayString ref = new AMutableCharArrayString();
- MutableBoolean absolute = new MutableBoolean();
- ((AttributeReference) tree.self()).getComponents(expr, ref, absolute);
+ ExprTreeHolder expr = objectPool.mutableExprPool.get(); //needs initialization
+ AMutableCharArrayString ref = objectPool.strPool.get();
+ MutableBoolean absolute = objectPool.boolPool.get();
+ ((AttributeReference) tree.getTree()).getComponents(expr, ref, absolute);
unparseAux(buffer, expr, ref, absolute.booleanValue());
return;
}
case OP_NODE: { //string
- AMutableInt32 op = new AMutableInt32(0);
- ExprTreeHolder t1 = new ExprTreeHolder();
- ExprTreeHolder t2 = new ExprTreeHolder();
- ExprTreeHolder t3 = new ExprTreeHolder();
- ((Operation) tree.self()).getComponents(op, t1, t2, t3);
+ AMutableInt32 op = objectPool.int32Pool.get();
+ ExprTreeHolder t1 = objectPool.mutableExprPool.get();
+ ExprTreeHolder t2 = objectPool.mutableExprPool.get();
+ ExprTreeHolder t3 = objectPool.mutableExprPool.get();
+ ((Operation) tree.getTree()).getComponents(op, t1, t2, t3);
unparseAux(buffer, op.getIntegerValue().intValue(), t1, t2, t3);
return;
}
case FN_CALL_NODE: { // string
- AMutableCharArrayString fnName = new AMutableCharArrayString();
- ExprList args = new ExprList();
- ((FunctionCall) tree.self()).getComponents(fnName, args);
+ AMutableCharArrayString fnName = objectPool.strPool.get();
+ ExprList args = objectPool.exprListPool.get();
+ ((FunctionCall) tree.getTree()).getComponents(fnName, args);
unparseAux(buffer, fnName, args);
return;
}
case CLASSAD_NODE: { // nested record
- Map<CaseInsensitiveString, ExprTree> attrs = new HashMap<CaseInsensitiveString, ExprTree>();
- ((ClassAd) tree.self()).getComponents(attrs);
+ Map<CaseInsensitiveString, ExprTree> attrs = objectPool.strToExprPool.get();
+ ((ClassAd) tree.getTree()).getComponents(attrs, objectPool);
unparseAux(buffer, attrs);
return;
}
case EXPR_LIST_NODE: { // list
- ExprList exprs = new ExprList();
- ((ExprList) tree.self()).getComponents(exprs);
+ ExprList exprs = objectPool.exprListPool.get();
+ ((ExprList) tree.getTree()).getComponents(exprs);
unparseAux(buffer, exprs);
return;
}
@@ -345,8 +348,8 @@ public class ClassAdUnParser {
// to unparse attribute names (quoted & unquoted attributes)
public void unparseAux(AMutableCharArrayString buffer, AMutableCharArrayString identifier)
throws HyracksDataException {
- Value val = new Value();
- AMutableCharArrayString idstr = new AMutableCharArrayString();
+ Value val = objectPool.valuePool.get();
+ AMutableCharArrayString idstr = objectPool.strPool.get();
val.setStringValue(identifier);
setDelimiter('\''); // change the delimiter from string-literal mode to quoted attribute mode
@@ -386,8 +389,9 @@ public class ClassAdUnParser {
buffer.appendString("." + attrName);
return;
}
- if (absolute)
+ if (absolute) {
buffer.appendChar('.');
+ }
unparseAux(buffer, attrName);
}
@@ -474,8 +478,8 @@ public class ClassAdUnParser {
* it's unparsed either as a quoted attribute or non-quoted attribute
*/
public void unparseAux(AMutableCharArrayString buffer, String identifier) throws HyracksDataException {
- Value val = new Value();
- AMutableCharArrayString idstr = new AMutableCharArrayString();
+ Value val = objectPool.valuePool.get();
+ AMutableCharArrayString idstr = objectPool.strPool.get();
val.setStringValue(identifier);
setDelimiter('\''); // change the delimiter from string-literal mode to quoted attribute mode
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/EvalState.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/EvalState.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/EvalState.java
index 0719fd8..41f34bc 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/EvalState.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/EvalState.java
@@ -18,6 +18,9 @@
*/
package org.apache.asterix.external.classad;
+import org.apache.asterix.external.classad.object.pool.ClassAdObjectPool;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
public class EvalState {
private int depthRemaining; // max recursion depth - current depth
@@ -27,10 +30,13 @@ public class EvalState {
// It can be set to a closer parent scope. Then that ClassAd is
// treated like it has no parent scope for LookupInScope() and
// Evaluate().
+ private final ClassAd rootAdTemp;
+ private final ClassAd curAdTemp;
private ClassAd rootAd;
private ClassAd curAd;
private boolean flattenAndInline; // NAC
private boolean inAttrRefScope;
+ private final ClassAdObjectPool objectPool;
public boolean isInAttrRefScope() {
return inAttrRefScope;
@@ -44,12 +50,15 @@ public class EvalState {
this.inAttrRefScope = inAttrRefScope;
}
- public EvalState() {
- rootAd = new ClassAd();
- curAd = new ClassAd();
+ public EvalState(ClassAdObjectPool objectPool) {
+ this.objectPool = objectPool;
+ rootAd = new ClassAd(this.objectPool);
+ curAd = new ClassAd(this.objectPool);
depthRemaining = ExprTree.MAX_CLASSAD_RECURSION;
flattenAndInline = false; // NAC
inAttrRefScope = false;
+ rootAdTemp = rootAd;
+ curAdTemp = curAd;
}
public boolean isFlattenAndInline() {
@@ -76,13 +85,14 @@ public class EvalState {
prevScope = curScope;
curScope = curScope.getParentScope();
}
-
rootAd = prevScope;
}
return;
}
public void reset() {
+ rootAd = rootAdTemp;
+ curAd = curAdTemp;
rootAd.reset();
curAd.reset();
depthRemaining = ExprTree.MAX_CLASSAD_RECURSION;
@@ -117,4 +127,16 @@ public class EvalState {
public void setRootAd(ClassAd classAd) {
this.rootAd = classAd;
}
+
+ public void set(EvalState state) throws HyracksDataException {
+ rootAd = rootAdTemp;
+ curAd = curAdTemp;
+ rootAd.reset();
+ curAd.reset();
+ rootAd.copyFrom(state.rootAd);
+ curAd.copyFrom(state.curAd);
+ depthRemaining = state.depthRemaining;
+ flattenAndInline = state.flattenAndInline;
+ inAttrRefScope = state.inAttrRefScope;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/ExprList.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/ExprList.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/ExprList.java
index e9376e2..3c8405f 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/ExprList.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/ExprList.java
@@ -22,20 +22,41 @@ import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
+import org.apache.asterix.external.classad.object.pool.ClassAdObjectPool;
import org.apache.asterix.om.base.AMutableInt32;
import org.apache.hyracks.api.exceptions.HyracksDataException;
public class ExprList extends ExprTree {
- private List<ExprTree> exprList;
- private EvalState state = new EvalState();
+ private final List<ExprTree> exprList;
+ private final EvalState state;
public boolean isShared = false;
+ public ExprList(List<ExprTree> exprs, ClassAdObjectPool objectPool) {
+ super(objectPool);
+ exprList = new ArrayList<ExprTree>();
+ this.state = new EvalState(this.objectPool);
+ copyList(exprs);
+ return;
+ }
+
+ public ExprList(ClassAdObjectPool objectPool) {
+ super(objectPool);
+ this.state = new EvalState(this.objectPool);
+ this.exprList = new ArrayList<ExprTree>();
+ }
+
+ public ExprList(boolean b, ClassAdObjectPool objectPool) {
+ super(objectPool);
+ this.state = new EvalState(this.objectPool);
+ this.exprList = new ArrayList<ExprTree>();
+ this.isShared = b;
+ }
+
public boolean copyFrom(ExprList exprList) throws HyracksDataException {
this.exprList.clear();
- for (ExprTree expr : exprList.exprList) {
- this.exprList.add(expr.copy());
- }
+ copyList(exprList.getExprList());
+ this.state.set(exprList.state);
return true;
}
@@ -86,28 +107,8 @@ public class ExprList extends ExprTree {
}
public void setExprList(List<ExprTree> exprList) {
- this.exprList = exprList;
- }
-
- public ExprList(List<ExprTree> exprs) {
- exprList = new ArrayList<ExprTree>();
- copyList(exprs);
- return;
- }
-
- public ExprList(ExprList other_list) throws HyracksDataException {
- exprList = new ArrayList<ExprTree>();
- copyFrom(other_list);
- return;
- }
-
- public ExprList() {
- exprList = new ArrayList<ExprTree>();
- }
-
- public ExprList(boolean b) {
- this.exprList = new ArrayList<ExprTree>();
- this.isShared = b;
+ this.exprList.clear();
+ this.exprList.addAll(exprList);
}
public void clear() {
@@ -116,7 +117,7 @@ public class ExprList extends ExprTree {
@Override
public ExprTree copy() throws HyracksDataException {
- ExprList newList = new ExprList();
+ ExprList newList = objectPool.exprListPool.get();
newList.copyFrom(this);
return newList;
}
@@ -129,7 +130,7 @@ public class ExprList extends ExprTree {
} else if (tree.getKind() != NodeKind.EXPR_LIST_NODE) {
is_same = false;
} else {
- ExprList other_list = (ExprList) tree;
+ ExprList other_list = (ExprList) tree.getTree();
if (exprList.size() != other_list.size()) {
is_same = false;
} else {
@@ -145,14 +146,14 @@ public class ExprList extends ExprTree {
return is_same;
}
- public static ExprList createExprList(List<ExprTree> exprs) {
- ExprList el = new ExprList();
+ public static ExprList createExprList(List<ExprTree> exprs, ClassAdObjectPool objectPool) {
+ ExprList el = objectPool.exprListPool.get();
el.copyList(exprs);
return el;
}
- public static ExprList createExprList(ExprList exprs) {
- ExprList el = new ExprList();
+ public static ExprList createExprList(ExprList exprs, ClassAdObjectPool objectPool) {
+ ExprList el = objectPool.exprListPool.get();
el.copyList(exprs.exprList);
return el;
}
@@ -217,9 +218,9 @@ public class ExprList extends ExprTree {
@Override
public boolean privateFlatten(EvalState state, Value val, ExprTreeHolder tree, AMutableInt32 aInt)
throws HyracksDataException {
- ExprTreeHolder nexpr = new ExprTreeHolder();
- Value tempVal = new Value();
- ExprList newList = new ExprList();
+ ExprTreeHolder nexpr = objectPool.mutableExprPool.get();
+ Value tempVal = objectPool.valuePool.get();
+ ExprList newList = objectPool.exprListPool.get();
tree.setInnerTree(null);; // Just to be safe... wenger 2003-12-11.
@@ -230,7 +231,7 @@ public class ExprList extends ExprTree {
}
// if only a value was obtained, convert to an expression
if (nexpr.getInnerTree() == null) {
- nexpr.setInnerTree(Literal.createLiteral(tempVal));
+ nexpr.setInnerTree(Literal.createLiteral(tempVal, objectPool));
if (nexpr.getInnerTree() == null) {
return false;
}
@@ -249,10 +250,11 @@ public class ExprList extends ExprTree {
}
public boolean getValue(Value val, ExprTree tree, EvalState es) throws HyracksDataException {
- EvalState currentState = new EvalState();
+ EvalState currentState = objectPool.evalStatePool.get();
- if (tree == null)
+ if (tree == null) {
return false;
+ }
// if called from user code, es == NULL so we use &state instead
currentState = (es != null) ? es : state;
@@ -276,5 +278,6 @@ public class ExprList extends ExprTree {
@Override
public void reset() {
exprList.clear();
+ state.reset();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/ExprTree.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/ExprTree.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/ExprTree.java
index ccbfd8b..2b5af38 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/ExprTree.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/ExprTree.java
@@ -18,6 +18,7 @@
*/
package org.apache.asterix.external.classad;
+import org.apache.asterix.external.classad.object.pool.ClassAdObjectPool;
import org.apache.asterix.om.base.AMutableDouble;
import org.apache.asterix.om.base.AMutableInt32;
import org.apache.asterix.om.base.AMutableInt64;
@@ -71,15 +72,24 @@ public abstract class ExprTree {
public ClassAd parentScope;
private CallableDebugFunction userDebugFunction;
+ protected final ClassAdObjectPool objectPool;
public abstract void reset();
- public ExprTree() {
+ public ExprTree(ClassAdObjectPool objectPool) {
+ if (objectPool == null) {
+ System.out.println();
+ }
+ this.objectPool = objectPool;
this.parentScope = null;
this.size = 0;
}
- public ExprTree(ExprTree expr) {
+ public ExprTree(ExprTree expr, ClassAdObjectPool objectPool) {
+ if (objectPool == null) {
+ System.out.println();
+ }
+ this.objectPool = objectPool;
this.size = expr.size;
}
@@ -112,14 +122,7 @@ public abstract class ExprTree {
* into a ClassAd.
*/
public void setParentScope(ClassAd scope) {
- if (scope == null) {
- parentScope = null;
- return;
- }
- if (parentScope == null) {
- parentScope = new ClassAd();
- }
- parentScope.setValue(scope);
+ parentScope = scope;
privateSetParentScope(scope);
}
@@ -169,8 +172,8 @@ public abstract class ExprTree {
/// A debugging method; send expression to stdout
public void puke() throws HyracksDataException {
- PrettyPrint unp = new PrettyPrint();
- AMutableCharArrayString buffer = new AMutableCharArrayString();
+ PrettyPrint unp = objectPool.prettyPrintPool.get();
+ AMutableCharArrayString buffer = objectPool.strPool.get();
unp.unparse(buffer, this);
System.out.println(buffer.toString());
}
@@ -193,16 +196,17 @@ public abstract class ExprTree {
}
public void debugFormatValue(Value value, double time) throws HyracksDataException {
- MutableBoolean boolValue = new MutableBoolean(false);
- AMutableInt64 intValue = new AMutableInt64(0);
- AMutableDouble doubleValue = new AMutableDouble(0.0);
- AMutableCharArrayString stringValue = new AMutableCharArrayString();
+ MutableBoolean boolValue = objectPool.boolPool.get();
+ AMutableInt64 intValue = objectPool.int64Pool.get();
+ AMutableDouble doubleValue = objectPool.doublePool.get();
+ AMutableCharArrayString stringValue = objectPool.strPool.get();
- if (NodeKind.CLASSAD_NODE == getKind())
+ if (NodeKind.CLASSAD_NODE == getKind()) {
return;
+ }
- PrettyPrint unp = new PrettyPrint();
- AMutableCharArrayString buffer = new AMutableCharArrayString();
+ PrettyPrint unp = objectPool.prettyPrintPool.get();
+ AMutableCharArrayString buffer = objectPool.strPool.get();
unp.unparse(buffer, this);
String result = "Classad debug: ";
@@ -230,8 +234,9 @@ public abstract class ExprTree {
result += "UNDEFINED\n";
break;
case BOOLEAN_VALUE:
- if (value.isBooleanValue(boolValue))
+ if (value.isBooleanValue(boolValue)) {
result += boolValue.booleanValue() ? "TRUE\n" : "FALSE\n";
+ }
break;
case INTEGER_VALUE:
if (value.isIntegerValue(intValue)) {
@@ -305,7 +310,7 @@ public abstract class ExprTree {
* @throws HyracksDataException
*/
public boolean publicEvaluate(Value val) throws HyracksDataException {
- EvalState state = new EvalState();
+ EvalState state = objectPool.evalStatePool.get();
if (parentScope == null) {
val.setErrorValue();
return false;
@@ -340,13 +345,13 @@ public abstract class ExprTree {
}
public boolean publicEvaluate(Value val, ExprTreeHolder sig) throws HyracksDataException {
- EvalState state = new EvalState();
+ EvalState state = objectPool.evalStatePool.get();
state.setScopes(parentScope);
return (publicEvaluate(state, val, sig));
}
public boolean publicFlatten(Value val, ExprTreeHolder tree) throws HyracksDataException {
- EvalState state = new EvalState();
+ EvalState state = objectPool.evalStatePool.get();
state.setScopes(parentScope);
return (publicFlatten(state, val, tree));
}
@@ -375,8 +380,9 @@ public abstract class ExprTree {
@Override
public String toString() {
- ClassAdUnParser unparser = new PrettyPrint();
- AMutableCharArrayString string_representation = new AMutableCharArrayString();
+ ClassAdObjectPool objectPool = new ClassAdObjectPool();
+ ClassAdUnParser unparser = new ClassAdUnParser(objectPool);
+ AMutableCharArrayString string_representation = objectPool.strPool.get();
try {
unparser.unparse(string_representation, this);
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/ExprTreeHolder.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/ExprTreeHolder.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/ExprTreeHolder.java
index 89c5c0b..0e2f894 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/ExprTreeHolder.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/ExprTreeHolder.java
@@ -18,6 +18,7 @@
*/
package org.apache.asterix.external.classad;
+import org.apache.asterix.external.classad.object.pool.ClassAdObjectPool;
import org.apache.asterix.om.base.AMutableInt32;
import org.apache.hyracks.api.exceptions.HyracksDataException;
@@ -34,9 +35,6 @@ public class ExprTreeHolder extends ExprTree {
if (tree == null) {
innerTree = null;
} else {
- if (tree.isTreeHolder()) {
- tree = ((ExprTreeHolder) tree).innerTree;
- }
if (innerTree == null) {
innerTree = tree.copy();
} else {
@@ -52,8 +50,8 @@ public class ExprTreeHolder extends ExprTree {
@Override
public void puke() throws HyracksDataException {
- PrettyPrint unp = new PrettyPrint();
- AMutableCharArrayString buffer = new AMutableCharArrayString();
+ PrettyPrint unp = objectPool.prettyPrintPool.get();
+ AMutableCharArrayString buffer = objectPool.strPool.get();
unp.unparse(buffer, innerTree);
System.out.println(buffer.toString());
}
@@ -78,11 +76,13 @@ public class ExprTreeHolder extends ExprTree {
return true;
}
- public ExprTreeHolder() {
+ public ExprTreeHolder(ClassAdObjectPool objectPool) {
+ super(objectPool);
innerTree = null;
}
- public ExprTreeHolder(ExprTree tree) {
+ public ExprTreeHolder(ExprTree tree, ClassAdObjectPool objectPool) {
+ super(objectPool);
setInnerTree(tree);
}
@@ -100,7 +100,10 @@ public class ExprTreeHolder extends ExprTree {
@Override
public ExprTree copy() throws HyracksDataException {
- return innerTree.copy();
+ if (innerTree != null) {
+ return innerTree.copy();
+ }
+ return null;
}
@Override
@@ -118,18 +121,18 @@ public class ExprTreeHolder extends ExprTree {
@Override
public boolean privateEvaluate(EvalState state, Value val) throws HyracksDataException {
- return innerTree.privateEvaluate(state, val);
+ return innerTree == null ? false : innerTree.privateEvaluate(state, val);
}
@Override
public boolean privateEvaluate(EvalState state, Value val, ExprTreeHolder tree) throws HyracksDataException {
- return innerTree.privateEvaluate(state, val, tree);
+ return innerTree == null ? false : innerTree.privateEvaluate(state, val, tree);
}
@Override
public boolean privateFlatten(EvalState state, Value val, ExprTreeHolder tree, AMutableInt32 op)
throws HyracksDataException {
- return innerTree.privateFlatten(state, val, tree, op);
+ return innerTree == null ? false : innerTree.privateFlatten(state, val, tree, op);
}
@Override
@@ -139,6 +142,8 @@ public class ExprTreeHolder extends ExprTree {
@Override
protected void privateSetParentScope(ClassAd scope) {
- innerTree.privateSetParentScope(scope);
+ if (innerTree != null) {
+ innerTree.privateSetParentScope(scope);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/FunctionCall.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/FunctionCall.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/FunctionCall.java
index bbc0e7a..234ba20 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/FunctionCall.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/FunctionCall.java
@@ -20,6 +20,7 @@ package org.apache.asterix.external.classad;
import java.util.HashMap;
+import org.apache.asterix.external.classad.object.pool.ClassAdObjectPool;
import org.apache.asterix.om.base.AMutableInt32;
import org.apache.asterix.om.base.AMutableString;
import org.apache.commons.lang3.mutable.MutableBoolean;
@@ -28,7 +29,7 @@ import org.apache.hyracks.api.exceptions.HyracksDataException;
public class FunctionCall extends ExprTree {
public static boolean initialized = false;
-
+ public static final HashMap<String, ClassAdFunc> funcTable = new HashMap<String, ClassAdFunc>();
public static final ClassAdFunc[] ClassAdBuiltinFunc = { BuiltinClassAdFunctions.IsType,
BuiltinClassAdFunctions.TestMember, BuiltinClassAdFunctions.Size, BuiltinClassAdFunctions.SumAvg,
BuiltinClassAdFunctions.MinMax, BuiltinClassAdFunctions.ListCompare, BuiltinClassAdFunctions.debug,
@@ -43,12 +44,6 @@ public class FunctionCall extends ExprTree {
BuiltinClassAdFunctions.ifThenElse, BuiltinClassAdFunctions.stringListsIntersect,
BuiltinClassAdFunctions.interval, BuiltinClassAdFunctions.eval };
- // function call specific information
- private String functionName;
- private ClassAdFunc function;
- private ExprList arguments;
- public static final HashMap<String, ClassAdFunc> funcTable = new HashMap<String, ClassAdFunc>();
-
static {
// load up the function dispatch table
// type predicates
@@ -144,47 +139,53 @@ public class FunctionCall extends ExprTree {
initialized = true;
}
- /**
- * Returns true if the function expression points to a valid
- * function in the ClassAd library.
- */
- public boolean functionIsDefined() {
- return function != null;
- }
-
- public void copyFrom(FunctionCall copiedFrom) throws HyracksDataException {
- this.function = copiedFrom.function;
- this.functionName = copiedFrom.functionName;
- if (this.arguments == null) {
- this.arguments = (ExprList) copiedFrom.arguments.copy();
- } else {
- this.arguments.copyFrom(copiedFrom.arguments);
- }
- }
+ // function call specific information
+ private final CaseInsensitiveString functionName;
+ private ClassAdFunc function;
+ private final ExprList arguments;
- public FunctionCall() {
- functionName = null;
+ public FunctionCall(ClassAdObjectPool objectPool) {
+ super(objectPool);
+ functionName = new CaseInsensitiveString();
+ arguments = new ExprList(objectPool);
function = null;
- arguments = null;
}
- public static FunctionCall createFunctionCall(String functionName, ExprList args) {
- FunctionCall fc = new FunctionCall();
+ public static FunctionCall createFunctionCall(String functionName, ExprList args, ClassAdObjectPool objectPool) {
+ FunctionCall fc = objectPool != null ? objectPool.funcPool.get() : new FunctionCall(null);
fc.function = funcTable.get(functionName.toLowerCase());
- fc.functionName = functionName;
- fc.arguments = args;
+ fc.functionName.set(functionName);
+ fc.arguments.setExprList(args.getExprList());
return fc;
}
// start up with an argument list of size 4
- public FunctionCall(FunctionCall functioncall) throws HyracksDataException {
+ public FunctionCall(FunctionCall functioncall, ClassAdObjectPool objectPool) throws HyracksDataException {
+ super(objectPool);
+ functionName = new CaseInsensitiveString();
+ arguments = new ExprList(objectPool);
+ function = null;
copyFrom(functioncall);
}
+ /**
+ * Returns true if the function expression points to a valid
+ * function in the ClassAd library.
+ */
+ public boolean functionIsDefined() {
+ return function != null;
+ }
+
+ public void copyFrom(FunctionCall copiedFrom) throws HyracksDataException {
+ this.function = copiedFrom.function;
+ this.functionName.set(copiedFrom.functionName.get());
+ this.arguments.setExprList(copiedFrom.arguments.getExprList());
+ }
+
@Override
public ExprTree copy() throws HyracksDataException {
- FunctionCall newTree = new FunctionCall();
+ FunctionCall newTree = objectPool.funcPool.get();
newTree.copyFrom(this);
return newTree;
}
@@ -192,7 +193,7 @@ public class FunctionCall extends ExprTree {
@Override
public void copyFrom(ExprTree tree) throws HyracksDataException {
FunctionCall functioncall = (FunctionCall) tree;
- functionName = functioncall.functionName;
+ functionName.set(functioncall.functionName.get());
function = functioncall.function;
arguments.copyFrom(arguments);
super.copyFrom(functioncall);
@@ -209,13 +210,17 @@ public class FunctionCall extends ExprTree {
} else if (pSelfTree.getKind() != NodeKind.FN_CALL_NODE) {
is_same = false;
} else {
- other_fn = (FunctionCall) pSelfTree;
- if (functionName == other_fn.functionName && function.equals(other_fn.function)
- && arguments.equals(other_fn.arguments)) {
- is_same = true;
+ try {
+ other_fn = (FunctionCall) pSelfTree;
+ if (functionName == other_fn.functionName && function.equals(other_fn.function)
+ && arguments.equals(other_fn.arguments)) {
+ is_same = true;
- } else {
- is_same = false;
+ } else {
+ is_same = false;
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
}
}
return is_same;
@@ -242,14 +247,14 @@ public class FunctionCall extends ExprTree {
//This will move pointers to objects (not create clones)
public void getComponents(AMutableString fn, ExprList exprList) {
- fn.setValue(functionName);
+ fn.setValue(functionName.get());
for (ExprTree tree : arguments.getExprList()) {
exprList.add(tree);
}
}
public void getComponents(AMutableCharArrayString fn, ExprList exprList) {
- fn.setValue(functionName);
+ fn.setValue(functionName.get());
for (ExprTree tree : arguments.getExprList()) {
exprList.add(tree);
}
@@ -258,7 +263,7 @@ public class FunctionCall extends ExprTree {
@Override
public boolean privateEvaluate(EvalState state, Value value) throws HyracksDataException {
if (function != null) {
- return function.call(functionName, arguments, state, value);
+ return function.call(functionName.get(), arguments, state, value, objectPool);
} else {
value.setErrorValue();
return (true);
@@ -267,19 +272,20 @@ public class FunctionCall extends ExprTree {
@Override
public boolean privateEvaluate(EvalState state, Value value, ExprTreeHolder tree) throws HyracksDataException {
- FunctionCall tmpSig = new FunctionCall();
- Value tmpVal = new Value();
- ExprTreeHolder argSig = new ExprTreeHolder();
- MutableBoolean rval = new MutableBoolean();
+ FunctionCall tmpSig = objectPool.funcPool.get();
+ Value tmpVal = objectPool.valuePool.get();
+ ExprTreeHolder argSig = objectPool.mutableExprPool.get();
+ MutableBoolean rval = objectPool.boolPool.get();
if (!privateEvaluate(state, value)) {
return false;
}
- tmpSig.functionName = functionName;
+ tmpSig.functionName.set(functionName.get());
rval.setValue(true);
for (ExprTree i : arguments.getExprList()) {
rval.setValue(i.publicEvaluate(state, tmpVal, argSig));
- if (rval.booleanValue())
+ if (rval.booleanValue()) {
tmpSig.arguments.add(argSig.getInnerTree());
+ }
}
tree.setInnerTree(tmpSig);
return rval.booleanValue();
@@ -288,11 +294,10 @@ public class FunctionCall extends ExprTree {
@Override
public boolean privateFlatten(EvalState state, Value value, ExprTreeHolder tree, AMutableInt32 i)
throws HyracksDataException {
- FunctionCall newCall = new FunctionCall();
- ExprTreeHolder argTree = new ExprTreeHolder();
- Value argValue = new Value();
+ FunctionCall newCall = objectPool.funcPool.get();
+ ExprTreeHolder argTree = objectPool.mutableExprPool.get();
+ Value argValue = objectPool.valuePool.get();
boolean fold = true;
-
tree.setInnerTree(null); // Just to be safe... wenger 2003-12-11.
// if the function cannot be resolved, the value is "error"
@@ -301,7 +306,7 @@ public class FunctionCall extends ExprTree {
return true;
}
- newCall.functionName = functionName;
+ newCall.functionName.set(functionName.get());
newCall.function = function;
// flatten the arguments
@@ -313,7 +318,7 @@ public class FunctionCall extends ExprTree {
continue;
} else {
// Assert: argTree == NULL
- argTree.setInnerTree(Literal.createLiteral(argValue));
+ argTree.setInnerTree(Literal.createLiteral(argValue, objectPool));
if (argTree.getInnerTree() != null) {
newCall.arguments.add(argTree.getInnerTree());
continue;
@@ -330,7 +335,7 @@ public class FunctionCall extends ExprTree {
// assume all functions are "pure" (i.e., side-affect free)
if (fold) {
// flattened to a value
- if (!function.call(functionName, arguments, state, value)) {
+ if (!function.call(functionName.get(), arguments, state, value, objectPool)) {
return false;
}
tree.setInnerTree(null);
@@ -349,6 +354,6 @@ public class FunctionCall extends ExprTree {
public void reset() {
this.arguments.clear();
this.function = null;
- this.functionName = "";
+ this.functionName.set("");;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/Lexer.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/Lexer.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/Lexer.java
index 97b7ea8..2ef9620 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/Lexer.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/Lexer.java
@@ -433,8 +433,9 @@ public class Lexer {
if (och == '.' || ch == '.') {
// fraction part of real or selection operator
- if (ch == '.')
+ if (ch == '.') {
wind();
+ }
if (Character.isDigit(ch)) {
// real; get digits after decimal point
numberType = NumberType.REAL;
@@ -459,8 +460,9 @@ public class Lexer {
// i.e., [eE][+-]?[0-9]+
if (numberType == NumberType.REAL && Character.toLowerCase(ch) == 'e') {
wind();
- if (ch == '+' || ch == '-')
+ if (ch == '+' || ch == '-') {
wind();
+ }
if (!Character.isDigit(ch)) {
cut();
tokenType = TokenType.LEX_TOKEN_ERROR;
[07/13] incubator-asterixdb git commit: Improve Error Handling in
Local Directory Feeds
Posted by am...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/BuiltinClassAdFunctions.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/BuiltinClassAdFunctions.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/BuiltinClassAdFunctions.java
index 1eca7e1..68bd220 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/BuiltinClassAdFunctions.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/BuiltinClassAdFunctions.java
@@ -21,7 +21,6 @@ package org.apache.asterix.external.classad;
import java.io.IOException;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
-import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Random;
@@ -32,6 +31,7 @@ import java.util.regex.Pattern;
import org.apache.asterix.external.classad.ExprTree.NodeKind;
import org.apache.asterix.external.classad.Value.ValueType;
+import org.apache.asterix.external.classad.object.pool.ClassAdObjectPool;
import org.apache.asterix.external.library.ClassAdParser;
import org.apache.asterix.om.base.AMutableDouble;
import org.apache.asterix.om.base.AMutableInt32;
@@ -43,7 +43,8 @@ public class BuiltinClassAdFunctions {
public static final ClassAdFunc IsType = new ClassAdFunc() {
@Override
- public boolean call(String name, ExprList argList, EvalState state, Value val) throws HyracksDataException {
+ public boolean call(String name, ExprList argList, EvalState state, Value val, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
// need a single argument
if (argList.size() != 1) {
val.setErrorValue();
@@ -95,13 +96,14 @@ public class BuiltinClassAdFunctions {
public static final ClassAdFunc TestMember = new ClassAdFunc() {
@Override
- public boolean call(String name, ExprList argList, EvalState state, Value val) throws HyracksDataException {
- Value arg0 = new Value();
- Value arg1 = new Value();
- Value cArg = new Value();
-
- ExprList el = new ExprList();
- MutableBoolean b = new MutableBoolean();
+ public boolean call(String name, ExprList argList, EvalState state, Value val, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ Value arg0 = objectPool.valuePool.get();
+ Value arg1 = objectPool.valuePool.get();
+ Value cArg = objectPool.valuePool.get();
+
+ ExprList el = objectPool.exprListPool.get();
+ MutableBoolean b = objectPool.boolPool.get();
boolean useIS = name.equalsIgnoreCase("identicalmember");
// need two arguments
@@ -126,10 +128,10 @@ public class BuiltinClassAdFunctions {
// Swap
if (arg0.isListValue() && !arg1.isListValue()) {
- Value swap = new Value();
- swap.copyFrom(arg0);
- arg0.copyFrom(arg1);
- arg1.copyFrom(swap);
+ Value swap = objectPool.valuePool.get();
+ swap.setValue(arg0);
+ arg0.setValue(arg1);
+ arg1.setValue(swap);
}
// arg1 must be a list; arg0 must be comparable
@@ -151,7 +153,8 @@ public class BuiltinClassAdFunctions {
val.setErrorValue();
return (false);
}
- Operation.operate(useIS ? Operation.OpKind_IS_OP : Operation.OpKind_EQUAL_OP, cArg, arg0, val);
+ Operation.operate(useIS ? Operation.OpKind_IS_OP : Operation.OpKind_EQUAL_OP, cArg, arg0, val,
+ objectPool);
if (val.isBooleanValue(b) && b.booleanValue()) {
return true;
}
@@ -163,11 +166,13 @@ public class BuiltinClassAdFunctions {
public static final ClassAdFunc Size = new ClassAdFunc() {
@Override
- public boolean call(String name, ExprList argList, EvalState state, Value val) throws HyracksDataException {
- Value arg = new Value();
- ExprList listToSize = new ExprList();
- ClassAd classadToSize = new ClassAd();
- AMutableInt32 length = new AMutableInt32(0);
+ public boolean call(String name, ExprList argList, EvalState state, Value val, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ Value arg = objectPool.valuePool.get();
+ ExprList listToSize = objectPool.exprListPool.get();
+ ClassAd classadToSize = objectPool.classAdPool.get();
+ AMutableInt32 length = objectPool.int32Pool.get();
+ length.setValue(0);
// we accept only one argument
if (argList.size() != 1) {
val.setErrorValue();
@@ -197,15 +202,17 @@ public class BuiltinClassAdFunctions {
public static final ClassAdFunc SumAvg = new ClassAdFunc() {
@Override
- public boolean call(String name, ExprList argList, EvalState state, Value val) throws HyracksDataException {
-
- Value listElementValue = new Value();
- Value listVal = new Value();
- Value numElements = new Value();
- Value result = new Value();
- ExprList listToSum = new ExprList();
- MutableBoolean first = new MutableBoolean();
- AMutableInt64 len = new AMutableInt64(0);
+ public boolean call(String name, ExprList argList, EvalState state, Value val, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+
+ Value listElementValue = objectPool.valuePool.get();
+ Value listVal = objectPool.valuePool.get();
+ Value numElements = objectPool.valuePool.get();
+ Value result = objectPool.valuePool.get();
+ ExprList listToSum = objectPool.exprListPool.get();
+ MutableBoolean first = objectPool.boolPool.get();
+ AMutableInt64 len = objectPool.int64Pool.get();
+ len.setValue(0);
boolean onlySum = name.equalsIgnoreCase("sum");
// we accept only one argument
@@ -245,28 +252,28 @@ public class BuiltinClassAdFunctions {
// Either take the number if it's the first,
// or add to the running sum.
if (first.booleanValue()) {
- result.copyFrom(listElementValue);
+ result.setValue(listElementValue);
first.setValue(false);
} else {
- Operation.operate(Operation.OpKind_ADDITION_OP, result, listElementValue, result);
+ Operation.operate(Operation.OpKind_ADDITION_OP, result, listElementValue, result, objectPool);
}
}
// if the sum() function was called, we don't need to find the average
if (onlySum) {
- val.copyFrom(result);
+ val.setValue(result);
return true;
}
if (len.getLongValue() > 0) {
numElements.setRealValue(len.getLongValue());
- Operation.operate(Operation.OpKind_DIVISION_OP, result, numElements, result);
+ Operation.operate(Operation.OpKind_DIVISION_OP, result, numElements, result, objectPool);
} else {
val.setUndefinedValue();
}
- val.copyFrom(result);
+ val.setValue(result);
return true;
}
@@ -274,14 +281,16 @@ public class BuiltinClassAdFunctions {
public static final ClassAdFunc MinMax = new ClassAdFunc() {
@Override
- public boolean call(String fn, ExprList argList, EvalState state, Value val) throws HyracksDataException {
- Value listElementValue = new Value();
- Value listVal = new Value();
- Value cmp = new Value();
- Value result = new Value();
- ExprList listToBound = new ExprList();
+ public boolean call(String fn, ExprList argList, EvalState state, Value val, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ Value listElementValue = objectPool.valuePool.get();
+ Value listVal = objectPool.valuePool.get();
+ Value cmp = objectPool.valuePool.get();
+ Value result = objectPool.valuePool.get();
+ ExprList listToBound = objectPool.exprListPool.get();
boolean first = true;
- MutableBoolean b = new MutableBoolean(false);
+ MutableBoolean b = objectPool.boolPool.get();
+ b.setValue(false);
int comparisonOperator;
// we accept only one argument
@@ -326,17 +335,17 @@ public class BuiltinClassAdFunctions {
// If it's the first element, copy it to the bound,
// otherwise compare to decide what to do.
if (first) {
- result.copyFrom(listElementValue);
+ result.setValue(listElementValue);
first = false;
} else {
- Operation.operate(comparisonOperator, listElementValue, result, cmp);
+ Operation.operate(comparisonOperator, listElementValue, result, cmp, objectPool);
if (cmp.isBooleanValue(b) && b.booleanValue()) {
- result.copyFrom(listElementValue);
+ result.setValue(listElementValue);
}
}
}
- val.copyFrom(result);
+ val.setValue(result);
return true;
}
@@ -344,15 +353,16 @@ public class BuiltinClassAdFunctions {
public static final ClassAdFunc ListCompare = new ClassAdFunc() {
@Override
- public boolean call(String fn, ExprList argList, EvalState state, Value val) throws HyracksDataException {
-
- Value listElementValue = new Value();
- Value listVal = new Value();
- Value compareVal = new Value();
- Value stringValue = new Value();
- ExprList listToCompare = new ExprList();
+ public boolean call(String fn, ExprList argList, EvalState state, Value val, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+
+ Value listElementValue = objectPool.valuePool.get();
+ Value listVal = objectPool.valuePool.get();
+ Value compareVal = objectPool.valuePool.get();
+ Value stringValue = objectPool.valuePool.get();
+ ExprList listToCompare = objectPool.exprListPool.get();
boolean needAllMatch;
- AMutableCharArrayString comparison_string = new AMutableCharArrayString();
+ AMutableCharArrayString comparison_string = objectPool.strPool.get();
int comparisonOperator;
// We take three arguments:
@@ -445,10 +455,10 @@ public class BuiltinClassAdFunctions {
val.setErrorValue();
return false;
} else {
- Value compareResult = new Value();
- MutableBoolean b = new MutableBoolean();
+ Value compareResult = objectPool.valuePool.get();
+ MutableBoolean b = objectPool.boolPool.get();
- Operation.operate(comparisonOperator, listElementValue, compareVal, compareResult);
+ Operation.operate(comparisonOperator, listElementValue, compareVal, compareResult, objectPool);
if (!compareResult.isBooleanValue(b)) {
if (compareResult.isUndefinedValue()) {
if (needAllMatch) {
@@ -491,32 +501,37 @@ public class BuiltinClassAdFunctions {
};
public static final ClassAdFunc timeZoneOffset = new ClassAdFunc() {
@Override
- public boolean call(String name, ExprList argList, EvalState state, Value val) throws HyracksDataException {
+ public boolean call(String name, ExprList argList, EvalState state, Value val, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
// no arguments
if (argList.size() > 0) {
val.setErrorValue();
return (true);
}
- val.setRelativeTimeValue(new ClassAdTime());
+ val.setRelativeTimeValue(objectPool.classAdTimePool.get());
return (true);
}
};
public static final ClassAdFunc debug = new ClassAdFunc() {
@Override
- public boolean call(String name, ExprList argList, EvalState state, Value result) throws HyracksDataException {
+ public boolean call(String name, ExprList argList, EvalState state, Value result, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
return false;
}
};
public static final ClassAdFunc formatTime = new ClassAdFunc() {
@Override
- public boolean call(String name, ExprList argList, EvalState state, Value result) throws HyracksDataException {
- Value time_arg = new Value();
- Value format_arg = new Value();
- AMutableInt64 int64 = new AMutableInt64(0);
- ClassAdTime epoch_time = new ClassAdTime();
- ClassAdTime time_components = new ClassAdTime("GMT");
- ClassAd splitClassAd = new ClassAd();
+ public boolean call(String name, ExprList argList, EvalState state, Value result, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ Value time_arg = objectPool.valuePool.get();
+ Value format_arg = objectPool.valuePool.get();
+ AMutableInt64 int64 = objectPool.int64Pool.get();
+ int64.setValue(0);
+ ClassAdTime epoch_time = objectPool.classAdTimePool.get();
+ ClassAdTime time_components = objectPool.classAdTimePool.get();
+ time_components.setTimeZone("GMT");
+ ClassAd splitClassAd = objectPool.classAdPool.get();
String format;
int number_of_args;
boolean did_eval;
@@ -535,8 +550,7 @@ public class BuiltinClassAdFunctions {
} else if (time_arg.isRelativeTimeValue()) {
result.setErrorValue();
} else if (time_arg.isAbsoluteTimeValue(time_components)) {
- } else if (!time_arg
- .isClassAdValue(splitClassAd) /* doSplitTime(time_arg, splitClassAd) */) {
+ } else if (!time_arg.isClassAdValue(splitClassAd) /* doSplitTime(time_arg, splitClassAd) */) {
result.setErrorValue();
} else {
if (!splitClassAd.evaluateAttrInt("Seconds", int64)) {
@@ -579,7 +593,7 @@ public class BuiltinClassAdFunctions {
if (!argList.get(1).publicEvaluate(state, format_arg)) {
did_eval = false;
} else {
- AMutableCharArrayString formatString = new AMutableCharArrayString();
+ AMutableCharArrayString formatString = objectPool.strPool.get();
if (!format_arg.isStringValue(formatString)) {
result.setErrorValue();
} else {
@@ -618,12 +632,13 @@ public class BuiltinClassAdFunctions {
public static final ClassAdFunc getField = new ClassAdFunc() {
@Override
- public boolean call(String name, ExprList argList, EvalState state, Value val) throws HyracksDataException {
- Value arg = new Value();
- ClassAdTime asecs = new ClassAdTime();
- ClassAdTime rsecs = new ClassAdTime();
- ClassAdTime clock = new ClassAdTime();
- ClassAdTime tms = new ClassAdTime();
+ public boolean call(String name, ExprList argList, EvalState state, Value val, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ Value arg = objectPool.valuePool.get();
+ ClassAdTime asecs = objectPool.classAdTimePool.get();
+ ClassAdTime rsecs = objectPool.classAdTimePool.get();
+ ClassAdTime clock = objectPool.classAdTimePool.get();
+ ClassAdTime tms = objectPool.classAdTimePool.get();
if (argList.size() != 1) {
val.setErrorValue();
@@ -708,13 +723,14 @@ public class BuiltinClassAdFunctions {
};
public static final ClassAdFunc currentTime = new ClassAdFunc() {
@Override
- public boolean call(String name, ExprList argList, EvalState state, Value val) throws HyracksDataException {
+ public boolean call(String name, ExprList argList, EvalState state, Value val, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
// no arguments
if (argList.size() > 0) {
val.setErrorValue();
return (true);
}
- Literal time_literal = Literal.createAbsTime(new ClassAdTime());
+ Literal time_literal = Literal.createAbsTime(objectPool.classAdTimePool.get(), objectPool);
time_literal.GetValue(val);
return true;
}
@@ -722,9 +738,10 @@ public class BuiltinClassAdFunctions {
public static final ClassAdFunc splitTime = new ClassAdFunc() {
@Override
- public boolean call(String name, ExprList argList, EvalState state, Value result) throws HyracksDataException {
- Value arg = new Value();
- ClassAd split = new ClassAd();
+ public boolean call(String name, ExprList argList, EvalState state, Value result, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ Value arg = objectPool.valuePool.get();
+ ClassAd split = objectPool.classAdPool.get();
if (argList.size() != 1) {
result.setErrorValue();
@@ -736,7 +753,7 @@ public class BuiltinClassAdFunctions {
return false;
}
- if (!arg.isClassAdValue() && doSplitTime(arg, split)) {
+ if (!arg.isClassAdValue() && doSplitTime(arg, split, objectPool)) {
result.setClassAdValue(split);
} else {
result.setErrorValue();
@@ -746,13 +763,16 @@ public class BuiltinClassAdFunctions {
};
- public static boolean doSplitTime(Value time, ClassAd splitClassAd) throws HyracksDataException {
+ public static boolean doSplitTime(Value time, ClassAd splitClassAd, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
boolean did_conversion;
- AMutableInt64 integer = new AMutableInt64(0);
- AMutableDouble real = new AMutableDouble(0);
- ClassAdTime asecs = new ClassAdTime();
- ClassAdTime rsecs = new ClassAdTime();
- ClassAd classad = new ClassAd();
+ AMutableInt64 integer = objectPool.int64Pool.get();
+ integer.setValue(0);
+ AMutableDouble real = objectPool.doublePool.get();
+ real.setValue(0);
+ ClassAdTime asecs = objectPool.classAdTimePool.get();
+ ClassAdTime rsecs = objectPool.classAdTimePool.get();
+ ClassAd classad = objectPool.classAdPool.get();
did_conversion = true;
if (time.isIntegerValue(integer)) {
asecs.setValue(integer.getLongValue());
@@ -767,7 +787,7 @@ public class BuiltinClassAdFunctions {
} else if (time.isRelativeTimeValue(rsecs)) {
relTimeToClassAd((rsecs.getRelativeTime() / 1000.0), splitClassAd);
} else if (time.isClassAdValue(classad)) {
- splitClassAd = new ClassAd();
+ splitClassAd = objectPool.classAdPool.get();
splitClassAd.copyFrom(classad);
} else {
did_conversion = false;
@@ -828,14 +848,16 @@ public class BuiltinClassAdFunctions {
public static final ClassAdFunc dayTime = new ClassAdFunc() {
@Override
- public boolean call(String name, ExprList argList, EvalState state, Value val) throws HyracksDataException {
- val.setRelativeTimeValue(new ClassAdTime());
+ public boolean call(String name, ExprList argList, EvalState state, Value val, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ val.setRelativeTimeValue(objectPool.classAdTimePool.get());
return (true);
}
};
public static final ClassAdFunc epochTime = new ClassAdFunc() {
@Override
- public boolean call(String name, ExprList argList, EvalState state, Value val) throws HyracksDataException {
+ public boolean call(String name, ExprList argList, EvalState state, Value val, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
// no arguments
if (argList.size() > 0) {
val.setErrorValue();
@@ -847,15 +869,16 @@ public class BuiltinClassAdFunctions {
};
public static final ClassAdFunc strCat = new ClassAdFunc() {
@Override
- public boolean call(String name, ExprList argList, EvalState state, Value result) throws HyracksDataException {
- AMutableCharArrayString buf = new AMutableCharArrayString();
- AMutableCharArrayString s = new AMutableCharArrayString();
+ public boolean call(String name, ExprList argList, EvalState state, Value result, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ AMutableCharArrayString buf = objectPool.strPool.get();
+ AMutableCharArrayString s = objectPool.strPool.get();
boolean errorFlag = false;
boolean undefFlag = false;
boolean rval = false;
- Value val = new Value();
- Value stringVal = new Value();
+ Value val = objectPool.valuePool.get();
+ Value stringVal = objectPool.valuePool.get();
for (int i = 0; i < argList.size(); i++) {
@@ -867,7 +890,7 @@ public class BuiltinClassAdFunctions {
if (val.isStringValue(s)) {
buf.appendString(s);
} else {
- Value.convertValueToStringValue(val, stringVal);
+ Value.convertValueToStringValue(val, stringVal, objectPool);
if (stringVal.isUndefinedValue()) {
undefFlag = true;
break;
@@ -906,10 +929,11 @@ public class BuiltinClassAdFunctions {
};
public static final ClassAdFunc changeCase = new ClassAdFunc() {
@Override
- public boolean call(String name, ExprList argList, EvalState state, Value result) throws HyracksDataException {
- Value val = new Value();
- Value stringVal = new Value();
- AMutableCharArrayString str = new AMutableCharArrayString();
+ public boolean call(String name, ExprList argList, EvalState state, Value result, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ Value val = objectPool.valuePool.get();
+ Value stringVal = objectPool.valuePool.get();
+ AMutableCharArrayString str = objectPool.strPool.get();
boolean lower = name.equalsIgnoreCase("tolower");
int len;
@@ -926,7 +950,7 @@ public class BuiltinClassAdFunctions {
}
if (!val.isStringValue(str)) {
- Value.convertValueToStringValue(val, stringVal);
+ Value.convertValueToStringValue(val, stringVal, objectPool);
if (stringVal.isUndefinedValue()) {
result.setUndefinedValue();
return true;
@@ -949,14 +973,18 @@ public class BuiltinClassAdFunctions {
public static final ClassAdFunc subString = new ClassAdFunc() {
@Override
- public boolean call(String name, ExprList argList, EvalState state, Value result) throws HyracksDataException {
- Value arg0 = new Value();
- Value arg1 = new Value();
- Value arg2 = new Value();
- AMutableCharArrayString buf = new AMutableCharArrayString();
- AMutableInt64 offset = new AMutableInt64(0);
- AMutableInt64 len = new AMutableInt64(0);
- AMutableInt64 alen = new AMutableInt64(0);
+ public boolean call(String name, ExprList argList, EvalState state, Value result, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ Value arg0 = objectPool.valuePool.get();
+ Value arg1 = objectPool.valuePool.get();
+ Value arg2 = objectPool.valuePool.get();
+ AMutableCharArrayString buf = objectPool.strPool.get();
+ AMutableInt64 offset = objectPool.int64Pool.get();
+ offset.setValue(0);
+ AMutableInt64 len = objectPool.int64Pool.get();
+ len.setValue(0);
+ AMutableInt64 alen = objectPool.int64Pool.get();
+ len.setValue(0);
// two or three arguments
if (argList.size() < 2 || argList.size() > 3) {
@@ -1004,10 +1032,12 @@ public class BuiltinClassAdFunctions {
// to make sure that if length is specified as 0 explicitly
// then, len is set to 0
if (argList.size() == 3) {
- AMutableInt64 templen = new AMutableInt64(0);
+ AMutableInt64 templen = objectPool.int64Pool.get();
+ templen.setValue(0);
arg2.isIntegerValue(templen);
- if (templen.getLongValue() == 0)
+ if (templen.getLongValue() == 0) {
len.setValue(0);
+ }
}
result.setStringValue(buf.substr((int) offset.getLongValue(), (int) len.getLongValue()));
return (true);
@@ -1016,8 +1046,9 @@ public class BuiltinClassAdFunctions {
public static final ClassAdFunc convInt = new ClassAdFunc() {
@Override
- public boolean call(String name, ExprList argList, EvalState state, Value result) throws HyracksDataException {
- Value arg = new Value();
+ public boolean call(String name, ExprList argList, EvalState state, Value result, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ Value arg = objectPool.valuePool.get();
// takes exactly one argument
if (argList.size() != 1) {
result.setErrorValue();
@@ -1027,18 +1058,19 @@ public class BuiltinClassAdFunctions {
result.setErrorValue();
return (false);
}
- Value.convertValueToIntegerValue(arg, result);
+ Value.convertValueToIntegerValue(arg, result, objectPool);
return true;
}
};
public static final ClassAdFunc compareString = new ClassAdFunc() {
@Override
- public boolean call(String name, ExprList argList, EvalState state, Value result) throws HyracksDataException {
- Value arg0 = new Value();
- Value arg1 = new Value();
- Value arg0_s = new Value();
- Value arg1_s = new Value();
+ public boolean call(String name, ExprList argList, EvalState state, Value result, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ Value arg0 = objectPool.valuePool.get();
+ Value arg1 = objectPool.valuePool.get();
+ Value arg0_s = objectPool.valuePool.get();
+ Value arg1_s = objectPool.valuePool.get();
// Must have two arguments
if (argList.size() != 2) {
@@ -1059,25 +1091,28 @@ public class BuiltinClassAdFunctions {
return true;
}
- AMutableCharArrayString s0 = new AMutableCharArrayString();
- AMutableCharArrayString s1 = new AMutableCharArrayString();
- if (Value.convertValueToStringValue(arg0, arg0_s) && Value.convertValueToStringValue(arg1, arg1_s)
- && arg0_s.isStringValue(s0) && arg1_s.isStringValue(s1)) {
+ AMutableCharArrayString s0 = objectPool.strPool.get();
+ AMutableCharArrayString s1 = objectPool.strPool.get();
+ if (Value.convertValueToStringValue(arg0, arg0_s, objectPool)
+ && Value.convertValueToStringValue(arg1, arg1_s, objectPool) && arg0_s.isStringValue(s0)
+ && arg1_s.isStringValue(s1)) {
int order;
if (name.equalsIgnoreCase("strcmp")) {
order = s0.compareTo(s1);
- if (order < 0)
+ if (order < 0) {
order = -1;
- else if (order > 0)
+ } else if (order > 0) {
order = 1;
+ }
} else {
order = s0.compareToIgnoreCase(s1);
- if (order < 0)
+ if (order < 0) {
order = -1;
- else if (order > 0)
+ } else if (order > 0) {
order = 1;
+ }
}
result.setIntegerValue(order);
} else {
@@ -1090,15 +1125,16 @@ public class BuiltinClassAdFunctions {
public static final ClassAdFunc matchPattern = new ClassAdFunc() {
@Override
- public boolean call(String name, ExprList argList, EvalState state, Value result) throws HyracksDataException {
+ public boolean call(String name, ExprList argList, EvalState state, Value result, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
boolean have_options;
- Value arg0 = new Value();
- Value arg1 = new Value();
- Value arg2 = new Value();
+ Value arg0 = objectPool.valuePool.get();
+ Value arg1 = objectPool.valuePool.get();
+ Value arg2 = objectPool.valuePool.get();
- AMutableCharArrayString pattern = new AMutableCharArrayString();
- AMutableCharArrayString target = new AMutableCharArrayString();
- AMutableCharArrayString options_string = new AMutableCharArrayString();
+ AMutableCharArrayString pattern = objectPool.strPool.get();
+ AMutableCharArrayString target = objectPool.strPool.get();
+ AMutableCharArrayString options_string = objectPool.strPool.get();
// need two or three arguments: pattern, string, optional settings
if (argList.size() != 2 && argList.size() != 3) {
@@ -1184,8 +1220,9 @@ public class BuiltinClassAdFunctions {
public static final ClassAdFunc convReal = new ClassAdFunc() {
@Override
- public boolean call(String name, ExprList argList, EvalState state, Value result) throws HyracksDataException {
- Value arg = new Value();
+ public boolean call(String name, ExprList argList, EvalState state, Value result, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ Value arg = objectPool.valuePool.get();
// takes exactly one argument
if (argList.size() != 1) {
@@ -1196,15 +1233,16 @@ public class BuiltinClassAdFunctions {
result.setErrorValue();
return (false);
}
- Value.convertValueToRealValue(arg, result);
+ Value.convertValueToRealValue(arg, result, objectPool);
return true;
}
};
public static final ClassAdFunc convString = new ClassAdFunc() {
@Override
- public boolean call(String name, ExprList argList, EvalState state, Value result) throws HyracksDataException {
- Value arg = new Value();
+ public boolean call(String name, ExprList argList, EvalState state, Value result, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ Value arg = objectPool.valuePool.get();
// takes exactly one argument
if (argList.size() != 1) {
@@ -1216,21 +1254,22 @@ public class BuiltinClassAdFunctions {
return (false);
}
- Value.convertValueToStringValue(arg, result);
+ Value.convertValueToStringValue(arg, result, objectPool);
return true;
}
};
public static final ClassAdFunc unparse = new ClassAdFunc() {
@Override
- public boolean call(String name, ExprList argList, EvalState state, Value result) throws HyracksDataException {
+ public boolean call(String name, ExprList argList, EvalState state, Value result, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
if (argList.size() != 1 || argList.get(0).getKind() != NodeKind.ATTRREF_NODE) {
result.setErrorValue();
} else {
// use the printpretty on arg0 to spew out
- PrettyPrint unp = new PrettyPrint();
- AMutableCharArrayString szAttribute = new AMutableCharArrayString();
- AMutableCharArrayString szValue = new AMutableCharArrayString();
+ PrettyPrint unp = objectPool.prettyPrintPool.get();
+ AMutableCharArrayString szAttribute = objectPool.strPool.get();
+ AMutableCharArrayString szValue = objectPool.strPool.get();
ExprTree pTree;
unp.unparse(szAttribute, argList.get(0));
@@ -1247,8 +1286,9 @@ public class BuiltinClassAdFunctions {
};
public static final ClassAdFunc convBool = new ClassAdFunc() {
@Override
- public boolean call(String name, ExprList argList, EvalState state, Value result) throws HyracksDataException {
- Value arg = new Value();
+ public boolean call(String name, ExprList argList, EvalState state, Value result, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ Value arg = objectPool.valuePool.get();
// takes exactly one argument
if (argList.size() != 1) {
result.setErrorValue();
@@ -1273,25 +1313,26 @@ public class BuiltinClassAdFunctions {
return (true);
case BOOLEAN_VALUE:
- result.copyFrom(arg);
+ result.setValue(arg);
return (true);
case INTEGER_VALUE: {
- AMutableInt64 ival = new AMutableInt64(0);
+ AMutableInt64 ival = objectPool.int64Pool.get();
+ ival.setValue(0);
arg.isIntegerValue(ival);
result.setBooleanValue(ival.getLongValue() != 0);
return (true);
}
case REAL_VALUE: {
- AMutableDouble rval = new AMutableDouble(0);
+ AMutableDouble rval = objectPool.doublePool.get();
arg.isRealValue(rval);
result.setBooleanValue(rval.getDoubleValue() != 0.0);
return (true);
}
case STRING_VALUE: {
- AMutableCharArrayString buf = new AMutableCharArrayString();
+ AMutableCharArrayString buf = objectPool.strPool.get();
arg.isStringValue(buf);
if (buf.equalsIgnoreCase("false") || buf.size() == 0) {
result.setBooleanValue(false);
@@ -1302,7 +1343,7 @@ public class BuiltinClassAdFunctions {
}
case RELATIVE_TIME_VALUE: {
- ClassAdTime rsecs = new ClassAdTime();
+ ClassAdTime rsecs = objectPool.classAdTimePool.get();
arg.isRelativeTimeValue(rsecs);
result.setBooleanValue(rsecs.getTimeInMillis() != 0);
return (true);
@@ -1315,16 +1356,18 @@ public class BuiltinClassAdFunctions {
};
public static final ClassAdFunc convTime = new ClassAdFunc() {
@Override
- public boolean call(String name, ExprList argList, EvalState state, Value result) throws HyracksDataException {
- Value arg = new Value();
- Value arg2 = new Value();
+ public boolean call(String name, ExprList argList, EvalState state, Value result, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ Value arg = objectPool.valuePool.get();
+ Value arg2 = objectPool.valuePool.get();
boolean relative = name.equalsIgnoreCase("reltime");
boolean secondarg = false; // says whether a 2nd argument exists
- AMutableInt64 arg2num = new AMutableInt64(0);
+ AMutableInt64 arg2num = objectPool.int64Pool.get();
+ arg2num.setValue(0);
if (argList.size() == 0 && !relative) {
// absTime with no arguments returns the current time.
- return currentTime.call(name, argList, state, result);
+ return currentTime.call(name, argList, state, result, objectPool);
}
if ((argList.size() < 1) || (argList.size() > 2)) {
result.setErrorValue();
@@ -1340,9 +1383,10 @@ public class BuiltinClassAdFunctions {
result.setErrorValue();
return (false);
}
- AMutableInt64 ivalue2 = new AMutableInt64(0);
- AMutableDouble rvalue2 = new AMutableDouble(0);
- ClassAdTime rsecs = new ClassAdTime();
+ AMutableInt64 ivalue2 = objectPool.int64Pool.get();
+ ivalue2.setValue(0);
+ AMutableDouble rvalue2 = objectPool.doublePool.get();
+ ClassAdTime rsecs = objectPool.classAdTimePool.get();
if (relative) {// 2nd argument is N/A for reltime
result.setErrorValue();
return (true);
@@ -1377,41 +1421,49 @@ public class BuiltinClassAdFunctions {
return (true);
case INTEGER_VALUE: {
- AMutableInt64 ivalue = new AMutableInt64(0);
+ AMutableInt64 ivalue = objectPool.int64Pool.get();
+ ivalue.setValue(0);
arg.isIntegerValue(ivalue);
if (relative) {
- result.setRelativeTimeValue(new ClassAdTime(ivalue.getLongValue(), false));
+ ClassAdTime time = objectPool.classAdTimePool.get();
+ time.setRelativeTime(ivalue.getLongValue());
+ result.setRelativeTimeValue(time);
} else {
- ClassAdTime atvalue = new ClassAdTime(true);
+ ClassAdTime atvalue = objectPool.classAdTimePool.get();
atvalue.setValue(ivalue.getLongValue());
- if (secondarg) //2nd arg is the offset in secs
+ if (secondarg) {
atvalue.setTimeZone((int) arg2num.getLongValue());
- else
+ } else {
// the default offset is the current timezone
atvalue.setTimeZone(Literal.findOffset(atvalue));
+ }
if (atvalue.getOffset() == -1) {
result.setErrorValue();
return (false);
- } else
+ } else {
result.setAbsoluteTimeValue(atvalue);
+ }
}
return (true);
}
case REAL_VALUE: {
- AMutableDouble rvalue = new AMutableDouble(0);
+ AMutableDouble rvalue = objectPool.doublePool.get();
arg.isRealValue(rvalue);
if (relative) {
- result.setRelativeTimeValue(new ClassAdTime((long) (1000 * rvalue.getDoubleValue()), false));
+ ClassAdTime time = objectPool.classAdTimePool.get();
+ time.setRelativeTime((long) (1000 * rvalue.getDoubleValue()));
+ result.setRelativeTimeValue(time);
} else {
- ClassAdTime atvalue = new ClassAdTime();
+ ClassAdTime atvalue = objectPool.classAdTimePool.get();
atvalue.setValue((long) rvalue.getDoubleValue());
- if (secondarg) //2nd arg is the offset in secs
+ if (secondarg) {
atvalue.setTimeZone((int) arg2num.getLongValue());
- else
+ } else {
// the default offset is the current timezone
atvalue.setTimeZone(Literal.findOffset(atvalue));
+ }
result.setAbsoluteTimeValue(atvalue);
}
return (true);
@@ -1423,28 +1475,29 @@ public class BuiltinClassAdFunctions {
}
case ABSOLUTE_TIME_VALUE: {
- ClassAdTime secs = new ClassAdTime();
+ ClassAdTime secs = objectPool.classAdTimePool.get();
arg.isAbsoluteTimeValue(secs);
if (relative) {
result.setRelativeTimeValue(secs);
} else {
- result.copyFrom(arg);
+ result.setValue(arg);
}
return (true);
}
case RELATIVE_TIME_VALUE: {
if (relative) {
- result.copyFrom(arg);
+ result.setValue(arg);
} else {
- ClassAdTime secs = new ClassAdTime();
+ ClassAdTime secs = objectPool.classAdTimePool.get();
arg.isRelativeTimeValue(secs);
- ClassAdTime atvalue = new ClassAdTime();
+ ClassAdTime atvalue = objectPool.classAdTimePool.get();
atvalue.setValue(secs);
- if (secondarg) //2nd arg is the offset in secs
+ if (secondarg) {
atvalue.setTimeZone((int) arg2num.getLongValue());
- else
+ } else {
// the default offset is the current timezone
atvalue.setTimeZone(Literal.findOffset(atvalue));
+ }
result.setAbsoluteTimeValue(atvalue);
}
return (true);
@@ -1458,9 +1511,10 @@ public class BuiltinClassAdFunctions {
public static final ClassAdFunc doRound = new ClassAdFunc() {
@Override
- public boolean call(String name, ExprList argList, EvalState state, Value result) throws HyracksDataException {
- Value arg = new Value();
- Value realValue = new Value();
+ public boolean call(String name, ExprList argList, EvalState state, Value result, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ Value arg = objectPool.valuePool.get();
+ Value realValue = objectPool.valuePool.get();
// takes exactly one argument
if (argList.size() != 1) {
result.setErrorValue();
@@ -1471,12 +1525,12 @@ public class BuiltinClassAdFunctions {
return (false);
}
if (arg.getType() == ValueType.INTEGER_VALUE) {
- result.copyFrom(arg);
+ result.setValue(arg);
} else {
- if (!Value.convertValueToRealValue(arg, realValue)) {
+ if (!Value.convertValueToRealValue(arg, realValue, objectPool)) {
result.setErrorValue();
} else {
- AMutableDouble rvalue = new AMutableDouble(0);
+ AMutableDouble rvalue = objectPool.doublePool.get();
realValue.isRealValue(rvalue);
if (name.equalsIgnoreCase("floor")) {
result.setIntegerValue((long) Math.floor(rvalue.getDoubleValue()));
@@ -1495,9 +1549,10 @@ public class BuiltinClassAdFunctions {
public static final ClassAdFunc doMath2 = new ClassAdFunc() {
@Override
- public boolean call(String name, ExprList argList, EvalState state, Value result) throws HyracksDataException {
- Value arg = new Value();
- Value arg2 = new Value();
+ public boolean call(String name, ExprList argList, EvalState state, Value result, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ Value arg = objectPool.valuePool.get();
+ Value arg2 = objectPool.valuePool.get();
// takes 2 arguments pow(val,base)
if (argList.size() != 2) {
@@ -1511,20 +1566,24 @@ public class BuiltinClassAdFunctions {
if (name.equalsIgnoreCase("pow")) {
// take arg2 to the power of arg2
- AMutableInt64 ival = new AMutableInt64(0);
- AMutableInt64 ibase = new AMutableInt64(0);
+ AMutableInt64 ival = objectPool.int64Pool.get();
+ ival.setValue(0);
+ AMutableInt64 ibase = objectPool.int64Pool.get();
+ ibase.setValue(0);
if (arg.isIntegerValue(ival) && arg2.isIntegerValue(ibase) && ibase.getLongValue() >= 0) {
ival.setValue((long) (Math.pow(ival.getLongValue(), ibase.getLongValue())));
result.setIntegerValue(ival.getLongValue());
} else {
- Value realValue = new Value();
- Value realBase = new Value();
- if (!Value.convertValueToRealValue(arg, realValue)
- || !Value.convertValueToRealValue(arg2, realBase)) {
+ Value realValue = objectPool.valuePool.get();
+ Value realBase = objectPool.valuePool.get();
+ if (!Value.convertValueToRealValue(arg, realValue, objectPool)
+ || !Value.convertValueToRealValue(arg2, realBase, objectPool)) {
result.setErrorValue();
} else {
- AMutableDouble rvalue = new AMutableDouble(0);
- AMutableDouble rbase = new AMutableDouble(1);
+ AMutableDouble rvalue = objectPool.doublePool.get();
+ rvalue.setValue(0);
+ AMutableDouble rbase = objectPool.doublePool.get();
+ rbase.setValue(1);
realValue.isRealValue(rvalue);
realBase.isRealValue(rbase);
result.setRealValue(Math.pow(rvalue.getDoubleValue(), rbase.getDoubleValue()));
@@ -1535,17 +1594,17 @@ public class BuiltinClassAdFunctions {
// if arg2 is a list, choose the first item from the list that is larger than arg1
// if arg1 is larger than all of the items in the list, the result is an error.
- Value val = new Value();
- Value base = new Value();
- if (!Value.convertValueToRealValue(arg, val)) {
+ Value val = objectPool.valuePool.get();
+ Value base = objectPool.valuePool.get();
+ if (!Value.convertValueToRealValue(arg, val, objectPool)) {
result.setErrorValue();
} else {
// get the value to quantize into rval.
- AMutableDouble rval = new AMutableDouble(0);
- AMutableDouble rbase = new AMutableDouble(0);
+ AMutableDouble rval = objectPool.doublePool.get();
+ AMutableDouble rbase = objectPool.doublePool.get();
val.isRealValue(rval);
if (arg2.isListValue()) {
- ExprList list = new ExprList();
+ ExprList list = objectPool.exprListPool.get();
arg2.isListValue(list);
base.setRealValue(0.0);
rbase.setValue(0.0); // treat an empty list as 'don't quantize'
@@ -1554,7 +1613,7 @@ public class BuiltinClassAdFunctions {
result.setErrorValue();
return false; // eval should not fail
}
- if (Value.convertValueToRealValue(base, val)) {
+ if (Value.convertValueToRealValue(base, val, objectPool)) {
val.isRealValue(rbase);
if (rbase.getDoubleValue() >= rval.getDoubleValue()) {
result.setValue(base);
@@ -1575,7 +1634,7 @@ public class BuiltinClassAdFunctions {
// if arg2 is not a list, then it must evaluate to a real value
// or we can't use it. (note that if it's an int, we still want
// to return an int, but we assume that all ints can be converted to real)
- if (!Value.convertValueToRealValue(arg2, base)) {
+ if (!Value.convertValueToRealValue(arg2, base, objectPool)) {
result.setErrorValue();
return true;
}
@@ -1585,13 +1644,15 @@ public class BuiltinClassAdFunctions {
// at this point rbase should contain the real value of either arg2 or the
// last entry in the list. and rval should contain the value to be quantized.
- AMutableInt64 ival = new AMutableInt64(0);
- AMutableInt64 ibase = new AMutableInt64(0);
+ AMutableInt64 ival = objectPool.int64Pool.get();
+ ival.setValue(0);
+ AMutableInt64 ibase = objectPool.int64Pool.get();
+ ibase.setValue(0);
if (arg2.isIntegerValue(ibase)) {
// quantize to an integer base,
- if (ibase.getLongValue() == 0L)
+ if (ibase.getLongValue() == 0L) {
result.setValue(arg);
- else if (arg.isIntegerValue(ival)) {
+ } else if (arg.isIntegerValue(ival)) {
ival.setValue(((ival.getLongValue() + ibase.getLongValue() - 1) / ibase.getLongValue())
* ibase.getLongValue());
result.setIntegerValue(ival.getLongValue());
@@ -1620,10 +1681,12 @@ public class BuiltinClassAdFunctions {
}
};
public static final ClassAdFunc random = new ClassAdFunc() {
+ private Random randomGenerator = new Random(System.currentTimeMillis());
@Override
- public boolean call(String name, ExprList argList, EvalState state, Value result) throws HyracksDataException {
- Value arg = new Value();;
+ public boolean call(String name, ExprList argList, EvalState state, Value result, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ Value arg = objectPool.valuePool.get();;
// takes exactly one argument
if (argList.size() > 1) {
result.setErrorValue();
@@ -1634,9 +1697,9 @@ public class BuiltinClassAdFunctions {
result.setErrorValue();
return (false);
}
- AMutableInt64 int_max = new AMutableInt64(0);
- AMutableDouble double_max = new AMutableDouble(0);
- Random randomGenerator = new Random(System.currentTimeMillis());
+ AMutableInt64 int_max = objectPool.int64Pool.get();
+ int_max.setValue(0);
+ AMutableDouble double_max = objectPool.doublePool.get();
if (arg.isIntegerValue(int_max)) {
int random_int = randomGenerator.nextInt((int) int_max.getLongValue());
result.setIntegerValue(random_int);
@@ -1653,9 +1716,10 @@ public class BuiltinClassAdFunctions {
public static final ClassAdFunc ifThenElse = new ClassAdFunc() {
@Override
- public boolean call(String name, ExprList argList, EvalState state, Value result) throws HyracksDataException {
- Value arg1 = new Value();
- MutableBoolean arg1_bool = new MutableBoolean();
+ public boolean call(String name, ExprList argList, EvalState state, Value result, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ Value arg1 = objectPool.valuePool.get();
+ MutableBoolean arg1_bool = objectPool.boolPool.get();
// takes exactly three arguments
if (argList.size() != 3) {
result.setErrorValue();
@@ -1673,7 +1737,8 @@ public class BuiltinClassAdFunctions {
}
break;
case INTEGER_VALUE: {
- AMutableInt64 intval = new AMutableInt64(0);
+ AMutableInt64 intval = objectPool.int64Pool.get();
+ intval.setValue(0);
if (!arg1.isIntegerValue(intval)) {
result.setErrorValue();
return (false);
@@ -1682,7 +1747,7 @@ public class BuiltinClassAdFunctions {
break;
}
case REAL_VALUE: {
- AMutableDouble realval = new AMutableDouble(0);
+ AMutableDouble realval = objectPool.doublePool.get();
if (!arg1.isRealValue(realval)) {
result.setErrorValue();
return (false);
@@ -1721,14 +1786,15 @@ public class BuiltinClassAdFunctions {
public static final ClassAdFunc stringListsIntersect = new ClassAdFunc() {
@Override
- public boolean call(String name, ExprList argList, EvalState state, Value result) throws HyracksDataException {
- Value arg0 = new Value();
- Value arg1 = new Value();
- Value arg2 = new Value();
+ public boolean call(String name, ExprList argList, EvalState state, Value result, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ Value arg0 = objectPool.valuePool.get();
+ Value arg1 = objectPool.valuePool.get();
+ Value arg2 = objectPool.valuePool.get();
boolean have_delimiter;
- AMutableCharArrayString str0 = new AMutableCharArrayString();
- AMutableCharArrayString str1 = new AMutableCharArrayString();
- AMutableCharArrayString delimiter_string = new AMutableCharArrayString();
+ AMutableCharArrayString str0 = objectPool.strPool.get();
+ AMutableCharArrayString str1 = objectPool.strPool.get();
+ AMutableCharArrayString delimiter_string = objectPool.strPool.get();
// need two or three arguments: pattern, list, optional settings
if (argList.size() != 2 && argList.size() != 3) {
@@ -1782,7 +1848,7 @@ public class BuiltinClassAdFunctions {
}
result.setBooleanValue(false);
- List<String> list0 = new ArrayList<String>();
+ List<String> list0 = objectPool.stringArrayListPool.get();
Set<String> set1 = new HashSet<String>();
split_string_list(str0, have_delimiter ? delimiter_string.charAt(0) : ',', list0);
@@ -1800,10 +1866,12 @@ public class BuiltinClassAdFunctions {
};
public static final ClassAdFunc interval = new ClassAdFunc() {
@Override
- public boolean call(String name, ExprList argList, EvalState state, Value result) throws HyracksDataException {
- Value arg = new Value();
- Value intarg = new Value();
- AMutableInt64 tot_secs = new AMutableInt64(0);
+ public boolean call(String name, ExprList argList, EvalState state, Value result, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ Value arg = objectPool.valuePool.get();
+ Value intarg = objectPool.valuePool.get();
+ AMutableInt64 tot_secs = objectPool.int64Pool.get();
+ tot_secs.setValue(0);
// takes exactly one argument
if (argList.size() != 1) {
result.setErrorValue();
@@ -1813,7 +1881,7 @@ public class BuiltinClassAdFunctions {
result.setErrorValue();
return (false);
}
- if (!Value.convertValueToIntegerValue(arg, intarg)) {
+ if (!Value.convertValueToIntegerValue(arg, intarg, objectPool)) {
result.setErrorValue();
return (true);
}
@@ -1843,9 +1911,10 @@ public class BuiltinClassAdFunctions {
};
public static final ClassAdFunc eval = new ClassAdFunc() {
@Override
- public boolean call(String name, ExprList argList, EvalState state, Value result) throws HyracksDataException {
- Value arg = new Value();
- Value strarg = new Value();
+ public boolean call(String name, ExprList argList, EvalState state, Value result, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ Value arg = objectPool.valuePool.get();
+ Value strarg = objectPool.valuePool.get();
// takes exactly one argument
if (argList.size() != 1) {
result.setErrorValue();
@@ -1855,8 +1924,10 @@ public class BuiltinClassAdFunctions {
result.setErrorValue();
return false;
}
- AMutableCharArrayString s = new AMutableCharArrayString();
- if (!Value.convertValueToStringValue(arg, strarg) || !strarg.isStringValue(s)) {
+ ClassAdParser parser = objectPool.classAdParserPool.get();
+ ExprTreeHolder expr = objectPool.mutableExprPool.get();
+ AMutableCharArrayString s = objectPool.strPool.get();
+ if (!Value.convertValueToStringValue(arg, strarg, objectPool) || !strarg.isStringValue(s)) {
result.setErrorValue();
return true;
}
@@ -1864,8 +1935,8 @@ public class BuiltinClassAdFunctions {
result.setErrorValue();
return false;
}
- ClassAdParser parser = new ClassAdParser(null, false, true, false, null, null, null);
- ExprTreeHolder expr = new ExprTreeHolder();
+
+ expr.reset();
try {
if (!parser.parseExpression(s.toString(), expr, true) || (expr.getInnerTree() == null)) {
result.setErrorValue();
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/CaseInsensitiveString.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/CaseInsensitiveString.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/CaseInsensitiveString.java
index ecc69f8..7a93eda 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/CaseInsensitiveString.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/CaseInsensitiveString.java
@@ -20,6 +20,7 @@ package org.apache.asterix.external.classad;
public class CaseInsensitiveString implements Comparable<CaseInsensitiveString> {
private String aString;
+ private String aLowerCaseString;
public String get() {
return aString;
@@ -32,30 +33,33 @@ public class CaseInsensitiveString implements Comparable<CaseInsensitiveString>
public void set(String aString) {
this.aString = aString;
+ this.aLowerCaseString = aString == null ? null : aString.toLowerCase();
}
public CaseInsensitiveString(String aString) {
this.aString = aString;
+ this.aLowerCaseString = aString == null ? null : aString.toLowerCase();
}
public CaseInsensitiveString() {
this.aString = null;
+ this.aLowerCaseString = null;
}
@Override
public int compareTo(CaseInsensitiveString o) {
- return aString.compareToIgnoreCase(o.aString);
+ return aLowerCaseString.compareTo(o.aLowerCaseString);
}
@Override
public boolean equals(Object o) {
- return (o instanceof CaseInsensitiveString) ? aString.equalsIgnoreCase(((CaseInsensitiveString) o).aString)
- : false;
+ return (o instanceof CaseInsensitiveString)
+ ? aLowerCaseString.equals(((CaseInsensitiveString) o).aLowerCaseString) : false;
}
@Override
public int hashCode() {
- return aString.toLowerCase().hashCode();
+ return aLowerCaseString.hashCode();
}
}
[05/13] incubator-asterixdb git commit: Improve Error Handling in
Local Directory Feeds
Posted by am...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/Literal.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/Literal.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/Literal.java
index 5254a37..f04eb1d 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/Literal.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/Literal.java
@@ -23,6 +23,7 @@ import java.util.TimeZone;
import org.apache.asterix.external.classad.Value.NumberFactor;
import org.apache.asterix.external.classad.Value.ValueType;
+import org.apache.asterix.external.classad.object.pool.ClassAdObjectPool;
import org.apache.asterix.om.base.AMutableDouble;
import org.apache.asterix.om.base.AMutableInt32;
import org.apache.asterix.om.base.AMutableInt64;
@@ -34,12 +35,13 @@ public class Literal extends ExprTree {
* reals, booleans, strings, undefined and real.
*/
// literal specific information
- private Value value;
+ private final Value value;
private NumberFactor factor;
- public Literal() {
+ public Literal(ClassAdObjectPool objectPool) {
+ super(objectPool);
factor = Value.NumberFactor.NO_FACTOR;
- value = new Value();
+ value = new Value(objectPool);
}
@Override
@@ -69,47 +71,44 @@ public class Literal extends ExprTree {
}
}
- public Literal(Literal literal) throws HyracksDataException {
- copyFrom(literal);
- }
-
@Override
public ExprTree copy() throws HyracksDataException {
- Literal newTree = new Literal();
+ Literal newTree = objectPool.literalPool.get();
newTree.copyFrom(this);
return newTree;
}
public void copyFrom(Literal literal) throws HyracksDataException {
super.copyFrom(literal);
- value.copyFrom(literal.value);
+ value.setValue(literal.value);
factor = literal.factor;
return;
}
- public static Literal createReal(AMutableCharArrayString aString) throws HyracksDataException {
- Value val = new Value();
+ public static Literal createReal(AMutableCharArrayString aString, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ Value val = objectPool.valuePool.get();
double real;
real = Double.parseDouble(aString.toString());
val.setRealValue(real);
- return createLiteral(val);
+ return createLiteral(val, objectPool);
}
- public static Literal createReal(String aString) throws HyracksDataException {
- Value val = new Value();
+ public static Literal createReal(String aString, ClassAdObjectPool objectPool) throws HyracksDataException {
+ Value val = objectPool.valuePool.get();
double real;
real = Double.parseDouble(aString.toString());
val.setRealValue(real);
- return createLiteral(val);
+ return createLiteral(val, objectPool);
}
- public static Literal createAbsTime(ClassAdTime tim) throws HyracksDataException {
- Value val = new Value();
+ public static Literal createAbsTime(ClassAdTime tim, ClassAdObjectPool objectPool) throws HyracksDataException {
+ Value val = objectPool.valuePool.get();
if (tim == null) { // => current time/offset
- tim = new ClassAdTime();
+ tim = objectPool.classAdTimePool.get();
}
val.setAbsoluteTimeValue(tim);
- return (createLiteral(val));
+ return (createLiteral(val, objectPool));
}
/* Creates an absolute time literal, from the string timestr,
@@ -118,17 +117,21 @@ public class Literal extends ExprTree {
D => non-digit, d=> digit
Ex - 2003-01-25T09:00:00-06:00
*/
- public static Literal createAbsTime(AMutableCharArrayString timeStr) throws HyracksDataException {
- Value val = new Value();
+ public static Literal createAbsTime(AMutableCharArrayString timeStr, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ Value val = objectPool.valuePool.get();
boolean offset = false; // to check if the argument conatins a timezone offset parameter
- AMutableInt32 tzhr = new AMutableInt32(0); // corresponds to 1st "dd" in -|+dd:dd
- AMutableInt32 tzmin = new AMutableInt32(0); // corresponds to 2nd "dd" in -|+dd:dd
+ AMutableInt32 tzhr = objectPool.int32Pool.get(); // corresponds to 1st "dd" in -|+dd:dd
+ AMutableInt32 tzmin = objectPool.int32Pool.get(); // corresponds to 2nd "dd" in -|+dd:dd
int len = timeStr.getLength();
- AMutableInt32 index = new AMutableInt32(len - 1);
+ AMutableInt32 index = objectPool.int32Pool.get();
+ index.setValue(len - 1);
prevNonSpaceChar(timeStr, index);
- AMutableInt32 i = new AMutableInt32(index.getIntegerValue());
+ AMutableInt32 i = objectPool.int32Pool.get();
+ i.setValue(index.getIntegerValue());
+
if ((timeStr.charAt(i.getIntegerValue()) == 'z') || (timeStr.charAt(i.getIntegerValue()) == 'Z')) { // z|Z corresponds to a timezone offset of 0
offset = true;
timeStr.erase(i.getIntegerValue()); // remove the offset section from the string
@@ -145,7 +148,7 @@ public class Literal extends ExprTree {
nextDigitChar(timeStr, i);
if (i.getIntegerValue() > len - 4) { // string has to contain dddd (year)
val.setErrorValue();
- return (createLiteral(val));
+ return (createLiteral(val, objectPool));
}
int tm_year, tm_mon = 0, tm_mday = 0, tm_hour = 0, tm_min = 0, tm_sec = 0;
tm_year = Integer.parseInt((timeStr.substr(i.getIntegerValue(), 4)));// - 1900;
@@ -183,12 +186,12 @@ public class Literal extends ExprTree {
if ((i.getIntegerValue() <= len - 1) && (Character.isDigit(timeStr.charAt(i.getIntegerValue())))) { // there should be no more digit characters once the required
val.setErrorValue(); // parameteres are parsed
- return (createLiteral(val));
+ return (createLiteral(val, objectPool));
}
Calendar cal = Calendar.getInstance(TimeZone.getTimeZone("GMT"));
cal.clear();
cal.set(tm_year, tm_mon, tm_mday, tm_hour, tm_min, tm_sec);
- ClassAdTime time = new ClassAdTime();
+ ClassAdTime time = objectPool.classAdTimePool.get();
time.setValue(cal.getTimeInMillis());
if (offset) {
time.setTimeZone((tzhr.getIntegerValue() * 3600000) + (tzmin.getIntegerValue() * 60000));
@@ -197,28 +200,29 @@ public class Literal extends ExprTree {
time.setDefaultTimeZone();
}
val.setAbsoluteTimeValue(time);
- return (createLiteral(val));
+ return (createLiteral(val, objectPool));
}
public Literal createRelTime(ClassAdTime t1, ClassAdTime t2) throws HyracksDataException {
- Value val = new Value();
+ Value val = objectPool.valuePool.get();
val.setRelativeTimeValue(t1.subtract(t2));
- return (createLiteral(val));
+ return (createLiteral(val, objectPool));
}
Literal createRelTime(ClassAdTime secs) throws HyracksDataException {
- Value val = new Value();
+ Value val = objectPool.valuePool.get();
val.setRelativeTimeValue(secs);
- return (createLiteral(val));
+ return (createLiteral(val, objectPool));
}
/* Creates a relative time literal, from the string timestr,
*parsing it as [[[days+]hh:]mm:]ss
* Ex - 1+00:02:00
*/
- public static Literal createRelTime(AMutableCharArrayString timeStr) throws HyracksDataException {
- Value val = new Value();
- ClassAdTime rsecs = new ClassAdTime();
+ public static Literal createRelTime(AMutableCharArrayString timeStr, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ Value val = objectPool.valuePool.get();
+ ClassAdTime rsecs = objectPool.classAdTimePool.get();
int len = timeStr.getLength();
double secs = 0;
@@ -226,8 +230,8 @@ public class Literal extends ExprTree {
int hrs = 0;
int days = 0;
boolean negative = false;
-
- AMutableInt32 i = new AMutableInt32(len - 1);
+ AMutableInt32 i = objectPool.int32Pool.get();
+ i.setValue(len - 1);
prevNonSpaceChar(timeStr, i);
// checking for 'sec' parameter & collecting it if present (ss.sss)
if ((i.getIntegerValue() >= 0)
@@ -237,7 +241,7 @@ public class Literal extends ExprTree {
i.setValue(i.getIntegerValue() - 1);
}
prevNonSpaceChar(timeStr, i);
- AMutableCharArrayString revSecStr = new AMutableCharArrayString();
+ AMutableCharArrayString revSecStr = objectPool.strPool.get();
while ((i.getIntegerValue() >= 0) && (Character.isDigit(timeStr.charAt(i.getIntegerValue())))) {
revSecStr.appendChar(timeStr.charAt(i.getIntegerValue()));
i.setValue(i.getIntegerValue() - 1);
@@ -250,7 +254,7 @@ public class Literal extends ExprTree {
i.setValue(i.getIntegerValue() - 1);
}
}
- secs = revDouble(revSecStr);
+ secs = revDouble(revSecStr, objectPool);
}
prevNonSpaceChar(timeStr, i);
@@ -258,13 +262,13 @@ public class Literal extends ExprTree {
if ((i.getIntegerValue() >= 0) && ((timeStr.charAt(i.getIntegerValue()) == 'm')
|| (timeStr.charAt(i.getIntegerValue()) == 'M') || (timeStr.charAt(i.getIntegerValue()) == ':'))) {
i.setValue(i.getIntegerValue() - 1);
- AMutableCharArrayString revMinStr = new AMutableCharArrayString();
+ AMutableCharArrayString revMinStr = objectPool.strPool.get();
prevNonSpaceChar(timeStr, i);
while ((i.getIntegerValue() >= 0) && (Character.isDigit(timeStr.charAt(i.getIntegerValue())))) {
revMinStr.appendChar(timeStr.charAt(i.getIntegerValue()));
i.setValue(i.getIntegerValue() - 1);
}
- mins = revInt(revMinStr);
+ mins = revInt(revMinStr, objectPool);
}
prevNonSpaceChar(timeStr, i);
@@ -272,13 +276,13 @@ public class Literal extends ExprTree {
if ((i.getIntegerValue() >= 0) && ((timeStr.charAt(i.getIntegerValue()) == 'h')
|| (timeStr.charAt(i.getIntegerValue()) == 'H') || (timeStr.charAt(i.getIntegerValue()) == ':'))) {
i.setValue(i.getIntegerValue() - 1);
- AMutableCharArrayString revHrStr = new AMutableCharArrayString();
+ AMutableCharArrayString revHrStr = objectPool.strPool.get();
prevNonSpaceChar(timeStr, i);
while ((i.getIntegerValue() >= 0) && (Character.isDigit(timeStr.charAt(i.getIntegerValue())))) {
revHrStr.appendChar(timeStr.charAt(i.getIntegerValue()));
i.setValue(i.getIntegerValue() - 1);
}
- hrs = revInt(revHrStr);
+ hrs = revInt(revHrStr, objectPool);
}
prevNonSpaceChar(timeStr, i);
@@ -286,13 +290,13 @@ public class Literal extends ExprTree {
if ((i.getIntegerValue() >= 0) && ((timeStr.charAt(i.getIntegerValue()) == 'd')
|| (timeStr.charAt(i.getIntegerValue()) == 'D') || (timeStr.charAt(i.getIntegerValue()) == '+'))) {
i.setValue(i.getIntegerValue() - 1);
- AMutableCharArrayString revDayStr = new AMutableCharArrayString();
+ AMutableCharArrayString revDayStr = objectPool.strPool.get();
prevNonSpaceChar(timeStr, i);
while ((i.getIntegerValue() >= 0) && (Character.isDigit(timeStr.charAt(i.getIntegerValue())))) {
revDayStr.appendChar(timeStr.charAt(i.getIntegerValue()));
i.setValue(i.getIntegerValue() - 1);
}
- days = revInt(revDayStr);
+ days = revInt(revDayStr, objectPool);
}
prevNonSpaceChar(timeStr, i);
@@ -306,14 +310,14 @@ public class Literal extends ExprTree {
if ((i.getIntegerValue() >= 0) && (!(Character.isWhitespace(timeStr.charAt(i.getIntegerValue()))))) { // should not conatin any non-space char beyond -,d,h,m,s
val.setErrorValue();
- return (createLiteral(val));
+ return (createLiteral(val, objectPool));
}
rsecs.setRelativeTime(
(long) ((negative ? -1 : +1) * (days * 86400000 + hrs * 3600000 + mins * 60000 + secs * 1000.0)));
val.setRelativeTimeValue(rsecs);
- return (createLiteral(val));
+ return (createLiteral(val, objectPool));
}
/* Function which iterates through the string Str from the location 'index',
@@ -343,8 +347,8 @@ public class Literal extends ExprTree {
* order of the digits & returns the corresponding number as an
* integer.
*/
- public static int revInt(AMutableCharArrayString revNumStr) {
- AMutableCharArrayString numStr = new AMutableCharArrayString(revNumStr.getLength());
+ public static int revInt(AMutableCharArrayString revNumStr, ClassAdObjectPool objectPool) {
+ AMutableCharArrayString numStr = objectPool.strPool.get();
for (int i = revNumStr.getLength() - 1; i >= 0; i--) {
numStr.appendChar(revNumStr.charAt(i));
}
@@ -354,8 +358,8 @@ public class Literal extends ExprTree {
/* Function which takes a number in string format, and reverses the
* order of the digits & returns the corresponding number as a double.
*/
- public static double revDouble(AMutableCharArrayString revNumStr) {
- AMutableCharArrayString numStr = new AMutableCharArrayString(revNumStr.getLength());
+ public static double revDouble(AMutableCharArrayString revNumStr, ClassAdObjectPool objectPool) {
+ AMutableCharArrayString numStr = objectPool.strPool.get();
for (int i = revNumStr.getLength() - 1; i >= 0; i--) {
numStr.appendChar(revNumStr.charAt(i));
}
@@ -369,15 +373,17 @@ public class Literal extends ExprTree {
return Util.timezoneOffset(epochsecs);
}
- public static Literal createLiteral(Value val, NumberFactor f) throws HyracksDataException {
+ public static Literal createLiteral(Value val, NumberFactor f, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
if (val.getType() == ValueType.CLASSAD_VALUE || val.getType() == ValueType.LIST_VALUE
|| val.getType() == ValueType.SLIST_VALUE) {
throw new HyracksDataException("list and classad values are not literals");
}
- Literal lit = new Literal();
- lit.value.copyFrom(val);
- if (!val.isIntegerValue() && !val.isRealValue())
+ Literal lit = objectPool.literalPool.get();
+ lit.value.setValue(val);
+ if (!val.isIntegerValue() && !val.isRealValue()) {
f = NumberFactor.NO_FACTOR;
+ }
lit.factor = f;
return lit;
}
@@ -387,20 +393,21 @@ public class Literal extends ExprTree {
|| val.getType() == ValueType.SLIST_VALUE) {
throw new HyracksDataException("list and classad values are not literals");
}
- lit.value.copyFrom(val);
- if (!val.isIntegerValue() && !val.isRealValue())
+ lit.value.setValue(val);
+ if (!val.isIntegerValue() && !val.isRealValue()) {
f = NumberFactor.NO_FACTOR;
+ }
lit.factor = f;
}
- public static Literal createLiteral(Value val) throws HyracksDataException {
- return createLiteral(val, NumberFactor.NO_FACTOR);
+ public static Literal createLiteral(Value val, ClassAdObjectPool objectPool) throws HyracksDataException {
+ return createLiteral(val, NumberFactor.NO_FACTOR, objectPool);
}
public void GetValue(Value val) throws HyracksDataException {
- AMutableInt64 i = new AMutableInt64(0);
- AMutableDouble r = new AMutableDouble(0);
- val.copyFrom(value);
+ AMutableInt64 i = objectPool.int64Pool.get();
+ AMutableDouble r = objectPool.doublePool.get();
+ val.setValue(value);
// if integer or real, multiply by the factor
if (val.isIntegerValue(i)) {
@@ -415,7 +422,7 @@ public class Literal extends ExprTree {
}
public void getComponents(Value val, AMutableNumberFactor factor) throws HyracksDataException {
- val.copyFrom(value);
+ val.setValue(value);
factor.setFactor(this.factor);
}
@@ -446,10 +453,10 @@ public class Literal extends ExprTree {
@Override
public boolean privateEvaluate(EvalState eval, Value val) throws HyracksDataException {
- AMutableInt64 i = new AMutableInt64(0);
- AMutableDouble r = new AMutableDouble(0);
+ AMutableInt64 i = objectPool.int64Pool.get();
+ AMutableDouble r = objectPool.doublePool.get();
- val.copyFrom(value);
+ val.setValue(value);
// if integer or real, multiply by the factor
if (val.isIntegerValue(i)) {
@@ -511,7 +518,7 @@ public class Literal extends ExprTree {
@Override
public void reset() {
- value.clear();
+ value.reset();
factor = NumberFactor.NO_FACTOR;
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/Operation.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/Operation.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/Operation.java
index ae3c5ef..b2ed5b1 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/Operation.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/Operation.java
@@ -19,6 +19,7 @@
package org.apache.asterix.external.classad;
import org.apache.asterix.external.classad.Value.ValueType;
+import org.apache.asterix.external.classad.object.pool.ClassAdObjectPool;
import org.apache.asterix.om.base.AMutableDouble;
import org.apache.asterix.om.base.AMutableInt32;
import org.apache.asterix.om.base.AMutableInt64;
@@ -91,9 +92,9 @@ public class Operation extends ExprTree {
public static final int OpKind_LAST_OP = OpKind_MISC_END;
private int opKind;
- private ExprTree child1;
- private ExprTree child2;
- private ExprTree child3;
+ private final ExprTreeHolder child1;
+ private final ExprTreeHolder child2;
+ private final ExprTreeHolder child3;
/// node type
@Override
@@ -117,14 +118,17 @@ public class Operation extends ExprTree {
* @param e3
* The third sub-expression child of the node (if any).
* @return The constructed operation
+ * @throws HyracksDataException
*/
- public static Operation createOperation(int opkind, ExprTree e1, ExprTree e2) {
- return createOperation(opkind, e1, e2, null);
+ public static Operation createOperation(int opkind, ExprTree e1, ExprTree e2, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ return createOperation(opkind, e1, e2, null, objectPool);
}
- public static Operation createOperation(int opkind, ExprTree e1) {
- return createOperation(opkind, e1, null, null);
+ public static Operation createOperation(int opkind, ExprTree e1, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ return createOperation(opkind, e1, null, null, objectPool);
}
// public access to operation function
@@ -166,57 +170,36 @@ public class Operation extends ExprTree {
* @return true if the operator is strict, false otherwise.
*/
- public Operation() {
+ public Operation(ClassAdObjectPool objectPool) {
+ super(objectPool);
opKind = OpKind_NO_OP;
- child1 = null;
- child2 = null;
- child3 = null;
+ child1 = new ExprTreeHolder(objectPool);
+ child2 = new ExprTreeHolder(objectPool);
+ child3 = new ExprTreeHolder(objectPool);
}
- public Operation(Operation op) throws HyracksDataException {
+ public Operation(Operation op, ClassAdObjectPool objectPool) throws HyracksDataException {
+ super(objectPool);
+ child1 = new ExprTreeHolder(objectPool);
+ child2 = new ExprTreeHolder(objectPool);
+ child3 = new ExprTreeHolder(objectPool);
copyFrom(op);
- return;
}
@Override
public ExprTree copy() throws HyracksDataException {
- Operation newTree = new Operation();
+ Operation newTree = objectPool.operationPool.get();
newTree.copyFrom(this);
return newTree;
}
public boolean copyFrom(Operation op) throws HyracksDataException {
- boolean success = true;
- if (op.child1 == null) {
- child1 = null;
- } else {
- if (child1 == null) {
- child1 = new ExprTreeHolder();
- }
- child1.copyFrom(op.child1);
- child1 = child1.self();
- }
- if (op.child2 == null) {
- child2 = null;
- } else {
- if (child2 == null) {
- child2 = new ExprTreeHolder();
- }
- child2.copyFrom(op.child2);
- child2 = child2.self();
- }
- if (op.child3 == null) {
- child3 = null;
- } else {
- if (child3 == null) {
- child3 = new ExprTreeHolder();
- }
- child3.copyFrom(op.child3);
- child3 = child3.self();
- }
+ child1.copyFrom(op.child1);
+ child2.copyFrom(op.child2);
+ child3.copyFrom(op.child3);
this.opKind = op.opKind;
super.copyFrom(op);
- return success;
+ return true;
}
@Override
@@ -257,33 +240,28 @@ public class Operation extends ExprTree {
@Override
public void privateSetParentScope(ClassAd parent) {
- if (child1 != null) {
- child1.setParentScope(parent);
- }
- if (child2 != null) {
- child2.setParentScope(parent);
- }
- if (child3 != null) {
- child3.setParentScope(parent);
- }
+ child1.setParentScope(parent);
+ child2.setParentScope(parent);
+ child3.setParentScope(parent);
}
- public static void operate(int opKind, Value op1, Value op2, Value result) throws HyracksDataException {
- Value dummy = new Value();
- privateDoOperation(opKind, op1, op2, dummy, true, true, false, result, null);
+ public static void operate(int opKind, Value op1, Value op2, Value result, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ Value dummy = objectPool.valuePool.get();
+ privateDoOperation(opKind, op1, op2, dummy, true, true, false, result, null, objectPool);
}
public void operate(int op, Value op1, Value op2, Value op3, Value result) throws HyracksDataException {
- privateDoOperation(op, op1, op2, op3, true, true, true, result, null);
+ privateDoOperation(op, op1, op2, op3, true, true, true, result, null, objectPool);
}
public static int privateDoOperation(int op, Value val1, Value val2, Value val3, boolean valid1, boolean valid2,
- boolean valid3, Value result) throws HyracksDataException {
- return privateDoOperation(op, val1, val2, val3, valid1, valid2, valid3, result, null);
+ boolean valid3, Value result, ClassAdObjectPool objectPool) throws HyracksDataException {
+ return privateDoOperation(op, val1, val2, val3, valid1, valid2, valid3, result, null, objectPool);
}
public static int privateDoOperation(int op, Value val1, Value val2, Value val3, boolean valid1, boolean valid2,
- boolean valid3, Value result, EvalState es) throws HyracksDataException {
+ boolean valid3, Value result, EvalState es, ClassAdObjectPool objectPool) throws HyracksDataException {
ValueType vt1;
ValueType vt2;
ValueType vt3;
@@ -295,7 +273,7 @@ public class Operation extends ExprTree {
// take care of the easy cases
if (op == OpKind_NO_OP || op == OpKind_PARENTHESES_OP) {
- result.copyFrom(val1);
+ result.setValue(val1);
return SigValues.SIG_CHLD1.ordinal();
} else if (op == OpKind_UNARY_PLUS_OP) {
if (vt1 == ValueType.BOOLEAN_VALUE || vt1 == ValueType.STRING_VALUE || val1.isListValue()
@@ -303,7 +281,7 @@ public class Operation extends ExprTree {
result.setErrorValue();
} else {
// applies for ERROR, UNDEFINED and Numbers
- result.copyFrom(val1);
+ result.setValue(val1);
}
return SigValues.SIG_CHLD1.ordinal();
}
@@ -342,28 +320,28 @@ public class Operation extends ExprTree {
// comparison operations (binary, one unary)
if (op >= OpKind_COMPARISON_START && op <= OpKind_COMPARISON_END) {
- return (doComparison(op, val1, val2, result));
+ return (doComparison(op, val1, val2, result, objectPool));
}
// arithmetic operations (binary)
if (op >= OpKind_ARITHMETIC_START && op <= OpKind_ARITHMETIC_END) {
- return (doArithmetic(op, val1, val2, result));
+ return (doArithmetic(op, val1, val2, result, objectPool));
}
// logical operators (binary, one unary)
if (op >= OpKind_LOGIC_START && op <= OpKind_LOGIC_END) {
- return (doLogical(op, val1, val2, result));
+ return (doLogical(op, val1, val2, result, objectPool));
}
// bitwise operators (binary, one unary)
if (op >= OpKind_BITWISE_START && op <= OpKind_BITWISE_END) {
- return (doBitwise(op, val1, val2, result));
+ return (doBitwise(op, val1, val2, result, objectPool));
}
// misc.
if (op == OpKind_TERNARY_OP) {
// ternary (if-operator)
- MutableBoolean b = new MutableBoolean(false);
+ MutableBoolean b = objectPool.boolPool.get();
// if the selector is UNDEFINED, the result is undefined
if (vt1 == ValueType.UNDEFINED_VALUE) {
@@ -375,18 +353,18 @@ public class Operation extends ExprTree {
result.setErrorValue();
return SigValues.SIG_CHLD1.ordinal();
} else if (b.booleanValue()) {
- result.copyFrom(val2);
+ result.setValue(val2);
return (SigValues.SIG_CHLD2.ordinal());
} else {
- result.copyFrom(val3);
+ result.setValue(val3);
return (SigValues.SIG_CHLD3.ordinal());
}
} else if (op == OpKind_SUBSCRIPT_OP) {
// subscripting from a list (strict)
if (vt1 == ValueType.CLASSAD_VALUE && vt2 == ValueType.STRING_VALUE) {
- ClassAd classad = new ClassAd();
- AMutableCharArrayString index = new AMutableCharArrayString();
+ ClassAd classad = objectPool.classAdPool.get();
+ AMutableCharArrayString index = objectPool.strPool.get();
val1.isClassAdValue(classad);
val2.isStringValue(index);
@@ -402,8 +380,8 @@ public class Operation extends ExprTree {
return (SigValues.SIG_CHLD1.ordinal() | SigValues.SIG_CHLD2.ordinal());
} else if (val1.isListValue() && vt2 == ValueType.INTEGER_VALUE) {
- AMutableInt64 index = new AMutableInt64(0);
- ExprList elist = new ExprList();
+ AMutableInt64 index = objectPool.int64Pool.get();
+ ExprList elist = objectPool.exprListPool.get();
val1.isListValue(elist);
val2.isIntegerValue(index);
@@ -425,9 +403,9 @@ public class Operation extends ExprTree {
@Override
public boolean privateEvaluate(EvalState state, Value result) throws HyracksDataException {
- Value val1 = new Value();
- Value val2 = new Value();
- Value val3 = new Value();
+ Value val1 = objectPool.valuePool.get();
+ Value val2 = objectPool.valuePool.get();
+ Value val3 = objectPool.valuePool.get();
boolean valid1, valid2, valid3;
int rval = 0;
@@ -435,10 +413,10 @@ public class Operation extends ExprTree {
valid2 = false;
valid3 = false;
- AMutableInt32 operationKind = new AMutableInt32(OpKind_NO_OP);
- ExprTreeHolder child1 = new ExprTreeHolder();
- ExprTreeHolder child2 = new ExprTreeHolder();
- ExprTreeHolder child3 = new ExprTreeHolder();
+ AMutableInt32 operationKind = objectPool.int32Pool.get();
+ ExprTreeHolder child1 = objectPool.mutableExprPool.get();
+ ExprTreeHolder child2 = objectPool.mutableExprPool.get();
+ ExprTreeHolder child3 = objectPool.mutableExprPool.get();
getComponents(operationKind, child1, child2, child3);
// Evaluate all valid children
@@ -469,13 +447,13 @@ public class Operation extends ExprTree {
valid3 = true;
}
- rval = privateDoOperation(opKind, val1, val2, val3, valid1, valid2, valid3, result, state);
+ rval = privateDoOperation(opKind, val1, val2, val3, valid1, valid2, valid3, result, state, objectPool);
return (rval != SigValues.SIG_NONE.ordinal());
}
public boolean shortCircuit(EvalState state, Value arg1, Value result) throws HyracksDataException {
- MutableBoolean arg1_bool = new MutableBoolean();
+ MutableBoolean arg1_bool = objectPool.boolPool.get();
switch (opKind) {
case OpKind_LOGICAL_OR_OP:
if (arg1.isBooleanValueEquiv(arg1_bool) && arg1_bool.booleanValue()) {
@@ -512,21 +490,21 @@ public class Operation extends ExprTree {
@Override
public boolean privateEvaluate(EvalState state, Value result, ExprTreeHolder tree) throws HyracksDataException {
int sig;
- Value val1 = new Value();
- Value val2 = new Value();
- Value val3 = new Value();
- ExprTreeHolder t1 = new ExprTreeHolder();
- ExprTreeHolder t2 = new ExprTreeHolder();
- ExprTreeHolder t3 = new ExprTreeHolder();
+ Value val1 = objectPool.valuePool.get();
+ Value val2 = objectPool.valuePool.get();
+ Value val3 = objectPool.valuePool.get();
+ ExprTreeHolder t1 = objectPool.mutableExprPool.get();
+ ExprTreeHolder t2 = objectPool.mutableExprPool.get();
+ ExprTreeHolder t3 = objectPool.mutableExprPool.get();
boolean valid1 = false, valid2 = false, valid3 = false;
- AMutableInt32 opKind = new AMutableInt32(OpKind_NO_OP);
- ExprTreeHolder child1 = new ExprTreeHolder();
- ExprTreeHolder child2 = new ExprTreeHolder();
- ExprTreeHolder child3 = new ExprTreeHolder();
+ AMutableInt32 opKind = objectPool.int32Pool.get();
+ ExprTreeHolder child1 = objectPool.mutableExprPool.get();
+ ExprTreeHolder child2 = objectPool.mutableExprPool.get();
+ ExprTreeHolder child3 = objectPool.mutableExprPool.get();
getComponents(opKind, child1, child2, child3);
// Evaluate all valid children
- tree = new ExprTreeHolder();
+ tree = objectPool.mutableExprPool.get();
if (child1.getInnerTree() != null) {
if (!child1.publicEvaluate(state, val1, t1)) {
result.setErrorValue();
@@ -552,7 +530,7 @@ public class Operation extends ExprTree {
// do evaluation
sig = privateDoOperation(opKind.getIntegerValue().intValue(), val1, val2, val3, valid1, valid2, valid3, result,
- state);
+ state, objectPool);
// delete trees which were not significant
if (valid1 && 0 != (sig & SigValues.SIG_CHLD1.ordinal())) {
@@ -587,7 +565,7 @@ public class Operation extends ExprTree {
} else {
// the node operated on the value; the operator is also
// significant
- tree.setInnerTree(createOperation(opKind.getIntegerValue().intValue(), t1));
+ tree.setInnerTree(createOperation(opKind.getIntegerValue().intValue(), t1, objectPool));
}
return (true);
} else {
@@ -604,7 +582,7 @@ public class Operation extends ExprTree {
throw new HyracksDataException("Should not reach here");
} else {
// the node is also significant
- tree.setInnerTree(createOperation(opKind.getIntegerValue().intValue(), t1, t2));
+ tree.setInnerTree(createOperation(opKind.getIntegerValue().intValue(), t1, t2, objectPool));
return (true);
}
}
@@ -613,7 +591,7 @@ public class Operation extends ExprTree {
if (opKind.getIntegerValue().intValue() == OpKind_IS_OP
|| opKind.getIntegerValue().intValue() == OpKind_ISNT_OP) {
// the operation is *always* significant for IS and ISNT
- tree.setInnerTree(createOperation(opKind.getIntegerValue().intValue(), t1, t2));
+ tree.setInnerTree(createOperation(opKind.getIntegerValue().intValue(), t1, t2, objectPool));
return (true);
}
// other non-strict binary operators
@@ -621,7 +599,7 @@ public class Operation extends ExprTree {
|| opKind.getIntegerValue().intValue() == OpKind_LOGICAL_OR_OP) {
if ((SigValues.values()[sig].ordinal() & SigValues.SIG_CHLD1.ordinal()) != 0
&& (SigValues.values()[sig].ordinal() & SigValues.SIG_CHLD2.ordinal()) != 0) {
- tree.setInnerTree(createOperation(opKind.getIntegerValue().intValue(), t1, t2));
+ tree.setInnerTree(createOperation(opKind.getIntegerValue().intValue(), t1, t2, objectPool));
return (true);
} else if ((SigValues.values()[sig].ordinal() & SigValues.SIG_CHLD1.ordinal()) != 0) {
tree.setInnerTree(t1);
@@ -636,9 +614,9 @@ public class Operation extends ExprTree {
// non-strict ternary operator (conditional operator) s ? t : f
// selector is always significant (???)
if (opKind.getIntegerValue().intValue() == OpKind_TERNARY_OP) {
- Value tmpVal = new Value();
+ Value tmpVal = objectPool.valuePool.get();
tmpVal.setUndefinedValue();
- tree.setInnerTree(Literal.createLiteral(tmpVal));
+ tree.setInnerTree(Literal.createLiteral(tmpVal, objectPool));
// "true" consequent taken
if ((SigValues.values()[sig].ordinal() & SigValues.SIG_CHLD2.ordinal()) != 0) {
@@ -659,14 +637,17 @@ public class Operation extends ExprTree {
@Override
public boolean privateFlatten(EvalState state, Value val, ExprTreeHolder tree, AMutableInt32 opPtr)
throws HyracksDataException {
- AMutableInt32 childOp1 = new AMutableInt32(OpKind_NO_OP);
- AMutableInt32 childOp2 = new AMutableInt32(OpKind_NO_OP);
- ExprTreeHolder fChild1 = new ExprTreeHolder();
- ExprTreeHolder fChild2 = new ExprTreeHolder();;
- Value val1 = new Value();
- Value val2 = new Value();
- Value val3 = new Value();
- AMutableInt32 newOp = new AMutableInt32(opKind);
+ AMutableInt32 childOp1 = objectPool.int32Pool.get();
+ childOp1.setValue(OpKind_NO_OP);
+ AMutableInt32 childOp2 = objectPool.int32Pool.get();
+ childOp2.setValue(OpKind_NO_OP);
+ ExprTreeHolder fChild1 = objectPool.mutableExprPool.get();
+ ExprTreeHolder fChild2 = objectPool.mutableExprPool.get();;
+ Value val1 = objectPool.valuePool.get();
+ Value val2 = objectPool.valuePool.get();
+ Value val3 = objectPool.valuePool.get();
+ AMutableInt32 newOp = objectPool.int32Pool.get();
+ newOp.setValue(opKind);
int op = opKind;
tree.setInnerTree(null);; // Just to be safe... wenger 2003-12-11.
@@ -675,21 +656,22 @@ public class Operation extends ExprTree {
if ((op >= OpKind_COMPARISON_START && op <= OpKind_COMPARISON_END) || op == OpKind_SUBTRACTION_OP
|| op == OpKind_DIVISION_OP || op == OpKind_MODULUS_OP || op == OpKind_LEFT_SHIFT_OP
|| op == OpKind_RIGHT_SHIFT_OP || op == OpKind_URIGHT_SHIFT_OP) {
- if (opPtr != null)
+ if (opPtr != null) {
opPtr.setValue(OpKind_NO_OP);
+ }
if (child1.publicFlatten(state, val1, fChild1) && child2.publicFlatten(state, val2, fChild2)) {
if (fChild1.getInnerTree() == null && fChild2.getInnerTree() == null) {
- privateDoOperation(op, val1, val2, val3, true, true, false, val);
+ privateDoOperation(op, val1, val2, val3, true, true, false, val, objectPool);
tree.setInnerTree(null);
return true;
} else if (fChild1.getInnerTree() != null && fChild2.getInnerTree() != null) {
- tree.setInnerTree(Operation.createOperation(op, fChild1, fChild2));
+ tree.setInnerTree(Operation.createOperation(op, fChild1, fChild2, objectPool));
return true;
} else if (fChild1.getInnerTree() != null) {
- tree.setInnerTree(Operation.createOperation(op, fChild1, val2));
+ tree.setInnerTree(Operation.createOperation(op, fChild1, val2, objectPool));
return true;
} else if (fChild2.getInnerTree() != null) {
- tree.setInnerTree(Operation.createOperation(op, val1, fChild2));
+ tree.setInnerTree(Operation.createOperation(op, val1, fChild2, objectPool));
return true;
}
} else {
@@ -706,8 +688,8 @@ public class Operation extends ExprTree {
// any op that got past the above is binary, commutative and associative
// Flatten sub expressions
- if ((child1 != null && !child1.publicFlatten(state, val1, fChild1, childOp1))
- || (child2 != null && !child2.publicFlatten(state, val2, fChild2, childOp2))) {
+ if ((child1.getInnerTree() != null && !child1.publicFlatten(state, val1, fChild1, childOp1))
+ || (child2.getInnerTree() != null && !child2.publicFlatten(state, val2, fChild2, childOp2))) {
tree.setInnerTree(null);
return false;
}
@@ -724,7 +706,7 @@ public class Operation extends ExprTree {
// if splitting is disallowed, fold the value and tree into a tree
if (opPtr == null && newOp.getIntegerValue().intValue() != OpKind_NO_OP) {
- tree.setInnerTree(Operation.createOperation(newOp.getIntegerValue().intValue(), val, tree));
+ tree.setInnerTree(Operation.createOperation(newOp.getIntegerValue().intValue(), val, tree, objectPool));
if (tree.getInnerTree() == null) {
return false;
}
@@ -737,15 +719,15 @@ public class Operation extends ExprTree {
public boolean combine(AMutableInt32 op, Value val, ExprTreeHolder tree, AMutableInt32 op1, Value val1,
ExprTreeHolder tree1, AMutableInt32 op2, Value val2, ExprTreeHolder tree2) throws HyracksDataException {
- Operation newOp = new Operation();
- Value dummy = new Value(); // undefined
+ Operation newOp = objectPool.operationPool.get();
+ Value dummy = objectPool.valuePool.get(); // undefined
// special don't care cases for logical operators with exactly one value
if ((tree1.getInnerTree() == null || tree2.getInnerTree() == null)
&& (tree1.getInnerTree() != null || tree2.getInnerTree() != null)
&& (op.getIntegerValue() == OpKind_LOGICAL_OR_OP || op.getIntegerValue() == OpKind_LOGICAL_AND_OP)) {
privateDoOperation(op.getIntegerValue().intValue(), tree1.getInnerTree() == null ? val1 : dummy,
- tree2.getInnerTree() == null ? val2 : dummy, dummy, true, true, false, val);
+ tree2.getInnerTree() == null ? val2 : dummy, dummy, true, true, false, val, objectPool);
if (val.isBooleanValue()) {
tree.setInnerTree(null);
op.setValue(OpKind_NO_OP);
@@ -755,7 +737,7 @@ public class Operation extends ExprTree {
if (tree1.getInnerTree() == null && tree2.getInnerTree() == null) {
// left and rightsons are only values
- privateDoOperation(op.getIntegerValue().intValue(), val1, val2, dummy, true, true, false, val);
+ privateDoOperation(op.getIntegerValue().intValue(), val1, val2, dummy, true, true, false, val, objectPool);
tree.setInnerTree(null);
op.setValue(OpKind_NO_OP);
return true;
@@ -763,18 +745,18 @@ public class Operation extends ExprTree {
&& (tree2.getInnerTree() != null && op2.getIntegerValue().intValue() == OpKind_NO_OP)) {
// leftson is a value, rightson is a tree
tree.setInnerTree(tree2.getInnerTree());
- val.copyFrom(val1);
+ val.setValue(val1);
return true;
} else if (tree2.getInnerTree() == null
&& (tree1.getInnerTree() != null && op1.getIntegerValue().intValue() == OpKind_NO_OP)) {
// rightson is a value, leftson is a tree
tree.setInnerTree(tree1.getInnerTree());
- val.copyFrom(val2);
+ val.setValue(val2);
return true;
} else if ((tree1.getInnerTree() != null && op1.getIntegerValue().intValue() == OpKind_NO_OP)
&& (tree2.getInnerTree() != null && op2.getIntegerValue().intValue() == OpKind_NO_OP)) {
// left and rightsons are trees only
- if (null != (newOp = createOperation(op.getIntegerValue().intValue(), tree1, tree2))) {
+ if (null != (newOp = createOperation(op.getIntegerValue().intValue(), tree1, tree2, objectPool))) {
return false;
}
tree.setInnerTree(newOp);
@@ -787,23 +769,25 @@ public class Operation extends ExprTree {
&& !op.equals(op1) && !op.equals(op1)) {
// at least one of them returned a value and a tree, and parent does
// not share the same operation with either child
- ExprTreeHolder newOp1 = new ExprTreeHolder();
- ExprTreeHolder newOp2 = new ExprTreeHolder();
+ ExprTreeHolder newOp1 = objectPool.mutableExprPool.get();
+ ExprTreeHolder newOp2 = objectPool.mutableExprPool.get();
if (op1.getIntegerValue().intValue() != OpKind_NO_OP) {
- newOp1.setInnerTree(Operation.createOperation(op1.getIntegerValue().intValue(), val1, tree1));
+ newOp1.setInnerTree(
+ Operation.createOperation(op1.getIntegerValue().intValue(), val1, tree1, objectPool));
} else if (tree1.getInnerTree() != null) {
newOp1.setInnerTree(tree1.getInnerTree());
} else {
- newOp1.setInnerTree(Literal.createLiteral(val1));
+ newOp1.setInnerTree(Literal.createLiteral(val1, objectPool));
}
if (op2.getIntegerValue().intValue() != OpKind_NO_OP) {
- newOp2.setInnerTree(Operation.createOperation(op2.getIntegerValue().intValue(), val2, tree2));
+ newOp2.setInnerTree(
+ Operation.createOperation(op2.getIntegerValue().intValue(), val2, tree2, objectPool));
} else if (tree2.getInnerTree() != null) {
newOp2.setInnerTree(tree2);
} else {
- newOp2.setInnerTree(Literal.createLiteral(val2));
+ newOp2.setInnerTree(Literal.createLiteral(val2, objectPool));
}
if (newOp1.getInnerTree() == null || newOp2.getInnerTree() == null) {
@@ -811,7 +795,7 @@ public class Operation extends ExprTree {
op.setValue(OpKind_NO_OP);
return false;
}
- newOp = createOperation(op.getIntegerValue().intValue(), newOp1, newOp2);
+ newOp = createOperation(op.getIntegerValue().intValue(), newOp1, newOp2, objectPool);
if (newOp == null) {
tree.setInnerTree(null);
op.setValue(OpKind_NO_OP);
@@ -825,29 +809,30 @@ public class Operation extends ExprTree {
if (op.equals(op1) && op.equals(op2)) {
// same operators on both children . since op!=NO_OP, neither are op1,
// op2. so they both make tree and value contributions
- newOp = createOperation(op.getIntegerValue().intValue(), tree1, tree2);
+ newOp = createOperation(op.getIntegerValue().intValue(), tree1, tree2, objectPool);
if (newOp == null) {
return false;
}
- privateDoOperation(op.getIntegerValue().intValue(), val1, val2, dummy, true, true, false, val);
+ privateDoOperation(op.getIntegerValue().intValue(), val1, val2, dummy, true, true, false, val, objectPool);
tree.setInnerTree(newOp);
return true;
} else if (op.equals(op1)) {
// leftson makes a tree,value contribution
if (tree2.getInnerTree() == null) {
// rightson makes a value contribution
- privateDoOperation(op.getIntegerValue().intValue(), val1, val2, dummy, true, true, false, val);
+ privateDoOperation(op.getIntegerValue().intValue(), val1, val2, dummy, true, true, false, val,
+ objectPool);
tree.setInnerTree(tree1);
return true;
} else {
// rightson makes a tree contribution
- Operation local_newOp = createOperation(op.getIntegerValue().intValue(), tree1, tree2);
+ Operation local_newOp = createOperation(op.getIntegerValue().intValue(), tree1, tree2, objectPool);
if (local_newOp == null) {
tree.setInnerTree(null);
op.setValue(OpKind_NO_OP);
return false;
}
- val.copyFrom(val1);
+ val.setValue(val1);
tree.setInnerTree(local_newOp); // NAC - BUG FIX
return true;
}
@@ -855,19 +840,20 @@ public class Operation extends ExprTree {
// rightson makes a tree,value contribution
if (tree1.getInnerTree() == null) {
// leftson makes a value contribution
- privateDoOperation(op.getIntegerValue().intValue(), val1, val2, dummy, true, true, false, val);
+ privateDoOperation(op.getIntegerValue().intValue(), val1, val2, dummy, true, true, false, val,
+ objectPool);
tree.setInnerTree(tree2);
return true;
} else {
// leftson makes a tree contribution
- Operation local_newOp = createOperation(op.getIntegerValue().intValue(), tree1, tree2);
+ Operation local_newOp = createOperation(op.getIntegerValue().intValue(), tree1, tree2, objectPool);
if (local_newOp == null) {
tree.setInnerTree(null);
op.setValue(OpKind_NO_OP);
return false;
}
tree.setInnerTree(local_newOp); // NAC BUG FIX
- val.copyFrom(val2);
+ val.setValue(val2);
return true;
}
}
@@ -875,7 +861,8 @@ public class Operation extends ExprTree {
throw new HyracksDataException("Should not reach here");
}
- public static int doComparison(int op, Value v1, Value v2, Value result) throws HyracksDataException {
+ public static int doComparison(int op, Value v1, Value v2, Value result, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
ValueType vt1;
ValueType vt2;
ValueType coerceResult;
@@ -887,7 +874,7 @@ public class Operation extends ExprTree {
coerceResult = vt1;
} else {
// do numerical type promotions --- other types/values are unchanged
- coerceResult = coerceToNumber(v1, v2);
+ coerceResult = coerceToNumber(v1, v2, objectPool);
vt1 = v1.getType();
vt2 = v2.getType();
}
@@ -929,15 +916,15 @@ public class Operation extends ExprTree {
result.setErrorValue();
return (SigValues.SIG_CHLD1.ordinal() | SigValues.SIG_CHLD2.ordinal());
}
- compareStrings(op, v1, v2, result);
+ compareStrings(op, v1, v2, result, objectPool);
return (SigValues.SIG_CHLD1.ordinal() | SigValues.SIG_CHLD2.ordinal());
case INTEGER_VALUE:
- compareIntegers(op, v1, v2, result);
+ compareIntegers(op, v1, v2, result, objectPool);
return (SigValues.SIG_CHLD1.ordinal() | SigValues.SIG_CHLD2.ordinal());
case REAL_VALUE:
- compareReals(op, v1, v2, result);
+ compareReals(op, v1, v2, result, objectPool);
return (SigValues.SIG_CHLD1.ordinal() | SigValues.SIG_CHLD2.ordinal());
case BOOLEAN_VALUE:
@@ -946,7 +933,7 @@ public class Operation extends ExprTree {
result.setErrorValue();
return (SigValues.SIG_CHLD1.ordinal() | SigValues.SIG_CHLD2.ordinal());
}
- compareBools(op, v1, v2, result);
+ compareBools(op, v1, v2, result, objectPool);
return (SigValues.SIG_CHLD1.ordinal() | SigValues.SIG_CHLD2.ordinal());
case LIST_VALUE:
@@ -960,7 +947,7 @@ public class Operation extends ExprTree {
result.setErrorValue();
return (SigValues.SIG_CHLD1.ordinal() | SigValues.SIG_CHLD2.ordinal());
}
- compareAbsoluteTimes(op, v1, v2, result);
+ compareAbsoluteTimes(op, v1, v2, result, objectPool);
return (SigValues.SIG_CHLD1.ordinal() | SigValues.SIG_CHLD2.ordinal());
case RELATIVE_TIME_VALUE:
@@ -968,7 +955,7 @@ public class Operation extends ExprTree {
result.setErrorValue();
return (SigValues.SIG_CHLD1.ordinal() | SigValues.SIG_CHLD2.ordinal());
}
- compareRelativeTimes(op, v1, v2, result);
+ compareRelativeTimes(op, v1, v2, result, objectPool);
return (SigValues.SIG_CHLD1.ordinal() | SigValues.SIG_CHLD2.ordinal());
default:
@@ -977,12 +964,13 @@ public class Operation extends ExprTree {
}
}
- public static int doArithmetic(int op, Value v1, Value v2, Value result) throws HyracksDataException {
- AMutableInt64 i1 = new AMutableInt64(0);
- AMutableInt64 i2 = new AMutableInt64(0);
- ClassAdTime t1 = new ClassAdTime();
- AMutableDouble r1 = new AMutableDouble(0);
- MutableBoolean b1 = new MutableBoolean();
+ public static int doArithmetic(int op, Value v1, Value v2, Value result, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ AMutableInt64 i1 = objectPool.int64Pool.get();
+ AMutableInt64 i2 = objectPool.int64Pool.get();
+ ClassAdTime t1 = objectPool.classAdTimePool.get();
+ AMutableDouble r1 = objectPool.doublePool.get();
+ MutableBoolean b1 = objectPool.boolPool.get();
// ensure the operands have arithmetic types
if ((!v1.isIntegerValue() && !v1.isRealValue() && !v1.isAbsoluteTimeValue() && !v1.isRelativeTimeValue()
@@ -1009,7 +997,7 @@ public class Operation extends ExprTree {
result.setBooleanValue(!b1.booleanValue());
} else if (v1.isExceptional()) {
// undefined or error --- same as operand
- result.copyFrom(v1);
+ result.setValue(v1);
return SigValues.SIG_CHLD1.ordinal();
}
// unary minus not defined on any other operand type
@@ -1018,7 +1006,7 @@ public class Operation extends ExprTree {
}
// perform type promotions and proceed with arithmetic
- switch (coerceToNumber(v1, v2)) {
+ switch (coerceToNumber(v1, v2, objectPool)) {
case INTEGER_VALUE:
v1.isIntegerValue(i1);
v2.isIntegerValue(i2);
@@ -1057,11 +1045,11 @@ public class Operation extends ExprTree {
}
case REAL_VALUE: {
- return (doRealArithmetic(op, v1, v2, result));
+ return (doRealArithmetic(op, v1, v2, result, objectPool));
}
case ABSOLUTE_TIME_VALUE:
case RELATIVE_TIME_VALUE: {
- return (doTimeArithmetic(op, v1, v2, result));
+ return (doTimeArithmetic(op, v1, v2, result, objectPool));
}
default:
// should not get here
@@ -1069,9 +1057,10 @@ public class Operation extends ExprTree {
}
}
- public static int doLogical(int op, Value v1, Value v2, Value result) throws HyracksDataException {
- MutableBoolean b1 = new MutableBoolean();
- MutableBoolean b2 = new MutableBoolean();
+ public static int doLogical(int op, Value v1, Value v2, Value result, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ MutableBoolean b1 = objectPool.boolPool.get();
+ MutableBoolean b2 = objectPool.boolPool.get();
// first coerece inputs to boolean if they are considered equivalent
if (!v1.isBooleanValue(b1) && v1.isBooleanValueEquiv(b1)) {
@@ -1098,7 +1087,7 @@ public class Operation extends ExprTree {
if (vt1 == ValueType.BOOLEAN_VALUE) {
result.setBooleanValue(!b1.booleanValue());
} else {
- result.copyFrom(v1);
+ result.setValue(v1);
}
return SigValues.SIG_CHLD1.ordinal();
}
@@ -1111,9 +1100,9 @@ public class Operation extends ExprTree {
result.setErrorValue();
return SigValues.SIG_CHLD1.ordinal();
} else if (vt1 == ValueType.BOOLEAN_VALUE && !b1.booleanValue()) {
- result.copyFrom(v2);
+ result.setValue(v2);
} else if (vt2 != ValueType.BOOLEAN_VALUE) {
- result.copyFrom(v2);
+ result.setValue(v2);
} else if (b2.booleanValue()) {
result.setBooleanValue(true);
} else {
@@ -1128,9 +1117,9 @@ public class Operation extends ExprTree {
result.setErrorValue();
return SigValues.SIG_CHLD1.ordinal();
} else if (vt1 == ValueType.BOOLEAN_VALUE && b1.booleanValue()) {
- result.copyFrom(v2);
+ result.setValue(v2);
} else if (vt2 != ValueType.BOOLEAN_VALUE) {
- result.copyFrom(v2);
+ result.setValue(v2);
} else if (!b2.booleanValue()) {
result.setBooleanValue(false);
} else {
@@ -1142,9 +1131,10 @@ public class Operation extends ExprTree {
throw new HyracksDataException("Shouldn't reach here");
}
- public static int doBitwise(int op, Value v1, Value v2, Value result) throws HyracksDataException {
- AMutableInt64 i1 = new AMutableInt64(0);
- AMutableInt64 i2 = new AMutableInt64(0);
+ public static int doBitwise(int op, Value v1, Value v2, Value result, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ AMutableInt64 i1 = objectPool.int64Pool.get();
+ AMutableInt64 i2 = objectPool.int64Pool.get();
// bitwise operations are defined only on integers
if (op == OpKind_BITWISE_NOT_OP) {
@@ -1209,9 +1199,10 @@ public class Operation extends ExprTree {
//out of domain value
public static final int EDOM = 33;
- public static int doRealArithmetic(int op, Value v1, Value v2, Value result) throws HyracksDataException {
- AMutableDouble r1 = new AMutableDouble(0);
- AMutableDouble r2 = new AMutableDouble(0);
+ public static int doRealArithmetic(int op, Value v1, Value v2, Value result, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ AMutableDouble r1 = objectPool.doublePool.get();
+ AMutableDouble r2 = objectPool.doublePool.get();
double comp = 0;
// we want to prevent FPE and set the ERROR value on the result; on Unix
@@ -1251,9 +1242,9 @@ public class Operation extends ExprTree {
return (SigValues.SIG_CHLD1.ordinal() | SigValues.SIG_CHLD2.ordinal());
}
- public static int doTimeArithmetic(int op, Value v1, Value v2, Value result) {
- ClassAdTime asecs1 = new ClassAdTime();
- ClassAdTime asecs2 = new ClassAdTime();
+ public static int doTimeArithmetic(int op, Value v1, Value v2, Value result, ClassAdObjectPool objectPool) {
+ ClassAdTime asecs1 = objectPool.classAdTimePool.get();
+ ClassAdTime asecs2 = objectPool.classAdTimePool.get();
ValueType vt1 = v1.getType();
ValueType vt2 = v2.getType();
@@ -1309,8 +1300,8 @@ public class Operation extends ExprTree {
if (op == OpKind_MULTIPLICATION_OP || op == OpKind_DIVISION_OP) {
if (vt1 == ValueType.RELATIVE_TIME_VALUE && vt2 == ValueType.INTEGER_VALUE) {
- AMutableInt64 num = new AMutableInt64(0);
- ClassAdTime msecs = new ClassAdTime();
+ AMutableInt64 num = objectPool.int64Pool.get();
+ ClassAdTime msecs = objectPool.classAdTimePool.get();
v1.isRelativeTimeValue(asecs1);
v2.isIntegerValue(num);
if (op == OpKind_MULTIPLICATION_OP) {
@@ -1323,8 +1314,8 @@ public class Operation extends ExprTree {
}
if (vt1 == ValueType.RELATIVE_TIME_VALUE && vt2 == ValueType.REAL_VALUE) {
- AMutableDouble num = new AMutableDouble(0);
- AMutableDouble msecs = new AMutableDouble(0);
+ AMutableDouble num = objectPool.doublePool.get();
+ AMutableDouble msecs = objectPool.doublePool.get();
v1.isRelativeTimeValue(asecs1);
v2.isRealValue(num);
if (op == OpKind_MULTIPLICATION_OP) {
@@ -1332,25 +1323,30 @@ public class Operation extends ExprTree {
} else {
msecs.setValue(asecs1.getRelativeTime() * num.getDoubleValue());
}
- result.setRelativeTimeValue(new ClassAdTime(1000L * ((long) msecs.getDoubleValue()), false));
+ ClassAdTime time = objectPool.classAdTimePool.get();
+ time.setRelativeTime(1000L * ((long) msecs.getDoubleValue()));
+ result.setRelativeTimeValue(time);
return (SigValues.SIG_CHLD1.ordinal() | SigValues.SIG_CHLD2.ordinal());
}
if (vt1 == ValueType.INTEGER_VALUE && vt2 == ValueType.RELATIVE_TIME_VALUE
&& op == OpKind_MULTIPLICATION_OP) {
- AMutableInt64 num = new AMutableInt64(0);
+ AMutableInt64 num = objectPool.int64Pool.get();
v1.isIntegerValue(num);
v2.isRelativeTimeValue(asecs1);
- result.setRelativeTimeValue(new ClassAdTime(num.getLongValue() * asecs1.getRelativeTime(), false));
+ ClassAdTime time = objectPool.classAdTimePool.get();
+ time.setRelativeTime(num.getLongValue() * asecs1.getRelativeTime());
+ result.setRelativeTimeValue(time);
return (SigValues.SIG_CHLD1.ordinal() | SigValues.SIG_CHLD2.ordinal());
}
if (vt2 == ValueType.RELATIVE_TIME_VALUE && vt1 == ValueType.REAL_VALUE && op == OpKind_MULTIPLICATION_OP) {
- AMutableDouble num = new AMutableDouble(0);
+ AMutableDouble num = objectPool.doublePool.get();
v1.isRelativeTimeValue(asecs1);
v2.isRealValue(num);
- result.setRelativeTimeValue(
- new ClassAdTime((long) (asecs1.getRelativeTime() * num.getDoubleValue()), false));
+ ClassAdTime time = objectPool.classAdTimePool.get();
+ time.setRelativeTime((long) (asecs1.getRelativeTime() * num.getDoubleValue()));
+ result.setRelativeTimeValue(time);
return (SigValues.SIG_CHLD1.ordinal() | SigValues.SIG_CHLD2.ordinal());
}
}
@@ -1359,9 +1355,9 @@ public class Operation extends ExprTree {
return (SigValues.SIG_CHLD1.ordinal() | SigValues.SIG_CHLD2.ordinal());
}
- public static void compareStrings(int op, Value v1, Value v2, Value result) {
- AMutableCharArrayString s1 = new AMutableCharArrayString();
- AMutableCharArrayString s2 = new AMutableCharArrayString();
+ public static void compareStrings(int op, Value v1, Value v2, Value result, ClassAdObjectPool objectPool) {
+ AMutableCharArrayString s1 = objectPool.strPool.get();
+ AMutableCharArrayString s2 = objectPool.strPool.get();
int cmp;
v1.isStringValue(s1);
v2.isStringValue(s2);
@@ -1392,9 +1388,10 @@ public class Operation extends ExprTree {
}
}
- public static void compareAbsoluteTimes(int op, Value v1, Value v2, Value result) throws HyracksDataException {
- ClassAdTime asecs1 = new ClassAdTime();
- ClassAdTime asecs2 = new ClassAdTime();
+ public static void compareAbsoluteTimes(int op, Value v1, Value v2, Value result, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ ClassAdTime asecs1 = objectPool.classAdTimePool.get();
+ ClassAdTime asecs2 = objectPool.classAdTimePool.get();
boolean compResult = false;
v1.isAbsoluteTimeValue(asecs1);
v2.isAbsoluteTimeValue(asecs2);
@@ -1430,9 +1427,10 @@ public class Operation extends ExprTree {
result.setBooleanValue(compResult);
}
- public static void compareRelativeTimes(int op, Value v1, Value v2, Value result) throws HyracksDataException {
- ClassAdTime rsecs1 = new ClassAdTime();
- ClassAdTime rsecs2 = new ClassAdTime();
+ public static void compareRelativeTimes(int op, Value v1, Value v2, Value result, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ ClassAdTime rsecs1 = objectPool.classAdTimePool.get();
+ ClassAdTime rsecs2 = objectPool.classAdTimePool.get();
boolean compResult = false;
v1.isRelativeTimeValue(rsecs1);
@@ -1472,9 +1470,10 @@ public class Operation extends ExprTree {
result.setBooleanValue(compResult);
}
- public static void compareBools(int op, Value v1, Value v2, Value result) throws HyracksDataException {
- MutableBoolean b1 = new MutableBoolean();
- MutableBoolean b2 = new MutableBoolean();
+ public static void compareBools(int op, Value v1, Value v2, Value result, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ MutableBoolean b1 = objectPool.boolPool.get();
+ MutableBoolean b2 = objectPool.boolPool.get();
boolean compResult = false;
v1.isBooleanValue(b1);
v2.isBooleanValue(b2);
@@ -1511,9 +1510,10 @@ public class Operation extends ExprTree {
result.setBooleanValue(compResult);
}
- public static void compareIntegers(int op, Value v1, Value v2, Value result) throws HyracksDataException {
- AMutableInt64 i1 = new AMutableInt64(0);
- AMutableInt64 i2 = new AMutableInt64(0);
+ public static void compareIntegers(int op, Value v1, Value v2, Value result, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ AMutableInt64 i1 = objectPool.int64Pool.get();
+ AMutableInt64 i2 = objectPool.int64Pool.get();
boolean compResult = false;
v1.isIntegerValue(i1);
v2.isIntegerValue(i2);
@@ -1549,9 +1549,10 @@ public class Operation extends ExprTree {
result.setBooleanValue(compResult);
}
- public static void compareReals(int op, Value v1, Value v2, Value result) throws HyracksDataException {
- AMutableDouble r1 = new AMutableDouble(0);
- AMutableDouble r2 = new AMutableDouble(0);
+ public static void compareReals(int op, Value v1, Value v2, Value result, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ AMutableDouble r1 = objectPool.doublePool.get();
+ AMutableDouble r2 = objectPool.doublePool.get();
boolean compResult = false;
v1.isRealValue(r1);
@@ -1594,26 +1595,33 @@ public class Operation extends ExprTree {
// + if both v1 and v2 are Numbers and of the same type, return type
// + if v1 is an int and v2 is a real, convert v1 to real; return REAL_VALUE
// + if v1 is a real and v2 is an int, convert v2 to real; return REAL_VALUE
- public static ValueType coerceToNumber(Value v1, Value v2) {
- AMutableInt64 i = new AMutableInt64(0);
- AMutableDouble r = new AMutableDouble(0);
- MutableBoolean b = new MutableBoolean();
+ public static ValueType coerceToNumber(Value v1, Value v2, ClassAdObjectPool objectPool) {
+ AMutableInt64 i = objectPool.int64Pool.get();
+ AMutableDouble r = objectPool.doublePool.get();
+ MutableBoolean b = objectPool.boolPool.get();
// either of v1, v2 not numerical?
- if (v1.isClassAdValue() || v2.isClassAdValue())
+ if (v1.isClassAdValue() || v2.isClassAdValue()) {
return ValueType.CLASSAD_VALUE;
- if (v1.isListValue() || v2.isListValue())
+ }
+ if (v1.isListValue() || v2.isListValue()) {
return ValueType.LIST_VALUE;
- if (v1.isStringValue() || v2.isStringValue())
+ }
+ if (v1.isStringValue() || v2.isStringValue()) {
return ValueType.STRING_VALUE;
- if (v1.isUndefinedValue() || v2.isUndefinedValue())
+ }
+ if (v1.isUndefinedValue() || v2.isUndefinedValue()) {
return ValueType.UNDEFINED_VALUE;
- if (v1.isErrorValue() || v2.isErrorValue())
+ }
+ if (v1.isErrorValue() || v2.isErrorValue()) {
return ValueType.ERROR_VALUE;
- if (v1.isAbsoluteTimeValue() || v2.isAbsoluteTimeValue())
+ }
+ if (v1.isAbsoluteTimeValue() || v2.isAbsoluteTimeValue()) {
return ValueType.ABSOLUTE_TIME_VALUE;
- if (v1.isRelativeTimeValue() || v2.isRelativeTimeValue())
+ }
+ if (v1.isRelativeTimeValue() || v2.isRelativeTimeValue()) {
return ValueType.RELATIVE_TIME_VALUE;
+ }
// promote booleans to integers
if (v1.isBooleanValue(b)) {
@@ -1633,41 +1641,51 @@ public class Operation extends ExprTree {
}
// both v1 and v2 of same numerical type
- if (v1.isIntegerValue(i) && v2.isIntegerValue(i))
+ if (v1.isIntegerValue(i) && v2.isIntegerValue(i)) {
return ValueType.INTEGER_VALUE;
- if (v1.isRealValue(r) && v2.isRealValue(r))
+ }
+ if (v1.isRealValue(r) && v2.isRealValue(r)) {
return ValueType.REAL_VALUE;
+ }
// type promotions required
- if (v1.isIntegerValue(i) && v2.isRealValue(r))
+ if (v1.isIntegerValue(i) && v2.isRealValue(r)) {
v1.setRealValue(i.getLongValue());
- else if (v1.isRealValue(r) && v2.isIntegerValue(i))
+ } else if (v1.isRealValue(r) && v2.isIntegerValue(i)) {
v2.setRealValue(i.getLongValue());
+ }
return ValueType.REAL_VALUE;
}
- public Operation(int op, ExprTreeHolder e1, ExprTreeHolder e2, ExprTreeHolder e3) {
+ public Operation(int op, ExprTreeHolder e1, ExprTreeHolder e2, ExprTreeHolder e3, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ super(objectPool);
this.opKind = op;
- this.child1 = e1 == null ? null : e1.self();
- this.child2 = e2 == null ? null : e2.self();
- this.child3 = e3 == null ? null : e3.self();
+ this.child1 = new ExprTreeHolder(objectPool);
+ this.child2 = new ExprTreeHolder(objectPool);
+ this.child3 = new ExprTreeHolder(objectPool);
+ child1.copyFrom(e1);
+ child2.copyFrom(e2);
+ child3.copyFrom(e3);
}
- public static Operation createOperation(int op, ExprTree e1, ExprTree e2, ExprTree e3) {
- Operation opnode = new Operation();
+ public static Operation createOperation(int op, ExprTree e1, ExprTree e2, ExprTree e3, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ Operation opnode = objectPool.operationPool.get();
opnode.opKind = op;
- opnode.child1 = e1 == null ? null : e1.self();
- opnode.child2 = e2 == null ? null : e2.self();
- opnode.child3 = e3 == null ? null : e3.self();
+ opnode.child1.copyFrom(e1);
+ opnode.child2.copyFrom(e2);
+ opnode.child3.copyFrom(e3);
return opnode;
}
- public static void createOperation(int op, ExprTree e1, ExprTree e2, ExprTree e3, Operation opnode) {
+ public static void createOperation(int op, ExprTree e1, ExprTree e2, ExprTree e3, Operation opnode)
+ throws HyracksDataException {
opnode.opKind = op;
- opnode.child1 = e1 == null ? null : e1.self();
- opnode.child2 = e2 == null ? null : e2.self();
- opnode.child3 = e3 == null ? null : e3.self();
+ opnode.child1.copyFrom(e1);
+ opnode.child2.copyFrom(e2);
+ opnode.child3.copyFrom(e3);
}
public void getComponents(AMutableInt32 op, ExprTreeHolder e1, ExprTreeHolder e2, ExprTreeHolder e3) {
@@ -1677,37 +1695,39 @@ public class Operation extends ExprTree {
e3.setInnerTree(child3);
}
- public static Operation createOperation(int op, Value val, ExprTreeHolder tree) throws HyracksDataException {
+ public static Operation createOperation(int op, Value val, ExprTreeHolder tree, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
if (tree.getInnerTree() == null) {
return null;
}
- Literal lit = Literal.createLiteral(val);
+ Literal lit = Literal.createLiteral(val, objectPool);
if (lit == null) {
return null;
}
- Operation newOp = createOperation(op, lit, tree);
+ Operation newOp = createOperation(op, lit, tree, objectPool);
return newOp;
}
- public static Operation createOperation(int op, ExprTreeHolder tree, Value val) throws HyracksDataException {
+ public static Operation createOperation(int op, ExprTreeHolder tree, Value val, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
if (tree.getInnerTree() == null) {
return null;
}
- Literal lit = Literal.createLiteral(val);
+ Literal lit = Literal.createLiteral(val, objectPool);
if (lit == null) {
return null;
}
- Operation newOp = createOperation(op, lit, tree);
+ Operation newOp = createOperation(op, lit, tree, objectPool);
return newOp;
}
public boolean flattenSpecials(EvalState state, Value val, ExprTreeHolder tree) throws HyracksDataException {
- ExprTreeHolder fChild1 = new ExprTreeHolder();
- ExprTreeHolder fChild2 = new ExprTreeHolder();
- ExprTreeHolder fChild3 = new ExprTreeHolder();
- Value eval1 = new Value();
- Value eval2 = new Value();
- Value eval3 = new Value();
+ ExprTreeHolder fChild1 = objectPool.mutableExprPool.get();
+ ExprTreeHolder fChild2 = objectPool.mutableExprPool.get();
+ ExprTreeHolder fChild3 = objectPool.mutableExprPool.get();
+ Value eval1 = objectPool.valuePool.get();
+ Value eval2 = objectPool.valuePool.get();
+ Value eval3 = objectPool.valuePool.get();
switch (opKind) {
case OpKind_UNARY_PLUS_OP:
@@ -1720,12 +1740,12 @@ public class Operation extends ExprTree {
return false;
}
if (fChild1.getInnerTree() != null) {
- tree.setInnerTree(Operation.createOperation(opKind, fChild1));
+ tree.setInnerTree(Operation.createOperation(opKind, fChild1, objectPool));
return (tree.getInnerTree() != null);
} else {
- privateDoOperation(opKind, eval1, null, null, true, false, false, val);
+ privateDoOperation(opKind, eval1, null, null, true, false, false, val, objectPool);
tree.setInnerTree(null);
- eval1.clear();
+ eval1.setUndefinedValue();
return true;
}
case OpKind_TERNARY_OP:
@@ -1737,11 +1757,11 @@ public class Operation extends ExprTree {
// check if selector expression collapsed to a non-undefined value
if (fChild1.getInnerTree() == null && !eval1.isUndefinedValue()) {
- MutableBoolean b = new MutableBoolean();
+ MutableBoolean b = objectPool.boolPool.get();
// if the selector is not boolean-equivalent, propagate error
if (!eval1.isBooleanValueEquiv(b)) {
val.setErrorValue();
- eval1.clear();
+ eval1.setUndefinedValue();
tree.setInnerTree(null);
return true;
}
@@ -1761,10 +1781,12 @@ public class Operation extends ExprTree {
}
// if any arm collapsed into a value, make it a Literal
- if (fChild2.getInnerTree() == null)
- fChild2.setInnerTree(Literal.createLiteral(eval2));
- if (fChild3.getInnerTree() == null)
- fChild3.setInnerTree(Literal.createLiteral(eval3));
+ if (fChild2.getInnerTree() == null) {
+ fChild2.setInnerTree(Literal.createLiteral(eval2, objectPool));
+ }
+ if (fChild3.getInnerTree() == null) {
+ fChild3.setInnerTree(Literal.createLiteral(eval3, objectPool));
+ }
if (fChild2.getInnerTree() == null || fChild3.getInnerTree() == null) {
tree.setInnerTree(null);;
return false;
@@ -1775,7 +1797,7 @@ public class Operation extends ExprTree {
fChild1.setInnerTree(child1.copy());
}
- tree.setInnerTree(Operation.createOperation(opKind, fChild1, fChild2, fChild3));
+ tree.setInnerTree(Operation.createOperation(opKind, fChild1, fChild2, fChild3, objectPool));
if (tree.getInnerTree() == null) {
return false;
}
@@ -1790,22 +1812,24 @@ public class Operation extends ExprTree {
// if both arguments Flattened to values, Evaluate now
if (fChild1.getInnerTree() == null && fChild2.getInnerTree() == null) {
- privateDoOperation(opKind, eval1, eval2, null, true, true, false, val);
+ privateDoOperation(opKind, eval1, eval2, null, true, true, false, val, objectPool);
tree.setInnerTree(null);
return true;
}
// otherwise convert Flattened values into literals
- if (fChild1.getInnerTree() == null)
- fChild1.setInnerTree(Literal.createLiteral(eval1));
- if (fChild2.getInnerTree() == null)
- fChild2.setInnerTree(Literal.createLiteral(eval2));
+ if (fChild1.getInnerTree() == null) {
+ fChild1.setInnerTree(Literal.createLiteral(eval1, objectPool));
+ }
+ if (fChild2.getInnerTree() == null) {
+ fChild2.setInnerTree(Literal.createLiteral(eval2, objectPool));
+ }
if (fChild1.getInnerTree() == null || fChild2.getInnerTree() == null) {
tree.setInnerTree(null);
return false;
}
- tree.setInnerTree(Operation.createOperation(opKind, fChild1, fChild2));
+ tree.setInnerTree(Operation.createOperation(opKind, fChild1, fChild2, objectPool));
if (tree.getInnerTree() == null) {
return false;
}
@@ -1892,11 +1916,8 @@ public class Operation extends ExprTree {
@Override
public void reset() {
opKind = OpKind_NO_OP;
- if (child1 != null)
- child1.reset();
- if (child2 != null)
- child2.reset();
- if (child3 != null)
- child3.reset();
+ child1.reset();
+ child2.reset();
+ child3.reset();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/PrettyPrint.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/PrettyPrint.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/PrettyPrint.java
index e738bb9..2553bfa 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/PrettyPrint.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/PrettyPrint.java
@@ -22,6 +22,7 @@ import java.util.Map;
import java.util.Map.Entry;
import org.apache.asterix.external.classad.ExprTree.NodeKind;
+import org.apache.asterix.external.classad.object.pool.ClassAdObjectPool;
import org.apache.asterix.om.base.AMutableInt32;
import org.apache.hyracks.api.exceptions.HyracksDataException;
@@ -32,7 +33,8 @@ public class PrettyPrint extends ClassAdUnParser {
private boolean minimalParens;
private int indentLevel;
- public PrettyPrint() {
+ public PrettyPrint(ClassAdObjectPool objectPool) {
+ super(objectPool);
classadIndent = 4;
listIndent = 3;
wantStringQuotes = true;
@@ -116,8 +118,9 @@ public class PrettyPrint extends ClassAdUnParser {
return;
}
// all others are binary ops
- AMutableInt32 top = new AMutableInt32(0);
- ExprTreeHolder t1 = new ExprTreeHolder(), t2 = new ExprTreeHolder(), t3 = new ExprTreeHolder();
+ AMutableInt32 top = objectPool.int32Pool.get();
+ ExprTreeHolder t1 = objectPool.mutableExprPool.get(), t2 = objectPool.mutableExprPool.get(),
+ t3 = objectPool.mutableExprPool.get();
if (op1.getKind() == NodeKind.OP_NODE) {
((Operation) op1.getInnerTree()).getComponents(top, t1, t2, t3);
[10/13] incubator-asterixdb git commit: Improve Error Handling in
Local Directory Feeds
Posted by am...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-app/src/test/resources/runtimets/testsuite.xml
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/testsuite.xml b/asterix-app/src/test/resources/runtimets/testsuite.xml
index f159135..3a5140c 100644
--- a/asterix-app/src/test/resources/runtimets/testsuite.xml
+++ b/asterix-app/src/test/resources/runtimets/testsuite.xml
@@ -28,8 +28,35 @@
ResultOffsetPath="results"
QueryOffsetPath="queries"
QueryFileExtension=".aql">
+ <test-group name="external-library">
+ <test-case FilePath="external-library">
+ <compilation-unit name="typed_adapter">
+ <output-dir compare="Text">typed_adapter</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-library">
+ <compilation-unit name="classad-parser">
+ <output-dir compare="Text">classad-parser</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-library">
+ <compilation-unit name="classad-parser2">
+ <output-dir compare="Text">classad-parser2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-library">
+ <compilation-unit name="getCapital">
+ <output-dir compare="Text">getCapital</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
<test-group name="feeds">
<test-case FilePath="feeds">
+ <compilation-unit name="feed-with-external-parser">
+ <output-dir compare="Text">feed-with-external-parser</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="feeds">
<compilation-unit name="feed-with-external-function">
<output-dir compare="Text">feed-with-external-function</output-dir>
</compilation-unit>
@@ -59,11 +86,6 @@
<output-dir compare="Text">feeds_07</output-dir>
</compilation-unit>
</test-case>
- <test-case FilePath="feeds">
- <compilation-unit name="feed-with-external-parser">
- <output-dir compare="Text">feed-with-external-parser</output-dir>
- </compilation-unit>
- </test-case>
<!-- Fails constantly and not clear what is intended
<test-case FilePath="feeds">
<compilation-unit name="feeds_06">
@@ -193,28 +215,6 @@
</compilation-unit>
</test-case>
</test-group>
- <test-group name="external-library">
- <test-case FilePath="external-library">
- <compilation-unit name="typed_adapter">
- <output-dir compare="Text">typed_adapter</output-dir>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-library">
- <compilation-unit name="classad-parser">
- <output-dir compare="Text">classad-parser</output-dir>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-library">
- <compilation-unit name="classad-parser2">
- <output-dir compare="Text">classad-parser2</output-dir>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-library">
- <compilation-unit name="getCapital">
- <output-dir compare="Text">getCapital</output-dir>
- </compilation-unit>
- </test-case>
- </test-group>
<test-group name="flwor">
<test-case FilePath="flwor">
<compilation-unit name="at00">
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-common/src/main/java/org/apache/asterix/common/dataflow/AsterixLSMInsertDeleteOperatorNodePushable.java
----------------------------------------------------------------------
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/dataflow/AsterixLSMInsertDeleteOperatorNodePushable.java b/asterix-common/src/main/java/org/apache/asterix/common/dataflow/AsterixLSMInsertDeleteOperatorNodePushable.java
index afdbb31..d53236b 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/dataflow/AsterixLSMInsertDeleteOperatorNodePushable.java
+++ b/asterix-common/src/main/java/org/apache/asterix/common/dataflow/AsterixLSMInsertDeleteOperatorNodePushable.java
@@ -83,7 +83,6 @@ public class AsterixLSMInsertDeleteOperatorNodePushable extends LSMIndexInsertUp
@Override
public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
- boolean first = true;
accessor.reset(buffer);
ILSMIndexAccessor lsmAccessor = (ILSMIndexAccessor) indexAccessor;
int tupleCount = accessor.getTupleCount();
@@ -98,17 +97,15 @@ public class AsterixLSMInsertDeleteOperatorNodePushable extends LSMIndexInsertUp
tuple.reset(accessor, i);
switch (op) {
case INSERT:
- if (first && isPrimary) {
+ if (i == 0 && isPrimary) {
lsmAccessor.insert(tuple);
- first = false;
} else {
lsmAccessor.forceInsert(tuple);
}
break;
case DELETE:
- if (first && isPrimary) {
+ if (i == 0 && isPrimary) {
lsmAccessor.delete(tuple);
- first = false;
} else {
lsmAccessor.forceDelete(tuple);
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/api/AsterixInputStream.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/api/AsterixInputStream.java b/asterix-external-data/src/main/java/org/apache/asterix/external/api/AsterixInputStream.java
new file mode 100644
index 0000000..83d7a3a
--- /dev/null
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/api/AsterixInputStream.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.api;
+
+import java.io.InputStream;
+
+import org.apache.asterix.external.dataflow.AbstractFeedDataFlowController;
+import org.apache.asterix.external.util.FeedLogManager;
+
+public abstract class AsterixInputStream extends InputStream {
+
+ protected AbstractFeedDataFlowController controller;
+ protected FeedLogManager logManager;
+
+ public abstract boolean stop() throws Exception;
+
+ public abstract boolean handleException(Throwable th);
+
+ // TODO: Find a better way to send notifications
+ public void setController(AbstractFeedDataFlowController controller) {
+ this.controller = controller;
+ }
+
+ // TODO: Find a better way to send notifications
+ public void setFeedLogManager(FeedLogManager logManager) {
+ this.logManager = logManager;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IExternalIndexer.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IExternalIndexer.java b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IExternalIndexer.java
index 01ffd99..c261ae3 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IExternalIndexer.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IExternalIndexer.java
@@ -36,7 +36,7 @@ public interface IExternalIndexer extends Serializable {
* the new reader
* @throws Exception
*/
- public void reset(IRecordReader<?> reader) throws IOException;
+ public void reset(IIndexingDatasource reader) throws IOException;
/**
* This method is called by the dataflow controller with each tuple. the indexer is expected to append record ids to the tuple.
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IIndexingDatasource.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IIndexingDatasource.java b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IIndexingDatasource.java
index c247ef6..5381ef7 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IIndexingDatasource.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IIndexingDatasource.java
@@ -18,6 +18,33 @@
*/
package org.apache.asterix.external.api;
+import java.util.List;
+
+import org.apache.asterix.external.indexing.ExternalFile;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.mapred.RecordReader;
+
+/**
+ * An interface for external data sources which support indexing
+ */
public interface IIndexingDatasource {
+ /**
+ * @return an external indexer that is used to write RID fields for each record
+ */
public IExternalIndexer getIndexer();
+
+ /**
+ * @return a list of external files being accessed
+ */
+ public List<ExternalFile> getSnapshot();
+
+ /**
+ * @return the index of the currently being read file
+ */
+ public int getCurrentSplitIndex();
+
+ /**
+ * @return an HDFS record reader that is used to get the current position in the file
+ */
+ public RecordReader<?, ? extends Writable> getReader();
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamFactory.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamFactory.java
new file mode 100644
index 0000000..a460e49
--- /dev/null
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamFactory.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.api;
+
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+public interface IInputStreamFactory extends IExternalDataSourceFactory {
+
+ public AsterixInputStream createInputStream(IHyracksTaskContext ctx, int partition) throws HyracksDataException;
+
+ @Override
+ public default DataSourceType getDataSourceType() {
+ return DataSourceType.STREAM;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamProvider.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamProvider.java b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamProvider.java
deleted file mode 100644
index b10452e..0000000
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamProvider.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.external.api;
-
-import org.apache.asterix.external.input.stream.AInputStream;
-import org.apache.asterix.external.util.FeedLogManager;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-
-public interface IInputStreamProvider {
- public AInputStream getInputStream() throws HyracksDataException;
-
- public void setFeedLogManager(FeedLogManager feedLogManager);
-}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamProviderFactory.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamProviderFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamProviderFactory.java
deleted file mode 100644
index f52f7d3..0000000
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamProviderFactory.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.external.api;
-
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-
-public interface IInputStreamProviderFactory extends IExternalDataSourceFactory {
-
- public IInputStreamProvider createInputStreamProvider(IHyracksTaskContext ctx, int partition)
- throws HyracksDataException;
-
- @Override
- public default DataSourceType getDataSourceType() {
- return DataSourceType.STREAM;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReader.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReader.java b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReader.java
index b4d67d4..11e2472 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReader.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReader.java
@@ -21,6 +21,7 @@ package org.apache.asterix.external.api;
import java.io.Closeable;
import java.io.IOException;
+import org.apache.asterix.external.dataflow.AbstractFeedDataFlowController;
import org.apache.asterix.external.util.FeedLogManager;
/**
@@ -54,7 +55,7 @@ public interface IRecordReader<T> extends Closeable {
* set a pointer to the controller of the feed. the controller can be used to flush()
* parsed records when waiting for more records to be pushed
*/
- public void setController(IDataFlowController controller);
+ public void setController(AbstractFeedDataFlowController controller);
// TODO: Find a better way to perform logging. this doesn't fit here
/**
@@ -62,4 +63,9 @@ public interface IRecordReader<T> extends Closeable {
* progress and errors
*/
public void setFeedLogManager(FeedLogManager feedLogManager);
+
+ /**
+ * gives the record reader a chance to recover from IO errors during feed intake
+ */
+ public boolean handleException(Throwable th);
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java b/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java
index 2cc3c66..6401234 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java
@@ -67,22 +67,29 @@ public class FeedRecordDataFlowController<T> extends AbstractFeedDataFlowControl
continue;
}
tb.reset();
- dataParser.parse(record, tb.getDataOutput());
+ try {
+ dataParser.parse(record, tb.getDataOutput());
+ } catch (Exception e) {
+ e.printStackTrace();
+ LOGGER.warn(ExternalDataConstants.ERROR_PARSE_RECORD, e);
+ feedLogManager.logRecord(record.toString(), ExternalDataConstants.ERROR_PARSE_RECORD);
+ continue;
+ }
tb.addFieldEndOffset();
addMetaPart(tb, record);
addPrimaryKeys(tb, record);
if (tb.getSize() > tupleForwarder.getMaxRecordSize()) {
// log
- feedLogManager.logRecord(record.toString(), ExternalDataConstants.LARGE_RECORD_ERROR_MESSAGE);
+ feedLogManager.logRecord(record.toString(), ExternalDataConstants.ERROR_LARGE_RECORD);
continue;
}
tupleForwarder.addTuple(tb);
}
- } catch (Throwable th) {
+ } catch (Exception e) {
failed = true;
tupleForwarder.flush();
- LOGGER.warn("Failure during while operating a feed source", th);
- throw new HyracksDataException(th);
+ LOGGER.warn("Failure while operating a feed source", e);
+ throw new HyracksDataException(e);
}
try {
tupleForwarder.close();
@@ -156,6 +163,7 @@ public class FeedRecordDataFlowController<T> extends AbstractFeedDataFlowControl
@Override
public boolean handleException(Throwable th) {
- return true;
+ // This is not a parser record. most likely, this error happened in the record reader.
+ return recordReader.handleException(th);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedStreamDataFlowController.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedStreamDataFlowController.java b/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedStreamDataFlowController.java
index f233971..ed5e355 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedStreamDataFlowController.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedStreamDataFlowController.java
@@ -18,8 +18,8 @@
*/
package org.apache.asterix.external.dataflow;
+import org.apache.asterix.external.api.AsterixInputStream;
import org.apache.asterix.external.api.IStreamDataParser;
-import org.apache.asterix.external.input.stream.AInputStream;
import org.apache.asterix.external.util.FeedLogManager;
import org.apache.hyracks.api.comm.IFrameWriter;
import org.apache.hyracks.api.context.IHyracksTaskContext;
@@ -28,10 +28,11 @@ import org.apache.hyracks.api.exceptions.HyracksDataException;
public class FeedStreamDataFlowController extends AbstractFeedDataFlowController {
private final IStreamDataParser dataParser;
- private final AInputStream stream;
+ private final AsterixInputStream stream;
public FeedStreamDataFlowController(IHyracksTaskContext ctx, FeedTupleForwarder tupleForwarder,
- FeedLogManager feedLogManager, int numOfFields, IStreamDataParser streamParser, AInputStream inputStream) {
+ FeedLogManager feedLogManager, int numOfFields, IStreamDataParser streamParser,
+ AsterixInputStream inputStream) {
super(ctx, tupleForwarder, feedLogManager, numOfFields);
this.dataParser = streamParser;
this.stream = inputStream;
@@ -73,7 +74,7 @@ public class FeedStreamDataFlowController extends AbstractFeedDataFlowController
public boolean handleException(Throwable th) {
boolean handled = true;
try {
- handled &= stream.skipError();
+ handled &= stream.handleException(th);
if (handled) {
handled &= dataParser.reset(stream);
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedTupleForwarder.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedTupleForwarder.java b/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedTupleForwarder.java
index 73de838..63787ee 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedTupleForwarder.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedTupleForwarder.java
@@ -77,7 +77,7 @@ public class FeedTupleForwarder implements ITupleForwarder {
public void addTuple(ArrayTupleBuilder tb) throws HyracksDataException {
if (tb.getSize() > maxRecordSize) {
try {
- feedLogManager.logRecord(tb.toString(), ExternalDataConstants.LARGE_RECORD_ERROR_MESSAGE);
+ feedLogManager.logRecord(tb.toString(), ExternalDataConstants.ERROR_LARGE_RECORD);
} catch (IOException e) {
throw new HyracksDataException(e);
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedFrameHandler.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedFrameHandler.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedFrameHandler.java
index 9cb0feb..3b37500 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedFrameHandler.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedFrameHandler.java
@@ -26,9 +26,9 @@ import org.apache.hyracks.api.exceptions.HyracksDataException;
public interface IFeedFrameHandler {
- public void handleFrame(ByteBuffer frame) throws HyracksDataException;
+ public void handleFrame(ByteBuffer frame) throws HyracksDataException, InterruptedException;
- public void handleDataBucket(DataBucket bucket);
+ public void handleDataBucket(DataBucket bucket) throws InterruptedException;
public void close();
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedJoint.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedJoint.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedJoint.java
index 2a3932d..477ec36 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedJoint.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedJoint.java
@@ -20,7 +20,7 @@ package org.apache.asterix.external.feed.api;
import java.util.List;
-import org.apache.asterix.external.feed.api.IFeedLifecycleListener.ConnectionLocation;
+import org.apache.asterix.external.feed.api.IFeedRuntime.FeedRuntimeType;
import org.apache.asterix.external.feed.management.FeedConnectionId;
import org.apache.asterix.external.feed.management.FeedConnectionRequest;
import org.apache.asterix.external.feed.management.FeedId;
@@ -71,7 +71,7 @@ public interface IFeedJoint {
/**
* @return the subscription location {@link ConnectionLocation} associated with the FeedJoint
*/
- public ConnectionLocation getConnectionLocation();
+ public FeedRuntimeType getConnectionLocation();
/**
* @return the unique {@link FeedJointKey} associated with the FeedJoint
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedLifecycleListener.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedLifecycleListener.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedLifecycleListener.java
index ce82aaf..28b713e 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedLifecycleListener.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IFeedLifecycleListener.java
@@ -27,12 +27,6 @@ import org.apache.asterix.external.feed.management.FeedJointKey;
import org.apache.hyracks.api.job.IJobLifecycleListener;
public interface IFeedLifecycleListener extends IJobLifecycleListener, IClusterEventsSubscriber {
-
- public enum ConnectionLocation {
- SOURCE_FEED_INTAKE_STAGE,
- SOURCE_FEED_COMPUTE_STAGE
- }
-
public IFeedJoint getAvailableFeedJoint(FeedJointKey feedJoinKey);
public boolean isFeedJointAvailable(FeedJointKey feedJoinKey);
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IMessageReceiver.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IMessageReceiver.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IMessageReceiver.java
index bdfbfdb..e58d99e 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IMessageReceiver.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/IMessageReceiver.java
@@ -20,7 +20,7 @@ package org.apache.asterix.external.feed.api;
public interface IMessageReceiver<T> {
- public void sendMessage(T message);
+ public void sendMessage(T message) throws InterruptedException;
public void close(boolean processPending);
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/DistributeFeedFrameWriter.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/DistributeFeedFrameWriter.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/DistributeFeedFrameWriter.java
index d314f74..8b7e2ba 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/DistributeFeedFrameWriter.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/DistributeFeedFrameWriter.java
@@ -66,7 +66,7 @@ public class DistributeFeedFrameWriter implements IFrameWriter {
public DistributeFeedFrameWriter(IHyracksTaskContext ctx, FeedId feedId, IFrameWriter writer,
FeedRuntimeType feedRuntimeType, int partition, FrameTupleAccessor fta, IFeedManager feedManager)
- throws IOException {
+ throws IOException {
this.feedId = feedId;
this.frameDistributor = new FrameDistributor(feedId, feedRuntimeType, partition, true,
feedManager.getFeedMemoryManager(), fta);
@@ -108,7 +108,7 @@ public class DistributeFeedFrameWriter implements IFrameWriter {
}
}
- public void notifyEndOfFeed() {
+ public void notifyEndOfFeed() throws InterruptedException {
frameDistributor.notifyEndOfFeed();
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedCollectRuntimeInputHandler.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedCollectRuntimeInputHandler.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedCollectRuntimeInputHandler.java
index 4a361fb..447c4bc 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedCollectRuntimeInputHandler.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedCollectRuntimeInputHandler.java
@@ -19,7 +19,6 @@
package org.apache.asterix.external.feed.dataflow;
import java.io.IOException;
-import java.nio.ByteBuffer;
import org.apache.asterix.external.feed.api.IFeedManager;
import org.apache.asterix.external.feed.management.FeedConnectionId;
@@ -35,8 +34,8 @@ public class FeedCollectRuntimeInputHandler extends FeedRuntimeInputHandler {
private final FeedFrameCache feedFrameCache;
- public FeedCollectRuntimeInputHandler(IHyracksTaskContext ctx, FeedConnectionId connectionId, FeedRuntimeId runtimeId,
- IFrameWriter coreOperator, FeedPolicyAccessor fpa, boolean bufferingEnabled,
+ public FeedCollectRuntimeInputHandler(IHyracksTaskContext ctx, FeedConnectionId connectionId,
+ FeedRuntimeId runtimeId, IFrameWriter coreOperator, FeedPolicyAccessor fpa, boolean bufferingEnabled,
FrameTupleAccessor fta, RecordDescriptor recordDesc, IFeedManager feedManager, int nPartitions)
throws IOException {
super(ctx, connectionId, runtimeId, coreOperator, fpa, bufferingEnabled, fta, recordDesc, feedManager,
@@ -44,11 +43,6 @@ public class FeedCollectRuntimeInputHandler extends FeedRuntimeInputHandler {
this.feedFrameCache = new FeedFrameCache(ctx, fta, coreOperator);
}
- public void process(ByteBuffer frame) throws HyracksDataException {
- feedFrameCache.sendMessage(frame);
- super.process(frame);
- }
-
public void replayFrom(int recordId) throws HyracksDataException {
feedFrameCache.replayRecords(recordId);
}
@@ -57,8 +51,7 @@ public class FeedCollectRuntimeInputHandler extends FeedRuntimeInputHandler {
feedFrameCache.dropTillRecordId(recordId);
}
- public void replayCached() throws HyracksDataException{
+ public void replayCached() throws HyracksDataException {
feedFrameCache.replayAll();
}
-
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedFrameHandlers.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedFrameHandlers.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedFrameHandlers.java
index 3c45a20..2a6fd79 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedFrameHandlers.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedFrameHandlers.java
@@ -141,9 +141,9 @@ public class FeedFrameHandlers {
}
@Override
- public void handleDataBucket(DataBucket bucket) {
+ public void handleDataBucket(DataBucket bucket) throws InterruptedException {
for (FeedFrameCollector collector : frameCollectors) {
- collector.sendMessage(bucket); // asynchronous call
+ collector.sendMessage(bucket);
}
}
@@ -174,7 +174,7 @@ public class FeedFrameHandlers {
}
@Override
- public void handleFrame(ByteBuffer frame) throws HyracksDataException {
+ public void handleFrame(ByteBuffer frame) throws HyracksDataException, InterruptedException {
receiver.sendMessage(frame);
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedRuntimeInputHandler.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedRuntimeInputHandler.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedRuntimeInputHandler.java
index 1ceb36b..dcd03a9 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedRuntimeInputHandler.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedRuntimeInputHandler.java
@@ -71,13 +71,13 @@ public class FeedRuntimeInputHandler implements IFrameWriter {
private final FrameEventCallback frameEventCallback;
private boolean bufferingEnabled;
- private IFrameWriter coreOperator;
private FrameCollection frameCollection;
private Mode mode;
private Mode lastMode;
private boolean finished;
private long nProcessed;
private boolean throttlingEnabled;
+ protected IFrameWriter coreOperator;
public FeedRuntimeInputHandler(IHyracksTaskContext ctx, FeedConnectionId connectionId, FeedRuntimeId runtimeId,
IFrameWriter coreOperator, FeedPolicyAccessor fpa, boolean bufferingEnabled, FrameTupleAccessor fta,
@@ -263,7 +263,6 @@ public class FeedRuntimeInputHandler implements IFrameWriter {
}
} else {
coreOperator.nextFrame(frame); // synchronous
- mBuffer.sendReport(frame);
}
} else {
DataBucket bucket = pool.getDataBucket();
@@ -374,8 +373,8 @@ public class FeedRuntimeInputHandler implements IFrameWriter {
}
mBuffer.close(false, disableMonitoring);
if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Closed input side handler for " + this.runtimeId + " disabled monitoring "
- + disableMonitoring + " Mode for runtime " + this.mode);
+ LOGGER.info("Closed input side handler for " + this.runtimeId + " disabled monitoring " + disableMonitoring
+ + " Mode for runtime " + this.mode);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FrameDistributor.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FrameDistributor.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FrameDistributor.java
index 0c9017b..802a791 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FrameDistributor.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FrameDistributor.java
@@ -74,7 +74,7 @@ public class FrameDistributor {
public FrameDistributor(FeedId feedId, FeedRuntimeType feedRuntimeType, int partition,
boolean enableSynchronousTransfer, IFeedMemoryManager memoryManager, FrameTupleAccessor fta)
- throws HyracksDataException {
+ throws HyracksDataException {
this.feedId = feedId;
this.feedRuntimeType = feedRuntimeType;
this.partition = partition;
@@ -85,7 +85,7 @@ public class FrameDistributor {
this.fta = fta;
}
- public void notifyEndOfFeed() {
+ public void notifyEndOfFeed() throws InterruptedException {
DataBucket bucket = getDataBucket();
if (bucket != null) {
sendEndOfFeedDataBucket(bucket);
@@ -104,7 +104,7 @@ public class FrameDistributor {
}
}
- private void sendEndOfFeedDataBucket(DataBucket bucket) {
+ private void sendEndOfFeedDataBucket(DataBucket bucket) throws InterruptedException {
bucket.setContentType(DataBucket.ContentType.EOD);
nextBucket(bucket);
if (LOGGER.isLoggable(Level.INFO)) {
@@ -207,45 +207,50 @@ public class FrameDistributor {
}
public synchronized void nextFrame(ByteBuffer frame) throws HyracksDataException {
- switch (distributionMode) {
- case INACTIVE:
- break;
- case SINGLE:
- FeedFrameCollector collector = registeredCollectors.values().iterator().next();
- switch (collector.getState()) {
- case HANDOVER:
- case ACTIVE:
- if (enableSynchronousTransfer) {
- collector.nextFrame(frame); // processing is synchronous
- } else {
+ try {
+ switch (distributionMode) {
+ case INACTIVE:
+ break;
+ case SINGLE:
+ FeedFrameCollector collector = registeredCollectors.values().iterator().next();
+ switch (collector.getState()) {
+ case HANDOVER:
+ case ACTIVE:
+ if (enableSynchronousTransfer) {
+ collector.nextFrame(frame); // processing is synchronous
+ } else {
+ handleDataBucket(frame);
+ }
+ break;
+ case TRANSITION:
handleDataBucket(frame);
- }
- break;
- case TRANSITION:
- handleDataBucket(frame);
- break;
- case FINISHED:
- if (LOGGER.isLoggable(Level.WARNING)) {
- LOGGER.warning("Discarding fetched tuples, feed has ended [" + registeredCollectors.get(0)
- + "]" + " Feed Id " + feedId + " frame distributor " + this.getFeedRuntimeType());
- }
- registeredCollectors.remove(0);
- break;
- }
- break;
- case SHARED:
- handleDataBucket(frame);
- break;
+ break;
+ case FINISHED:
+ if (LOGGER.isLoggable(Level.WARNING)) {
+ LOGGER.warning("Discarding fetched tuples, feed has ended ["
+ + registeredCollectors.get(0) + "]" + " Feed Id " + feedId
+ + " frame distributor " + this.getFeedRuntimeType());
+ }
+ registeredCollectors.remove(0);
+ break;
+ }
+ break;
+ case SHARED:
+ handleDataBucket(frame);
+ break;
+ }
+ } catch (Exception e) {
+ throw new HyracksDataException(e);
}
}
- private void nextBucket(DataBucket bucket) {
+ private void nextBucket(DataBucket bucket) throws InterruptedException {
for (FeedFrameCollector collector : registeredCollectors.values()) {
collector.sendMessage(bucket); // asynchronous call
}
}
- private void handleDataBucket(ByteBuffer frame) throws HyracksDataException {
+ private void handleDataBucket(ByteBuffer frame) throws HyracksDataException, InterruptedException {
DataBucket bucket = getDataBucket();
if (bucket == null) {
handleFrameDuringMemoryCongestion(frame);
@@ -281,32 +286,36 @@ public class FrameDistributor {
return distributionMode;
}
- public void close() {
- switch (distributionMode) {
- case INACTIVE:
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("FrameDistributor is " + distributionMode);
- }
- break;
- case SINGLE:
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Disconnecting single frame reader in " + distributionMode + " mode " + " for feedId "
- + feedId + " " + this.feedRuntimeType);
- }
- setMode(DistributionMode.INACTIVE);
- if (!enableSynchronousTransfer) {
+ public void close() throws HyracksDataException {
+ try {
+ switch (distributionMode) {
+ case INACTIVE:
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("FrameDistributor is " + distributionMode);
+ }
+ break;
+ case SINGLE:
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Disconnecting single frame reader in " + distributionMode + " mode "
+ + " for feedId " + feedId + " " + this.feedRuntimeType);
+ }
+ setMode(DistributionMode.INACTIVE);
+ if (!enableSynchronousTransfer) {
+ notifyEndOfFeed(); // send EOD Data Bucket
+ waitForCollectorsToFinish();
+ }
+ registeredCollectors.values().iterator().next().disconnect();
+ break;
+ case SHARED:
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Signalling End Of Feed; currently operating in " + distributionMode + " mode");
+ }
notifyEndOfFeed(); // send EOD Data Bucket
waitForCollectorsToFinish();
- }
- registeredCollectors.values().iterator().next().disconnect();
- break;
- case SHARED:
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Signalling End Of Feed; currently operating in " + distributionMode + " mode");
- }
- notifyEndOfFeed(); // send EOD Data Bucket
- waitForCollectorsToFinish();
- break;
+ break;
+ }
+ } catch (Exception e) {
+ throw new HyracksDataException(e);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/feed/management/FeedConnectionRequest.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/feed/management/FeedConnectionRequest.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/management/FeedConnectionRequest.java
index 0175dcb..b1d3300 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/feed/management/FeedConnectionRequest.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/management/FeedConnectionRequest.java
@@ -21,7 +21,7 @@ package org.apache.asterix.external.feed.management;
import java.util.List;
import java.util.Map;
-import org.apache.asterix.external.feed.api.IFeedLifecycleListener.ConnectionLocation;
+import org.apache.asterix.external.feed.api.IFeedRuntime.FeedRuntimeType;
import org.apache.commons.lang3.StringUtils;
/**
@@ -47,7 +47,7 @@ public class FeedConnectionRequest {
private final FeedJointKey feedJointKey;
/** Location in the source feed pipeline from where feed tuples are received. **/
- private final ConnectionLocation connectionLocation;
+ private final FeedRuntimeType connectionLocation;
/** List of functions that need to be applied in sequence after the data hand-off at the source feedPointKey. **/
private final List<String> functionsToApply;
@@ -66,8 +66,7 @@ public class FeedConnectionRequest {
private final FeedId receivingFeedId;
-
- public FeedConnectionRequest(FeedJointKey feedPointKey, ConnectionLocation connectionLocation,
+ public FeedConnectionRequest(FeedJointKey feedPointKey, FeedRuntimeType connectionLocation,
List<String> functionsToApply, String targetDataset, String policy, Map<String, String> policyParameters,
FeedId receivingFeedId) {
this.feedJointKey = feedPointKey;
@@ -100,7 +99,7 @@ public class FeedConnectionRequest {
return targetDataset;
}
- public ConnectionLocation getSubscriptionLocation() {
+ public FeedRuntimeType getSubscriptionLocation() {
return connectionLocation;
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/feed/message/MessageReceiver.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/feed/message/MessageReceiver.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/message/MessageReceiver.java
index f69c552..6ed176a 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/feed/message/MessageReceiver.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/message/MessageReceiver.java
@@ -18,9 +18,9 @@
*/
package org.apache.asterix.external.feed.message;
+import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
-import java.util.concurrent.LinkedBlockingQueue;
import java.util.logging.Level;
import java.util.logging.Logger;
@@ -31,11 +31,11 @@ public abstract class MessageReceiver<T> implements IMessageReceiver<T> {
protected static final Logger LOGGER = Logger.getLogger(MessageReceiver.class.getName());
- protected final LinkedBlockingQueue<T> inbox;
+ protected final ArrayBlockingQueue<T> inbox;
protected ExecutorService executor;
public MessageReceiver() {
- inbox = new LinkedBlockingQueue<T>();
+ inbox = new ArrayBlockingQueue<T>(2);
}
public abstract void processMessage(T message) throws Exception;
@@ -47,8 +47,8 @@ public abstract class MessageReceiver<T> implements IMessageReceiver<T> {
}
@Override
- public synchronized void sendMessage(T message) {
- inbox.add(message);
+ public synchronized void sendMessage(T message) throws InterruptedException {
+ inbox.put(message);
}
@Override
@@ -68,7 +68,7 @@ public abstract class MessageReceiver<T> implements IMessageReceiver<T> {
private static class MessageReceiverRunnable<T> implements Runnable {
- private final LinkedBlockingQueue<T> inbox;
+ private final ArrayBlockingQueue<T> inbox;
private final MessageReceiver<T> messageReceiver;
public MessageReceiverRunnable(MessageReceiver<T> messageReceiver) {
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/feed/runtime/IngestionRuntime.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/feed/runtime/IngestionRuntime.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/runtime/IngestionRuntime.java
index 34cb575..021ad36 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/feed/runtime/IngestionRuntime.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/runtime/IngestionRuntime.java
@@ -76,7 +76,7 @@ public class IngestionRuntime extends SubscribableRuntime {
subscribers.remove(collectionRuntime);
}
- public void endOfFeed() {
+ public void endOfFeed() throws InterruptedException {
dWriter.notifyEndOfFeed();
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info("Notified End Of Feed [" + this + "]");
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/indexing/FileOffsetIndexer.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/indexing/FileOffsetIndexer.java b/asterix-external-data/src/main/java/org/apache/asterix/external/indexing/FileOffsetIndexer.java
index 873b420..d7a20e1 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/indexing/FileOffsetIndexer.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/indexing/FileOffsetIndexer.java
@@ -21,8 +21,7 @@ package org.apache.asterix.external.indexing;
import java.io.IOException;
import org.apache.asterix.external.api.IExternalIndexer;
-import org.apache.asterix.external.api.IRecordReader;
-import org.apache.asterix.external.input.record.reader.hdfs.HDFSRecordReader;
+import org.apache.asterix.external.api.IIndexingDatasource;
import org.apache.asterix.formats.nontagged.AqlSerializerDeserializerProvider;
import org.apache.asterix.om.base.AMutableInt32;
import org.apache.asterix.om.base.AMutableInt64;
@@ -40,7 +39,7 @@ public class FileOffsetIndexer implements IExternalIndexer {
public static final int NUM_OF_FIELDS = 2;
protected final AMutableInt32 fileNumber = new AMutableInt32(0);
protected final AMutableInt64 offset = new AMutableInt64(0);
- protected RecordReader<?, Writable> recordReader;
+ protected RecordReader<?, ? extends Writable> recordReader;
@SuppressWarnings("unchecked")
private ISerializerDeserializer<IAObject> intSerde = AqlSerializerDeserializerProvider.INSTANCE
@@ -50,13 +49,11 @@ public class FileOffsetIndexer implements IExternalIndexer {
.getSerializerDeserializer(BuiltinType.AINT64);
@Override
- public void reset(IRecordReader<?> reader) throws HyracksDataException {
+ public void reset(IIndexingDatasource dataSource) throws HyracksDataException {
try {
- //TODO: Make it more generic since we can't assume it is always going to be HDFS records.
- @SuppressWarnings("unchecked")
- HDFSRecordReader<?, Writable> hdfsReader = (HDFSRecordReader<?, Writable>) reader;
- fileNumber.setValue(hdfsReader.getSnapshot().get(hdfsReader.getCurrentSplitIndex()).getFileNumber());
- recordReader = hdfsReader.getReader();
+ //TODO: Make it more generic since we can't assume it is always going to be HDFS records
+ fileNumber.setValue(dataSource.getSnapshot().get(dataSource.getCurrentSplitIndex()).getFileNumber());
+ recordReader = dataSource.getReader();
offset.setValue(recordReader.getPos());
} catch (IOException e) {
throw new HyracksDataException(e);
@@ -79,5 +76,4 @@ public class FileOffsetIndexer implements IExternalIndexer {
public int getNumberOfFields() {
return NUM_OF_FIELDS;
}
-
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/indexing/RecordColumnarIndexer.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/indexing/RecordColumnarIndexer.java b/asterix-external-data/src/main/java/org/apache/asterix/external/indexing/RecordColumnarIndexer.java
index a2641c8..c626be5 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/indexing/RecordColumnarIndexer.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/indexing/RecordColumnarIndexer.java
@@ -21,7 +21,7 @@ package org.apache.asterix.external.indexing;
import java.io.IOException;
import org.apache.asterix.external.api.IExternalIndexer;
-import org.apache.asterix.external.api.IRecordReader;
+import org.apache.asterix.external.api.IIndexingDatasource;
import org.apache.asterix.external.input.record.reader.hdfs.HDFSRecordReader;
import org.apache.asterix.formats.nontagged.AqlSerializerDeserializerProvider;
import org.apache.asterix.om.base.AMutableInt32;
@@ -52,7 +52,7 @@ public class RecordColumnarIndexer implements IExternalIndexer {
.getSerializerDeserializer(BuiltinType.AINT64);
@Override
- public void reset(IRecordReader<?> reader) throws HyracksDataException {
+ public void reset(IIndexingDatasource reader) throws HyracksDataException {
try {
//TODO: Make this more generic. right now, it works because we only index hdfs files.
@SuppressWarnings("unchecked")
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/input/HDFSDataSourceFactory.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/HDFSDataSourceFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/HDFSDataSourceFactory.java
index f44d7bc..de185e0 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/HDFSDataSourceFactory.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/HDFSDataSourceFactory.java
@@ -24,15 +24,15 @@ import java.util.List;
import java.util.Map;
import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.external.api.AsterixInputStream;
import org.apache.asterix.external.api.IIndexibleExternalDataSource;
-import org.apache.asterix.external.api.IInputStreamProvider;
-import org.apache.asterix.external.api.IInputStreamProviderFactory;
+import org.apache.asterix.external.api.IInputStreamFactory;
import org.apache.asterix.external.api.IRecordReader;
import org.apache.asterix.external.api.IRecordReaderFactory;
import org.apache.asterix.external.indexing.ExternalFile;
import org.apache.asterix.external.indexing.IndexingScheduler;
import org.apache.asterix.external.input.record.reader.hdfs.HDFSRecordReader;
-import org.apache.asterix.external.input.stream.provider.HDFSInputStreamProvider;
+import org.apache.asterix.external.input.stream.HDFSInputStream;
import org.apache.asterix.external.provider.ExternalIndexerProvider;
import org.apache.asterix.external.util.ExternalDataUtils;
import org.apache.asterix.external.util.HDFSUtils;
@@ -49,7 +49,7 @@ import org.apache.hyracks.hdfs.dataflow.InputSplitsFactory;
import org.apache.hyracks.hdfs.scheduler.Scheduler;
public class HDFSDataSourceFactory
- implements IInputStreamProviderFactory, IRecordReaderFactory<Object>, IIndexibleExternalDataSource {
+ implements IInputStreamFactory, IRecordReaderFactory<Object>, IIndexibleExternalDataSource {
protected static final long serialVersionUID = 1L;
protected transient AlgebricksAbsolutePartitionConstraint clusterLocations;
@@ -116,10 +116,9 @@ public class HDFSDataSourceFactory
* The method below was modified to take care of the following
* 1. when target files are not null, it generates a file aware input stream that validate
* against the files
- * 2. if the data is binary, it returns a generic reade */
+ * 2. if the data is binary, it returns a generic reader */
@Override
- public IInputStreamProvider createInputStreamProvider(IHyracksTaskContext ctx, int partition)
- throws HyracksDataException {
+ public AsterixInputStream createInputStream(IHyracksTaskContext ctx, int partition) throws HyracksDataException {
try {
if (!configured) {
conf = confFactory.getConf();
@@ -127,8 +126,7 @@ public class HDFSDataSourceFactory
nodeName = ctx.getJobletContext().getApplicationContext().getNodeId();
configured = true;
}
- return new HDFSInputStreamProvider<Object>(read, inputSplits, readSchedule, nodeName, conf, configuration,
- files);
+ return new HDFSInputStream(read, inputSplits, readSchedule, nodeName, conf, configuration, files);
} catch (Exception e) {
throw new HyracksDataException(e);
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/CharArrayRecord.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/CharArrayRecord.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/CharArrayRecord.java
index affdc84..6b8e10d 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/CharArrayRecord.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/CharArrayRecord.java
@@ -18,6 +18,7 @@
*/
package org.apache.asterix.external.input.record;
+import java.io.IOException;
import java.nio.CharBuffer;
import java.util.Arrays;
@@ -29,6 +30,16 @@ public class CharArrayRecord implements IRawRecord<char[]> {
private char[] value;
private int size;
+ public CharArrayRecord(int initialCapacity) {
+ value = new char[initialCapacity];
+ size = 0;
+ }
+
+ public CharArrayRecord() {
+ value = new char[ExternalDataConstants.DEFAULT_BUFFER_SIZE];
+ size = 0;
+ }
+
@Override
public byte[] getBytes() {
return new String(value).getBytes();
@@ -44,16 +55,6 @@ public class CharArrayRecord implements IRawRecord<char[]> {
return size;
}
- public CharArrayRecord(int initialCapacity) {
- value = new char[initialCapacity];
- size = 0;
- }
-
- public CharArrayRecord() {
- value = new char[ExternalDataConstants.DEFAULT_BUFFER_SIZE];
- size = 0;
- }
-
public void setValue(char[] recordBuffer, int offset, int length) {
if (value.length < length) {
value = new char[length];
@@ -62,13 +63,21 @@ public class CharArrayRecord implements IRawRecord<char[]> {
size = length;
}
- private void ensureCapacity(int len) {
+ private void ensureCapacity(int len) throws IOException {
if (value.length < len) {
- value = Arrays.copyOf(value, (int) (len * 1.25));
+ if (len > ExternalDataConstants.MAX_RECORD_SIZE) {
+ throw new IOException(
+ "Record is too large!. Maximum record size is " + ExternalDataConstants.MAX_RECORD_SIZE);
+ }
+ int newSize = len + ExternalDataConstants.DEFAULT_BUFFER_INCREMENT;
+ if (newSize > ExternalDataConstants.MAX_RECORD_SIZE) {
+ newSize = ExternalDataConstants.MAX_RECORD_SIZE;
+ }
+ value = Arrays.copyOf(value, newSize);
}
}
- public void append(char[] recordBuffer, int offset, int length) {
+ public void append(char[] recordBuffer, int offset, int length) throws IOException {
ensureCapacity(size + length);
System.arraycopy(recordBuffer, offset, value, size, length);
size += length;
@@ -84,25 +93,25 @@ public class CharArrayRecord implements IRawRecord<char[]> {
return String.valueOf(value, 0, size);
}
- public void endRecord() {
+ public void endRecord() throws IOException {
if (value[size - 1] != ExternalDataConstants.LF) {
appendChar(ExternalDataConstants.LF);
}
}
- private void appendChar(char c) {
+ private void appendChar(char c) throws IOException {
ensureCapacity(size + 1);
value[size] = c;
size++;
}
- public void append(char[] recordBuffer) {
+ public void append(char[] recordBuffer) throws IOException {
ensureCapacity(size + recordBuffer.length);
System.arraycopy(recordBuffer, 0, value, size, recordBuffer.length);
size += recordBuffer.length;
}
- public void append(CharBuffer chars) {
+ public void append(CharBuffer chars) throws IOException {
ensureCapacity(size + chars.limit());
chars.get(value, size, chars.limit());
size += chars.limit();
@@ -114,7 +123,7 @@ public class CharArrayRecord implements IRawRecord<char[]> {
this.size = value.length;
}
- public void set(StringBuilder builder) {
+ public void set(StringBuilder builder) throws IOException {
ensureCapacity(builder.length());
builder.getChars(0, builder.length(), value, 0);
this.size = builder.length();
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/converter/DCPRequestToRecordWithMetadataAndPKConverter.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/converter/DCPRequestToRecordWithMetadataAndPKConverter.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/converter/DCPRequestToRecordWithMetadataAndPKConverter.java
index 1f82e85..379bbf2 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/converter/DCPRequestToRecordWithMetadataAndPKConverter.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/converter/DCPRequestToRecordWithMetadataAndPKConverter.java
@@ -100,7 +100,7 @@ public class DCPRequestToRecordWithMetadataAndPKConverter
}
public static void set(final ByteBuf content, final CharsetDecoder decoder, final ByteBuffer bytes,
- final CharBuffer chars, final CharArrayRecord record) {
+ final CharBuffer chars, final CharArrayRecord record) throws IOException {
int position = content.readerIndex();
final int limit = content.writerIndex();
final int contentSize = content.readableBytes();
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/TestAsterixMembersReader.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/TestAsterixMembersReader.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/TestAsterixMembersReader.java
index 916904f..665d50b 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/TestAsterixMembersReader.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/TestAsterixMembersReader.java
@@ -20,9 +20,9 @@ package org.apache.asterix.external.input.record.reader;
import java.io.IOException;
-import org.apache.asterix.external.api.IDataFlowController;
import org.apache.asterix.external.api.IRawRecord;
import org.apache.asterix.external.api.IRecordReader;
+import org.apache.asterix.external.dataflow.AbstractFeedDataFlowController;
import org.apache.asterix.external.input.record.CharArrayRecord;
import org.apache.asterix.external.input.record.GenericRecord;
import org.apache.asterix.external.input.record.RecordWithPK;
@@ -79,10 +79,15 @@ public class TestAsterixMembersReader implements IRecordReader<RecordWithPK<char
}
@Override
- public void setController(final IDataFlowController controller) {
+ public void setController(final AbstractFeedDataFlowController controller) {
}
@Override
public void setFeedLogManager(final FeedLogManager feedLogManager) {
}
+
+ @Override
+ public boolean handleException(Throwable th) {
+ return false;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/AbstractCharRecordLookupReader.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/AbstractCharRecordLookupReader.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/AbstractCharRecordLookupReader.java
index 0627660..45d01e9 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/AbstractCharRecordLookupReader.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/AbstractCharRecordLookupReader.java
@@ -60,7 +60,7 @@ public abstract class AbstractCharRecordLookupReader extends AbstractHDFSLookupR
protected abstract void readRecord(RecordId rid) throws IOException;
- private void writeRecord() {
+ private void writeRecord() throws IOException {
reusableByteBuffer.clear();
if (reusableByteBuffer.remaining() < value.getLength()) {
reusableByteBuffer = ByteBuffer
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/HDFSRecordReader.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/HDFSRecordReader.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/HDFSRecordReader.java
index 5ed6dc5..3500d32 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/HDFSRecordReader.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/HDFSRecordReader.java
@@ -21,11 +21,11 @@ package org.apache.asterix.external.input.record.reader.hdfs;
import java.io.IOException;
import java.util.List;
-import org.apache.asterix.external.api.IDataFlowController;
import org.apache.asterix.external.api.IExternalIndexer;
import org.apache.asterix.external.api.IIndexingDatasource;
import org.apache.asterix.external.api.IRawRecord;
import org.apache.asterix.external.api.IRecordReader;
+import org.apache.asterix.external.dataflow.AbstractFeedDataFlowController;
import org.apache.asterix.external.indexing.ExternalFile;
import org.apache.asterix.external.input.record.GenericRecord;
import org.apache.asterix.external.util.FeedLogManager;
@@ -121,8 +121,9 @@ public class HDFSRecordReader<K, V extends Writable> implements IRecordReader<Wr
FileStatus fileStatus = hdfs.getFileStatus(new Path(fileName));
// Skip if not the same file stored in the files snapshot
if (fileStatus.getModificationTime() != snapshot.get(currentSplitIndex).getLastModefiedTime()
- .getTime())
+ .getTime()) {
continue;
+ }
}
reader.close();
@@ -160,10 +161,12 @@ public class HDFSRecordReader<K, V extends Writable> implements IRecordReader<Wr
return indexer;
}
+ @Override
public List<ExternalFile> getSnapshot() {
return snapshot;
}
+ @Override
public int getCurrentSplitIndex() {
return currentSplitIndex;
}
@@ -177,6 +180,11 @@ public class HDFSRecordReader<K, V extends Writable> implements IRecordReader<Wr
}
@Override
- public void setController(IDataFlowController controller) {
+ public void setController(AbstractFeedDataFlowController controller) {
+ }
+
+ @Override
+ public boolean handleException(Throwable th) {
+ return false;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/kv/KVReader.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/kv/KVReader.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/kv/KVReader.java
index 4e41357..185aea0 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/kv/KVReader.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/kv/KVReader.java
@@ -21,7 +21,6 @@ package org.apache.asterix.external.input.record.reader.kv;
import java.io.IOException;
import java.util.concurrent.ArrayBlockingQueue;
-import org.apache.asterix.external.api.IDataFlowController;
import org.apache.asterix.external.api.IRawRecord;
import org.apache.asterix.external.api.IRecordReader;
import org.apache.asterix.external.dataflow.AbstractFeedDataFlowController;
@@ -92,10 +91,10 @@ public class KVReader implements IRecordReader<DCPRequest> {
}
private void connect() {
- core.send(new SeedNodesRequest(sourceNodes))
- .timeout(KVReaderFactory.TIMEOUT, KVReaderFactory.TIME_UNIT).toBlocking().single();
- core.send(new OpenBucketRequest(bucket, password))
- .timeout(KVReaderFactory.TIMEOUT, KVReaderFactory.TIME_UNIT).toBlocking().single();
+ core.send(new SeedNodesRequest(sourceNodes)).timeout(KVReaderFactory.TIMEOUT, KVReaderFactory.TIME_UNIT)
+ .toBlocking().single();
+ core.send(new OpenBucketRequest(bucket, password)).timeout(KVReaderFactory.TIMEOUT, KVReaderFactory.TIME_UNIT)
+ .toBlocking().single();
this.pushThread = new Thread(new Runnable() {
@Override
public void run() {
@@ -183,11 +182,16 @@ public class KVReader implements IRecordReader<DCPRequest> {
}
@Override
- public void setController(IDataFlowController controller) {
- this.controller = (AbstractFeedDataFlowController) controller;
+ public void setController(AbstractFeedDataFlowController controller) {
+ this.controller = controller;
}
@Override
public void setFeedLogManager(FeedLogManager feedLogManager) {
}
+
+ @Override
+ public boolean handleException(Throwable th) {
+ return false;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/kv/KVTestReader.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/kv/KVTestReader.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/kv/KVTestReader.java
index f540f4e..b75f26c 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/kv/KVTestReader.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/kv/KVTestReader.java
@@ -22,9 +22,9 @@ import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
-import org.apache.asterix.external.api.IDataFlowController;
import org.apache.asterix.external.api.IRawRecord;
import org.apache.asterix.external.api.IRecordReader;
+import org.apache.asterix.external.dataflow.AbstractFeedDataFlowController;
import org.apache.asterix.external.input.record.GenericRecord;
import org.apache.asterix.external.util.ExternalDataConstants;
import org.apache.asterix.external.util.FeedLogManager;
@@ -169,11 +169,16 @@ public class KVTestReader implements IRecordReader<DCPRequest> {
}
@Override
- public void setController(final IDataFlowController controller) {
+ public void setController(final AbstractFeedDataFlowController controller) {
}
@Override
public void setFeedLogManager(final FeedLogManager feedLogManager) {
}
+ @Override
+ public boolean handleException(Throwable th) {
+ return false;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/rss/RSSRecordReader.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/rss/RSSRecordReader.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/rss/RSSRecordReader.java
index a78f780..5760a7a 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/rss/RSSRecordReader.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/rss/RSSRecordReader.java
@@ -25,9 +25,9 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Queue;
-import org.apache.asterix.external.api.IDataFlowController;
import org.apache.asterix.external.api.IRawRecord;
import org.apache.asterix.external.api.IRecordReader;
+import org.apache.asterix.external.dataflow.AbstractFeedDataFlowController;
import org.apache.asterix.external.input.record.GenericRecord;
import org.apache.asterix.external.util.FeedLogManager;
import org.apache.log4j.Logger;
@@ -138,7 +138,12 @@ public class RSSRecordReader implements IRecordReader<SyndEntryImpl> {
}
@Override
- public void setController(IDataFlowController controller) {
+ public void setController(AbstractFeedDataFlowController controller) {
+ }
+
+ @Override
+ public boolean handleException(Throwable th) {
+ return false;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/AbstractStreamRecordReader.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/AbstractStreamRecordReader.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/AbstractStreamRecordReader.java
index 6a5d776..7d6c1f3 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/AbstractStreamRecordReader.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/AbstractStreamRecordReader.java
@@ -19,21 +19,24 @@
package org.apache.asterix.external.input.record.reader.stream;
import java.io.IOException;
+import java.util.List;
-import org.apache.asterix.external.api.IDataFlowController;
+import org.apache.asterix.external.api.AsterixInputStream;
import org.apache.asterix.external.api.IExternalIndexer;
import org.apache.asterix.external.api.IIndexingDatasource;
import org.apache.asterix.external.api.IRawRecord;
import org.apache.asterix.external.api.IRecordReader;
import org.apache.asterix.external.dataflow.AbstractFeedDataFlowController;
+import org.apache.asterix.external.indexing.ExternalFile;
import org.apache.asterix.external.input.record.CharArrayRecord;
-import org.apache.asterix.external.input.stream.AInputStream;
-import org.apache.asterix.external.input.stream.AInputStreamReader;
+import org.apache.asterix.external.input.stream.AsterixInputStreamReader;
import org.apache.asterix.external.util.ExternalDataConstants;
import org.apache.asterix.external.util.FeedLogManager;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.mapred.RecordReader;
public abstract class AbstractStreamRecordReader implements IRecordReader<char[]>, IIndexingDatasource {
- protected final AInputStreamReader reader;
+ protected final AsterixInputStreamReader reader;
protected CharArrayRecord record;
protected char[] inputBuffer;
protected int bufferLength = 0;
@@ -42,8 +45,8 @@ public abstract class AbstractStreamRecordReader implements IRecordReader<char[]
protected boolean done = false;
protected FeedLogManager feedLogManager;
- public AbstractStreamRecordReader(AInputStream inputStream, IExternalIndexer indexer) {
- this.reader = new AInputStreamReader(inputStream);
+ public AbstractStreamRecordReader(AsterixInputStream inputStream, IExternalIndexer indexer) {
+ this.reader = new AsterixInputStreamReader(inputStream);
this.indexer = indexer;
record = new CharArrayRecord();
inputBuffer = new char[ExternalDataConstants.DEFAULT_BUFFER_SIZE];
@@ -79,8 +82,8 @@ public abstract class AbstractStreamRecordReader implements IRecordReader<char[]
}
@Override
- public void setController(IDataFlowController controller) {
- reader.setController((AbstractFeedDataFlowController) controller);
+ public void setController(AbstractFeedDataFlowController controller) {
+ reader.setController(controller);
}
@Override
@@ -88,4 +91,26 @@ public abstract class AbstractStreamRecordReader implements IRecordReader<char[]
this.feedLogManager = feedLogManager;
reader.setFeedLogManager(feedLogManager);
}
+
+ @Override
+ public boolean handleException(Throwable th) {
+ return reader.handleException(th);
+ }
+
+ //TODO: Fix the following method since they don't fit
+ //Already the fix is in another local branch
+ @Override
+ public List<ExternalFile> getSnapshot() {
+ return null;
+ }
+
+ @Override
+ public int getCurrentSplitIndex() {
+ return -1;
+ }
+
+ @Override
+ public RecordReader<?, Writable> getReader() {
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/AbstractStreamRecordReaderFactory.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/AbstractStreamRecordReaderFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/AbstractStreamRecordReaderFactory.java
index 12c0229..2c82f47 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/AbstractStreamRecordReaderFactory.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/AbstractStreamRecordReaderFactory.java
@@ -22,14 +22,13 @@ import java.util.List;
import java.util.Map;
import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.external.api.AsterixInputStream;
import org.apache.asterix.external.api.IExternalIndexer;
import org.apache.asterix.external.api.IIndexibleExternalDataSource;
import org.apache.asterix.external.api.IIndexingDatasource;
-import org.apache.asterix.external.api.IInputStreamProvider;
-import org.apache.asterix.external.api.IInputStreamProviderFactory;
+import org.apache.asterix.external.api.IInputStreamFactory;
import org.apache.asterix.external.api.IRecordReaderFactory;
import org.apache.asterix.external.indexing.ExternalFile;
-import org.apache.asterix.external.input.stream.AInputStream;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
import org.apache.hyracks.algebricks.common.utils.Pair;
import org.apache.hyracks.api.context.IHyracksTaskContext;
@@ -39,11 +38,11 @@ public abstract class AbstractStreamRecordReaderFactory<T>
implements IRecordReaderFactory<T>, IIndexibleExternalDataSource {
private static final long serialVersionUID = 1L;
- protected IInputStreamProviderFactory inputStreamFactory;
+ protected IInputStreamFactory inputStreamFactory;
protected Map<String, String> configuration;
public AbstractStreamRecordReaderFactory<T> setInputStreamFactoryProvider(
- IInputStreamProviderFactory inputStreamFactory) {
+ IInputStreamFactory inputStreamFactory) {
this.inputStreamFactory = inputStreamFactory;
return this;
}
@@ -82,15 +81,15 @@ public abstract class AbstractStreamRecordReaderFactory<T>
return false;
}
- protected Pair<AInputStream, IExternalIndexer> getStreamAndIndexer(IHyracksTaskContext ctx, int partition)
+ protected Pair<AsterixInputStream, IExternalIndexer> getStreamAndIndexer(IHyracksTaskContext ctx, int partition)
throws HyracksDataException {
- IInputStreamProvider inputStreamProvider = inputStreamFactory.createInputStreamProvider(ctx, partition);
+ AsterixInputStream inputStream = inputStreamFactory.createInputStream(ctx, partition);
IExternalIndexer indexer = null;
if (inputStreamFactory.isIndexible()) {
if (((IIndexibleExternalDataSource) inputStreamFactory).isIndexingOp()) {
- indexer = ((IIndexingDatasource) inputStreamProvider).getIndexer();
+ indexer = ((IIndexingDatasource) inputStream).getIndexer();
}
}
- return new Pair<AInputStream, IExternalIndexer>(inputStreamProvider.getInputStream(), indexer);
+ return new Pair<AsterixInputStream, IExternalIndexer>(inputStream, indexer);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/EmptyLineSeparatedRecordReader.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/EmptyLineSeparatedRecordReader.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/EmptyLineSeparatedRecordReader.java
index fb56062..6964a82 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/EmptyLineSeparatedRecordReader.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/EmptyLineSeparatedRecordReader.java
@@ -20,13 +20,13 @@ package org.apache.asterix.external.input.record.reader.stream;
import java.io.IOException;
+import org.apache.asterix.external.api.AsterixInputStream;
import org.apache.asterix.external.api.IExternalIndexer;
-import org.apache.asterix.external.input.stream.AInputStream;
import org.apache.asterix.external.util.ExternalDataConstants;
public class EmptyLineSeparatedRecordReader extends AbstractStreamRecordReader {
- public EmptyLineSeparatedRecordReader(AInputStream inputStream, IExternalIndexer indexer) {
+ public EmptyLineSeparatedRecordReader(AsterixInputStream inputStream, IExternalIndexer indexer) {
super(inputStream, indexer);
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/EmptyLineSeparatedRecordReaderFactory.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/EmptyLineSeparatedRecordReaderFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/EmptyLineSeparatedRecordReaderFactory.java
index 75d16c5..063ed11 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/EmptyLineSeparatedRecordReaderFactory.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/EmptyLineSeparatedRecordReaderFactory.java
@@ -18,9 +18,9 @@
*/
package org.apache.asterix.external.input.record.reader.stream;
+import org.apache.asterix.external.api.AsterixInputStream;
import org.apache.asterix.external.api.IExternalIndexer;
import org.apache.asterix.external.api.IRecordReader;
-import org.apache.asterix.external.input.stream.AInputStream;
import org.apache.hyracks.algebricks.common.utils.Pair;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
@@ -32,7 +32,7 @@ public class EmptyLineSeparatedRecordReaderFactory extends AbstractStreamRecordR
@Override
public IRecordReader<char[]> createRecordReader(IHyracksTaskContext ctx, int partition)
throws HyracksDataException {
- final Pair<AInputStream, IExternalIndexer> streamAndIndexer = getStreamAndIndexer(ctx, partition);
+ final Pair<AsterixInputStream, IExternalIndexer> streamAndIndexer = getStreamAndIndexer(ctx, partition);
return new EmptyLineSeparatedRecordReader(streamAndIndexer.first, streamAndIndexer.second);
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/LineRecordReader.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/LineRecordReader.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/LineRecordReader.java
index d55ac87..3089295 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/LineRecordReader.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/LineRecordReader.java
@@ -20,8 +20,8 @@ package org.apache.asterix.external.input.record.reader.stream;
import java.io.IOException;
+import org.apache.asterix.external.api.AsterixInputStream;
import org.apache.asterix.external.api.IExternalIndexer;
-import org.apache.asterix.external.input.stream.AInputStream;
import org.apache.asterix.external.util.ExternalDataConstants;
import org.apache.hyracks.api.exceptions.HyracksDataException;
@@ -31,7 +31,7 @@ public class LineRecordReader extends AbstractStreamRecordReader {
protected int newlineLength;
protected int recordNumber = 0;
- public LineRecordReader(final boolean hasHeader, final AInputStream stream, final IExternalIndexer indexer)
+ public LineRecordReader(final boolean hasHeader, final AsterixInputStream stream, final IExternalIndexer indexer)
throws HyracksDataException {
super(stream, indexer);
try {
[02/13] incubator-asterixdb git commit: Improve Error Handling in
Local Directory Feeds
Posted by am...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-installer/src/test/resources/integrationts/library/results/library-parsers/record-parser/record-parser.1.adm
----------------------------------------------------------------------
diff --git a/asterix-installer/src/test/resources/integrationts/library/results/library-parsers/record-parser/record-parser.1.adm b/asterix-installer/src/test/resources/integrationts/library/results/library-parsers/record-parser/record-parser.1.adm
index 858285e..cabaa67 100644
--- a/asterix-installer/src/test/resources/integrationts/library/results/library-parsers/record-parser/record-parser.1.adm
+++ b/asterix-installer/src/test/resources/integrationts/library/results/library-parsers/record-parser/record-parser.1.adm
@@ -1,100 +1,100 @@
-{ "GlobalJobId": "submit-3.chtc.wisc.edu#48968872.0#1445354636", "StatsLifetimeStarter": 572059, "JobStartDate": 1445362267, "SubmitEventNotes": "DAG Node: fabp4-0002+fabp4-0002", "JobStatus": 4, "LeaveJobInQueue": false, "WantGlidein": true, "StartdPrincipal": "execute-side@matchsession/128.104.119.175", "WantRHEL6": true, "OnExitRemove": true, "JobCurrentStartDate": 1445561276, "CoreSize": 0, "MATCH_EXP_JOBGLIDEIN_ResourceName": "wisc.edu", "Rank": 0.0d, "ExitStatus": 0, "ResidentSetSize": 125000, "ScheddBday": 1445383086, "RemoteWallClockTime": 769511.0d, "WantCheckpoint": false, "In": "/dev/null", "LastVacateTime": 1445546251, "MaxHosts": 1, "RootDir": "/", "NumRestarts": 0, "RecentBlockWriteKbytes": 0, "DiskUsage_RAW": 32543, "EnteredCurrentStatus": 1446133322, "ResidentSetSize_RAW": 100432, "RequestDisk": 1000000, "MyType": "Job", "PeriodicRemove": false, "Cmd": "/home/ssericksen/dude-14-xdock/ChtcRun/chtcjobwrapper", "CondorVersion": "$CondorVersion: 8.5.0 Sep 16 2015 BuildID
: 341710 $", "ShouldTransferFiles": "YES", "TargetType": "Machine", "MinHosts": 1, "NumCkpts_RAW": 0, "RequestCpus": 1, "RemoteUserCpu": 571737.0d, "BlockWrites": 0, "NiceUser": false, "Out": "process.out", "ImageSize_RAW": 690056, "BytesSent": 3113566.0d, "CumulativeSuspensionTime": 0, "TransferIn": false, "NumCkpts": 0, "Err": "process.err", "RecentBlockWrites": 0, "JobFinishedHookDone": 1446133322, "ProcId": 0, "ImageSize": 750000, "JobUniverse": 5, "EncryptExecuteDirectory": false, "TransferInputSizeMB": 12, "RecentBlockReads": 0, "SpooledOutputFiles": "ChtcWrapperfabp4-0002.out,AuditLog.fabp4-0002,poses.mol2,CURLTIME_4057178,harvest.log,time_elapsed.log,surf_scores.txt,CURLTIME_38803,count.log,fabp4-0002.out,CURLTIME_253463", "NumJobReconnects": 1, "WantFlocking": true, "BlockWriteKbytes": 0, "WhenToTransferOutput": "ON_EXIT_OR_EVICT", "JobCurrentStartExecutingDate": 1445561278, "ExitBySignal": false, "LastMatchTime": 1445561276, "OnExitHold": false, "OrigMaxHosts": 1, "Request
Memory": 1000, "NumJobStarts": 6, "TerminationPending": true, "TotalSuspensions": 0, "BlockReads": 0, "DAGManJobId": 48940805, "MemoryUsage": 122, "ExitCode": 0, "JobNotification": 0, "BlockReadKbytes": 0, "NumJobMatches": 6, "LocalUserCpu": 0.0d, "LastJobStatus": 2, "BufferBlockSize": 32768, "CommittedTime": 572046, "ExecutableSize_RAW": 6, "LastRejMatchReason": "no match found", "LastSuspensionTime": 0, "UserLog": "/home/ssericksen/dude-14-xdock/ChtcRun/dude14-surf-out-esr1/fabp4-0002/process.log", "DAGManNodesMask": "0,1,2,4,5,7,9,10,11,12,13,16,17,24,27", "CumulativeSlotTime": 769511.0d, "LastJobLeaseRenewal": 1446133322, "MachineAttrSlotWeight0": 1, "NumSystemHolds": 0, "BytesRecvd": 8.7351688E7d, "CondorPlatform": "$CondorPlatform: X86_64-RedHat_6.6 $", "JOBGLIDEIN_ResourceName": "$$([IfThenElse(IsUndefined(TARGET.GLIDEIN_ResourceName), IfThenElse(IsUndefined(TARGET.GLIDEIN_Site), \"wisc.edu\", TARGET.GLIDEIN_Site), TARGET.GLIDEIN_ResourceName)])", "DAGNodeName": "fabp4-0002+f
abp4-0002", "PeriodicRelease": "error", "JobRunCount": 7, "LastRemotePool": "condor.biochem.wisc.edu:9618?sock=collector", "JobLastStartDate": 1445546257, "LastRemoteHost": "slot1@cluster-0008.biochem.wisc.edu", "JobPrio": 0, "LocalSysCpu": 0.0d, "ExecutableSize": 7, "RemoteSysCpu": 0.0d, "TransferInput": "/home/ssericksen/dude-14-xdock/ChtcRun/dude14-surf-in/fabp4-0002/,/home/ssericksen/dude-14-xdock/ChtcRun/dude14-surf-in/shared/", "PeriodicHold": false, "WantRemoteIO": true, "CommittedSuspensionTime": 0, "DAGParentNodeNames": "", "CompletionDate": 1446133322, "StreamErr": false, "is_resumable": true, "RecentBlockReadKbytes": 0, "WantRemoteSyscalls": false, "NumShadowStarts": 7, "MachineAttrCpus0": 1, "DAGManNodesLog": "/home/ssericksen/dude-14-xdock/ChtcRun/dude14-surf-out-esr1/./mydag.dag.nodes.log", "Owner": "ssericksen", "Requirements": "undefined", "DiskUsage": 35000, "LastRejMatchTime": 1445375317, "JobLeaseDuration": 2400, "ClusterId": 48968872, "BufferSize": 524288, "IsCHT
CSubmit": true, "RecentStatsLifetimeStarter": 1200, "CommittedSlotTime": 572046.0d, "Args": "--type=Other --cmdtorun=surflex_run_DUDE_v1.8_esr1.sh --unique=fabp4-0002 --", "Environment": "", "LastPublicClaimId": "<128.104.119.175:9618>#1444067179#3317#...", "Iwd": "/home/ssericksen/dude-14-xdock/ChtcRun/dude14-surf-out-esr1/fabp4-0002", "QDate": 1445354636, "CurrentHosts": 0, "User": "ssericksen@chtc.wisc.edu", "StreamOut": false }
-{ "GlobalJobId": "submit-3.chtc.wisc.edu#49573720.0#1445938922", "StatsLifetimeStarter": 190245, "JobStartDate": 1445943852, "SubmitEventNotes": "DAG Node: 180+180", "JobStatus": 4, "LeaveJobInQueue": false, "StartdPrincipal": "execute-side@matchsession/128.105.245.72", "WantRHEL6": true, "OnExitRemove": true, "JobCurrentStartDate": 1445943852, "CoreSize": 0, "MATCH_EXP_JOBGLIDEIN_ResourceName": "wisc.edu", "Rank": 0.0d, "ExitStatus": 0, "ResidentSetSize": 125000, "RemoteWallClockTime": 190247.0d, "WantCheckpoint": false, "In": "/dev/null", "MaxHosts": 1, "RootDir": "/", "NumRestarts": 0, "RecentBlockWriteKbytes": 0, "DiskUsage_RAW": 1216669, "EnteredCurrentStatus": 1446134099, "ResidentSetSize_RAW": 123680, "RequestDisk": 1000000, "MyType": "Job", "PeriodicRemove": false, "Cmd": "/home/xguo23/finally/Simulation_condor/chtcjobwrapper", "CondorVersion": "$CondorVersion: 8.5.0 Sep 16 2015 BuildID: 341710 $", "ShouldTransferFiles": "YES", "TargetType": "Machine", "MinHosts": 1, "NumCkp
ts_RAW": 0, "RequestCpus": 1, "RemoteUserCpu": 185236.0d, "BlockWrites": 0, "NiceUser": false, "Out": "process.out", "ImageSize_RAW": 811948, "BytesSent": 30766.0d, "CumulativeSuspensionTime": 0, "TransferIn": false, "NumCkpts": 0, "Err": "process.err", "RecentBlockWrites": 0, "JobFinishedHookDone": 1446134099, "ProcId": 0, "ImageSize": 1000000, "JobUniverse": 5, "EncryptExecuteDirectory": false, "TransferInputSizeMB": 0, "RecentBlockReads": 0, "SpooledOutputFiles": "CURLTIME_3753852,ChtcWrapper180.out,AuditLog.180,simu_3_180.txt,harvest.log,180.out", "BlockWriteKbytes": 0, "WhenToTransferOutput": "ON_EXIT", "JobCurrentStartExecutingDate": 1445943853, "ExitBySignal": false, "LastMatchTime": 1445943852, "OnExitHold": false, "OrigMaxHosts": 1, "RequestMemory": 1000, "NumJobStarts": 1, "TerminationPending": true, "TotalSuspensions": 0, "BlockReads": 0, "DAGManJobId": 49572657, "MemoryUsage": 122, "ExitCode": 0, "JobNotification": 0, "BlockReadKbytes": 0, "NumJobMatches": 1, "LocalUserC
pu": 0.0d, "LastJobStatus": 2, "BufferBlockSize": 32768, "CommittedTime": 190247, "ExecutableSize_RAW": 6, "LastSuspensionTime": 0, "Matlab": "R2011b", "UserLog": "/home/xguo23/finally/Simulation_condor/model_3/180/process.log", "DAGManNodesMask": "0,1,2,4,5,7,9,10,11,12,13,16,17,24,27", "CumulativeSlotTime": 190247.0d, "LastJobLeaseRenewal": 1446134099, "MachineAttrSlotWeight0": 1, "NumSystemHolds": 0, "BytesRecvd": 284367.0d, "CondorPlatform": "$CondorPlatform: X86_64-RedHat_6.6 $", "JOBGLIDEIN_ResourceName": "$$([IfThenElse(IsUndefined(TARGET.GLIDEIN_ResourceName), IfThenElse(IsUndefined(TARGET.GLIDEIN_Site), \"wisc.edu\", TARGET.GLIDEIN_Site), TARGET.GLIDEIN_ResourceName)])", "DAGNodeName": "180+180", "PeriodicRelease": "error", "JobRunCount": 1, "LastRemoteHost": "slot1@e272.chtc.wisc.edu", "JobPrio": 0, "LocalSysCpu": 0.0d, "ExecutableSize": 7, "RemoteSysCpu": 1835.0d, "TransferInput": "/home/xguo23/finally/Simulation_condor/data/180/,/home/xguo23/finally/Simulation_condor/dat
a/shared/", "PeriodicHold": false, "WantRemoteIO": true, "CommittedSuspensionTime": 0, "DAGParentNodeNames": "", "CompletionDate": 1446134099, "StreamErr": false, "RecentBlockReadKbytes": 0, "WantRemoteSyscalls": false, "NumShadowStarts": 1, "MachineAttrCpus0": 1, "DAGManNodesLog": "/home/xguo23/finally/Simulation_condor/model_3/./mydag.dag.nodes.log", "Owner": "xguo23", "Requirements": "undefined", "DiskUsage": 1250000, "JobLeaseDuration": 2400, "ClusterId": 49573720, "BufferSize": 524288, "IsCHTCSubmit": true, "RecentStatsLifetimeStarter": 1200, "CommittedSlotTime": 190247.0d, "Args": "--type=Matlab --version=R2011b --cmdtorun=simu_condor --unique=180 -- 3", "Environment": "", "LastPublicClaimId": "<128.105.245.72:29075>#1444753997#6000#...", "Iwd": "/home/xguo23/finally/Simulation_condor/model_3/180", "QDate": 1445938922, "CurrentHosts": 0, "User": "xguo23@chtc.wisc.edu", "StreamOut": false }
-{ "GlobalJobId": "submit-3.chtc.wisc.edu#49581952.0#1446105329", "StatsLifetimeStarter": 27674, "JobStartDate": 1446106061, "SubmitEventNotes": "DAG Node: 40+40", "JobStatus": 4, "LeaveJobInQueue": false, "StartdPrincipal": "execute-side@matchsession/128.105.245.86", "WantRHEL6": true, "OnExitRemove": true, "JobCurrentStartDate": 1446106061, "CoreSize": 0, "MATCH_EXP_JOBGLIDEIN_ResourceName": "wisc.edu", "Rank": 0.0d, "ExitStatus": 0, "ResidentSetSize": 150000, "RemoteWallClockTime": 27676.0d, "WantCheckpoint": false, "In": "/dev/null", "MaxHosts": 1, "RootDir": "/", "NumRestarts": 0, "RecentBlockWriteKbytes": 0, "DiskUsage_RAW": 1216669, "EnteredCurrentStatus": 1446133737, "ResidentSetSize_RAW": 127252, "RequestDisk": 1000000, "MyType": "Job", "PeriodicRemove": false, "Cmd": "/home/xguo23/finally_2/Simulation_condor/chtcjobwrapper", "CondorVersion": "$CondorVersion: 8.5.0 Sep 16 2015 BuildID: 341710 $", "ShouldTransferFiles": "YES", "TargetType": "Machine", "MinHosts": 1, "NumCkpts
_RAW": 0, "RequestCpus": 1, "RemoteUserCpu": 27510.0d, "BlockWrites": 0, "NiceUser": false, "Out": "process.out", "ImageSize_RAW": 811948, "BytesSent": 30584.0d, "CumulativeSuspensionTime": 0, "TransferIn": false, "NumCkpts": 0, "Err": "process.err", "RecentBlockWrites": 0, "JobFinishedHookDone": 1446133737, "ProcId": 0, "ImageSize": 1000000, "JobUniverse": 5, "EncryptExecuteDirectory": false, "TransferInputSizeMB": 0, "RecentBlockReads": 0, "SpooledOutputFiles": "CURLTIME_369560,ChtcWrapper40.out,AuditLog.40,simu_3_40.txt,harvest.log,40.out", "BlockWriteKbytes": 0, "WhenToTransferOutput": "ON_EXIT", "JobCurrentStartExecutingDate": 1446106063, "ExitBySignal": false, "LastMatchTime": 1446106061, "OnExitHold": false, "OrigMaxHosts": 1, "RequestMemory": 1000, "NumJobStarts": 1, "TerminationPending": true, "TotalSuspensions": 0, "BlockReads": 0, "DAGManJobId": 49581933, "MemoryUsage": 146, "ExitCode": 0, "JobNotification": 0, "BlockReadKbytes": 0, "NumJobMatches": 1, "LocalUserCpu": 0.0
d, "LastJobStatus": 2, "BufferBlockSize": 32768, "CommittedTime": 27676, "ExecutableSize_RAW": 6, "LastSuspensionTime": 0, "Matlab": "R2011b", "UserLog": "/home/xguo23/finally_2/Simulation_condor/model_3/40/process.log", "DAGManNodesMask": "0,1,2,4,5,7,9,10,11,12,13,16,17,24,27", "CumulativeSlotTime": 27676.0d, "LastJobLeaseRenewal": 1446133737, "MachineAttrSlotWeight0": 1, "NumSystemHolds": 0, "BytesRecvd": 285053.0d, "CondorPlatform": "$CondorPlatform: X86_64-RedHat_6.6 $", "JOBGLIDEIN_ResourceName": "$$([IfThenElse(IsUndefined(TARGET.GLIDEIN_ResourceName), IfThenElse(IsUndefined(TARGET.GLIDEIN_Site), \"wisc.edu\", TARGET.GLIDEIN_Site), TARGET.GLIDEIN_ResourceName)])", "DAGNodeName": "40+40", "PeriodicRelease": "error", "JobRunCount": 1, "LastRemoteHost": "slot1@e286.chtc.wisc.edu", "JobPrio": 0, "LocalSysCpu": 0.0d, "ExecutableSize": 7, "RemoteSysCpu": 105.0d, "TransferInput": "/home/xguo23/finally_2/Simulation_condor/data/40/,/home/xguo23/finally_2/Simulation_condor/data/shared/
", "PeriodicHold": false, "WantRemoteIO": true, "CommittedSuspensionTime": 0, "DAGParentNodeNames": "", "CompletionDate": 1446133737, "StreamErr": false, "RecentBlockReadKbytes": 0, "WantRemoteSyscalls": false, "NumShadowStarts": 1, "MachineAttrCpus0": 1, "DAGManNodesLog": "/home/xguo23/finally_2/Simulation_condor/model_3/./mydag.dag.nodes.log", "Owner": "xguo23", "Requirements": "undefined", "DiskUsage": 1250000, "JobLeaseDuration": 2400, "ClusterId": 49581952, "BufferSize": 524288, "IsCHTCSubmit": true, "RecentStatsLifetimeStarter": 1200, "CommittedSlotTime": 27676.0d, "Args": "--type=Matlab --version=R2011b --cmdtorun=simu_condor --unique=40 -- 3", "Environment": "", "LastPublicClaimId": "<128.105.245.86:32129>#1444759888#6329#...", "Iwd": "/home/xguo23/finally_2/Simulation_condor/model_3/40", "QDate": 1446105329, "CurrentHosts": 0, "User": "xguo23@chtc.wisc.edu", "StreamOut": false }
-{ "GlobalJobId": "submit-3.chtc.wisc.edu#49581985.0#1446105368", "StatsLifetimeStarter": 26354, "JobStartDate": 1446106289, "SubmitEventNotes": "DAG Node: 36+36", "JobStatus": 4, "LeaveJobInQueue": false, "StartdPrincipal": "execute-side@matchsession/128.105.244.249", "WantRHEL6": true, "OnExitRemove": true, "JobCurrentStartDate": 1446106289, "CoreSize": 0, "MATCH_EXP_JOBGLIDEIN_ResourceName": "wisc.edu", "Rank": 0.0d, "ExitStatus": 0, "ResidentSetSize": 150000, "RemoteWallClockTime": 26357.0d, "WantCheckpoint": false, "In": "/dev/null", "MaxHosts": 1, "RootDir": "/", "NumRestarts": 0, "RecentBlockWriteKbytes": 0, "DiskUsage_RAW": 1216669, "EnteredCurrentStatus": 1446132646, "ResidentSetSize_RAW": 127452, "RequestDisk": 1000000, "MyType": "Job", "PeriodicRemove": false, "Cmd": "/home/xguo23/finally_2/Simulation_condor/chtcjobwrapper", "CondorVersion": "$CondorVersion: 8.5.0 Sep 16 2015 BuildID: 341710 $", "ShouldTransferFiles": "YES", "TargetType": "Machine", "MinHosts": 1, "NumCkpt
s_RAW": 0, "RequestCpus": 1, "RemoteUserCpu": 26239.0d, "BlockWrites": 0, "NiceUser": false, "Out": "process.out", "ImageSize_RAW": 811948, "BytesSent": 31898.0d, "CumulativeSuspensionTime": 0, "TransferIn": false, "NumCkpts": 0, "Err": "process.err", "RecentBlockWrites": 0, "JobFinishedHookDone": 1446132646, "ProcId": 0, "ImageSize": 1000000, "JobUniverse": 5, "EncryptExecuteDirectory": false, "TransferInputSizeMB": 0, "RecentBlockReads": 0, "SpooledOutputFiles": "CURLTIME_1151700,ChtcWrapper36.out,AuditLog.36,simu_3_36.txt,harvest.log,36.out", "BlockWriteKbytes": 0, "WhenToTransferOutput": "ON_EXIT", "JobCurrentStartExecutingDate": 1446106291, "ExitBySignal": false, "LastMatchTime": 1446106289, "OnExitHold": false, "OrigMaxHosts": 1, "RequestMemory": 1000, "NumJobStarts": 1, "TerminationPending": true, "TotalSuspensions": 0, "BlockReads": 0, "DAGManJobId": 49581933, "MemoryUsage": 146, "ExitCode": 0, "JobNotification": 0, "BlockReadKbytes": 0, "NumJobMatches": 1, "LocalUserCpu": 0
.0d, "LastJobStatus": 2, "BufferBlockSize": 32768, "CommittedTime": 26357, "ExecutableSize_RAW": 6, "LastSuspensionTime": 0, "Matlab": "R2011b", "UserLog": "/home/xguo23/finally_2/Simulation_condor/model_3/36/process.log", "DAGManNodesMask": "0,1,2,4,5,7,9,10,11,12,13,16,17,24,27", "CumulativeSlotTime": 26357.0d, "LastJobLeaseRenewal": 1446132646, "MachineAttrSlotWeight0": 1, "NumSystemHolds": 0, "BytesRecvd": 285053.0d, "CondorPlatform": "$CondorPlatform: X86_64-RedHat_6.6 $", "JOBGLIDEIN_ResourceName": "$$([IfThenElse(IsUndefined(TARGET.GLIDEIN_ResourceName), IfThenElse(IsUndefined(TARGET.GLIDEIN_Site), \"wisc.edu\", TARGET.GLIDEIN_Site), TARGET.GLIDEIN_ResourceName)])", "DAGNodeName": "36+36", "PeriodicRelease": "error", "JobRunCount": 1, "LastRemoteHost": "slot1@e457.chtc.wisc.edu", "JobPrio": 0, "LocalSysCpu": 0.0d, "ExecutableSize": 7, "RemoteSysCpu": 96.0d, "TransferInput": "/home/xguo23/finally_2/Simulation_condor/data/36/,/home/xguo23/finally_2/Simulation_condor/data/shared
/", "PeriodicHold": false, "WantRemoteIO": true, "CommittedSuspensionTime": 0, "DAGParentNodeNames": "", "CompletionDate": 1446132646, "StreamErr": false, "RecentBlockReadKbytes": 0, "WantRemoteSyscalls": false, "NumShadowStarts": 1, "MachineAttrCpus0": 1, "DAGManNodesLog": "/home/xguo23/finally_2/Simulation_condor/model_3/./mydag.dag.nodes.log", "Owner": "xguo23", "Requirements": "undefined", "DiskUsage": 1250000, "JobLeaseDuration": 2400, "ClusterId": 49581985, "BufferSize": 524288, "IsCHTCSubmit": true, "RecentStatsLifetimeStarter": 1200, "CommittedSlotTime": 26357.0d, "Args": "--type=Matlab --version=R2011b --cmdtorun=simu_condor --unique=36 -- 3", "Environment": "", "LastPublicClaimId": "<128.105.244.249:28476>#1444685646#10655#...", "Iwd": "/home/xguo23/finally_2/Simulation_condor/model_3/36", "QDate": 1446105368, "CurrentHosts": 0, "User": "xguo23@chtc.wisc.edu", "StreamOut": false }
-{ "GlobalJobId": "submit-3.chtc.wisc.edu#49581989.0#1446105374", "StatsLifetimeStarter": 27490, "JobStartDate": 1446106290, "SubmitEventNotes": "DAG Node: 82+82", "JobStatus": 4, "LeaveJobInQueue": false, "AutoClusterId": 24, "StartdPrincipal": "execute-side@matchsession/128.105.245.233", "WantRHEL6": true, "OnExitRemove": true, "JobCurrentStartDate": 1446106290, "CoreSize": 0, "MATCH_EXP_JOBGLIDEIN_ResourceName": "wisc.edu", "ExitStatus": 0, "Rank": 0.0d, "ResidentSetSize": 150000, "RemoteWallClockTime": 27491.0d, "WantCheckpoint": false, "In": "/dev/null", "MaxHosts": 1, "RootDir": "/", "NumRestarts": 0, "RecentBlockWriteKbytes": 0, "DiskUsage_RAW": 1216669, "EnteredCurrentStatus": 1446133781, "ResidentSetSize_RAW": 126932, "RequestDisk": 1000000, "MyType": "Job", "PeriodicRemove": false, "Cmd": "/home/xguo23/finally_2/Simulation_condor/chtcjobwrapper", "CondorVersion": "$CondorVersion: 8.5.0 Sep 16 2015 BuildID: 341710 $", "ShouldTransferFiles": "YES", "TargetType": "Machine", "M
inHosts": 1, "NumCkpts_RAW": 0, "RequestCpus": 1, "RemoteUserCpu": 27288.0d, "BlockWrites": 0, "NiceUser": false, "Out": "process.out", "ImageSize_RAW": 811948, "BytesSent": 30553.0d, "CumulativeSuspensionTime": 0, "TransferIn": false, "AutoClusterAttrs": "JobUniverse,LastCheckpointPlatform,NumCkpts,ClientMachine,_condor_RequestCpus,_condor_RequestDisk,_condor_RequestGPUs,_condor_RequestMemory,RequestCpus,RequestDisk,RequestGPUs,RequestMemory,BIOCHEM,MachineLastMatchTime,ConcurrencyLimits,NiceUser,Rank,Requirements,ImageSize,MemoryRequirements,User,RemoteGroup,SubmitterGroup,SubmitterUserPrio,Group,WIDsTheme,InteractiveJob,Is_Resumable,WantFlocking,WantGlidein,Scheduler,Owner,JobStart,MemoryUsage,IsExpressQueueJob,DiskUsage,HEP_VO,IsDesktop,OSG_VO,x509userproxysubject,PassedTest,IsLocalCMSJob,IsLocalCMSSlot,IsSAMSlot,IsSAMJob,MaxDiskTempC,IsDedicated,estimated_run_hours,IsCHTCSubmit,RequiresCVMFS,DynamicSlot,PartitionableSlot,Slot1_ExpectedMachineGracefulDrainingCompletion,Slot1_Job
Starts,Slot1_SelfMonitorAge,Slot2_ExpectedMachineGracefulDrainingCompletion,Slot2_JobStarts,Slot2_SelfMonitorAge,Slot3_ExpectedMachineGracefulDrainingCompletion,Slot3_JobStarts,Slot3_SelfMonitorAge,Slot4_ExpectedMachineGracefulDrainingCompletion,Slot4_JobStarts,Slot4_SelfMonitorAge,Slot5_ExpectedMachineGracefulDrainingCompletion,Slot5_JobStarts,Slot5_SelfMonitorAge,Slot6_ExpectedMachineGracefulDrainingCompletion,Slot6_JobStarts,Slot6_SelfMonitorAge,Slot7_ExpectedMachineGracefulDrainingCompletion,Slot7_JobStarts,Slot7_SelfMonitorAge,Slot8_ExpectedMachineGracefulDrainingCompletion,Slot8_JobStarts,Slot8_SelfMonitorAge,Slot1_TotalTimeClaimedBusy,Slot1_TotalTimeUnclaimedIdle,Slot2_TotalTimeClaimedBusy,Slot2_TotalTimeUnclaimedIdle,Slot3_TotalTimeClaimedBusy,Slot3_TotalTimeUnclaimedIdle,Slot4_TotalTimeClaimedBusy,Slot4_TotalTimeUnclaimedIdle,Slot5_TotalTimeClaimedBusy,Slot5_TotalTimeUnclaimedIdle,Slot6_TotalTimeClaimedBusy,Slot6_TotalTimeUnclaimedIdle,Slot7_TotalTimeClaimedBusy,Slot7_Total
TimeUnclaimedIdle,Slot8_TotalTimeClaimedBusy,Slot8_TotalTimeUnclaimedIdle,Slot10_ExpectedMachineGracefulDrainingCompletion,Slot10_JobStarts,Slot10_SelfMonitorAge,Slot11_ExpectedMachineGracefulDrainingCompletion,Slot11_JobStarts,Slot11_SelfMonitorAge,Slot12_ExpectedMachineGracefulDrainingCompletion,Slot12_JobStarts,Slot12_SelfMonitorAge,Slot9_ExpectedMachineGracefulDrainingCompletion,Slot9_JobStarts,Slot9_SelfMonitorAge,Slot12_TotalTimeClaimedBusy,Slot10_TotalTimeClaimedBusy,Slot10_TotalTimeUnclaimedIdle,Slot11_TotalTimeClaimedBusy,Slot11_TotalTimeUnclaimedIdle,Slot12_TotalTimeUnclaimedIdle,Slot9_TotalTimeClaimedBusy,Slot9_TotalTimeUnclaimedIdle,Slot13_ExpectedMachineGracefulDrainingCompletion,Slot13_JobStarts,Slot13_SelfMonitorAge,Slot14_ExpectedMachineGracefulDrainingCompletion,Slot14_JobStarts,Slot14_SelfMonitorAge,Slot15_ExpectedMachineGracefulDrainingCompletion,Slot15_JobStarts,Slot15_SelfMonitorAge,Slot16_ExpectedMachineGracefulDrainingCompletion,Slot16_JobStarts,Slot16_SelfMon
itorAge,IsResumable,WHEN_TO_TRANSFER_OUTPUT,_condor_Requestadmin_mutex_1,_condor_Requestadmin_mutex_2,_condor_Requestadmin_mutex_3,_condor_Requestmachine_token,Requestadmin_mutex_1,Requestadmin_mutex_2,Requestadmin_mutex_3,Requestmachine_token,nyehle,IsBuildJob,IsMatlabBuildJob,TotalJobRunTime,NodeOnline,Slot13_TotalTimeClaimedBusy,Slot13_TotalTimeUnclaimedIdle,Slot14_TotalTimeClaimedBusy,Slot14_TotalTimeUnclaimedIdle,Slot15_TotalTimeClaimedBusy,Slot15_TotalTimeUnclaimedIdle,Slot16_TotalTimeClaimedBusy,Slot16_TotalTimeUnclaimedIdle,TmpIsFull,trResumable,RequiresCMSFrontier,Slot17_ExpectedMachineGracefulDrainingCompletion,Slot17_JobStarts,Slot17_SelfMonitorAge,Slot17_TotalTimeClaimedBusy,Slot17_TotalTimeUnclaimedIdle,Slot18_ExpectedMachineGracefulDrainingCompletion,Slot18_JobStarts,Slot18_SelfMonitorAge,Slot18_TotalTimeClaimedBusy,Slot18_TotalTimeUnclaimedIdle,Slot19_ExpectedMachineGracefulDrainingCompletion,Slot19_JobStarts,Slot19_SelfMonitorAge,Slot19_TotalTimeClaimedBusy,Slot19_To
talTimeUnclaimedIdle,Slot20_ExpectedMachineGracefulDrainingCompletion,Slot20_JobStarts,Slot20_SelfMonitorAge,Slot20_TotalTimeClaimedBusy,Slot20_TotalTimeUnclaimedIdle,Slot21_ExpectedMachineGracefulDrainingCompletion,Slot21_JobStarts,Slot21_SelfMonitorAge,Slot21_TotalTimeClaimedBusy,Slot21_TotalTimeUnclaimedIdle,Slot22_ExpectedMachineGracefulDrainingCompletion,Slot22_JobStarts,Slot22_SelfMonitorAge,Slot22_TotalTimeClaimedBusy,Slot22_TotalTimeUnclaimedIdle,Slot23_ExpectedMachineGracefulDrainingCompletion,Slot23_JobStarts,Slot23_SelfMonitorAge,Slot23_TotalTimeClaimedBusy,Slot23_TotalTimeUnclaimedIdle,Slot24_ExpectedMachineGracefulDrainingCompletion,Slot24_JobStarts,Slot24_SelfMonitorAge,Slot24_TotalTimeClaimedBusy,Slot24_TotalTimeUnclaimedIdle,Slot25_ExpectedMachineGracefulDrainingCompletion,Slot25_JobStarts,Slot25_SelfMonitorAge,Slot25_TotalTimeClaimedBusy,Slot25_TotalTimeUnclaimedIdle,Slot26_ExpectedMachineGracefulDrainingCompletion,Slot26_JobStarts,Slot26_SelfMonitorAge,Slot26_Total
TimeClaimedBusy,Slot26_TotalTimeUnclaimedIdle,Slot27_ExpectedMachineGracefulDrainingCompletion,Slot27_JobStarts,Slot27_SelfMonitorAge,Slot27_TotalTimeClaimedBusy,Slot27_TotalTimeUnclaimedIdle,Slot28_ExpectedMachineGracefulDrainingCompletion,Slot28_JobStarts,Slot28_SelfMonitorAge,Slot28_TotalTimeClaimedBusy,Slot28_TotalTimeUnclaimedIdle,Slot29_ExpectedMachineGracefulDrainingCompletion,Slot29_JobStarts,Slot29_SelfMonitorAge,Slot29_TotalTimeClaimedBusy,Slot29_TotalTimeUnclaimedIdle,Slot30_ExpectedMachineGracefulDrainingCompletion,Slot30_JobStarts,Slot30_SelfMonitorAge,Slot30_TotalTimeClaimedBusy,Slot30_TotalTimeUnclaimedIdle,Slot31_ExpectedMachineGracefulDrainingCompletion,Slot31_JobStarts,Slot31_SelfMonitorAge,Slot31_TotalTimeClaimedBusy,Slot31_TotalTimeUnclaimedIdle,Slot32_ExpectedMachineGracefulDrainingCompletion,Slot32_JobStarts,Slot32_SelfMonitorAge,Slot32_TotalTimeClaimedBusy,Slot32_TotalTimeUnclaimedIdle,ResidentSetSize", "NumCkpts": 0, "Err": "process.err", "RecentBlockWrites":
0, "JobFinishedHookDone": 1446133782, "ProcId": 0, "ImageSize": 1000000, "JobUniverse": 5, "EncryptExecuteDirectory": false, "TransferInputSizeMB": 0, "RecentBlockReads": 0, "SpooledOutputFiles": "CURLTIME_4096502,ChtcWrapper82.out,AuditLog.82,simu_3_82.txt,harvest.log,82.out", "BlockWriteKbytes": 0, "WhenToTransferOutput": "ON_EXIT", "JobCurrentStartExecutingDate": 1446106291, "ExitBySignal": false, "LastMatchTime": 1446106290, "OnExitHold": false, "OrigMaxHosts": 1, "RequestMemory": 1000, "NumJobStarts": 1, "TerminationPending": true, "TotalSuspensions": 0, "BlockReads": 0, "DAGManJobId": 49581933, "MemoryUsage": 146, "ExitCode": 0, "JobNotification": 0, "BlockReadKbytes": 0, "NumJobMatches": 1, "LocalUserCpu": 0.0d, "LastJobStatus": 2, "BufferBlockSize": 32768, "CommittedTime": 27491, "ExecutableSize_RAW": 6, "LastSuspensionTime": 0, "Matlab": "R2011b", "UserLog": "/home/xguo23/finally_2/Simulation_condor/model_3/82/process.log", "DAGManNodesMask": "0,1,2,4,5,7,9,10,11,12,13,16,
17,24,27", "CumulativeSlotTime": 27491.0d, "LastJobLeaseRenewal": 1446133781, "MachineAttrSlotWeight0": 1, "NumSystemHolds": 0, "BytesRecvd": 285053.0d, "CondorPlatform": "$CondorPlatform: X86_64-RedHat_6.6 $", "JOBGLIDEIN_ResourceName": "$$([IfThenElse(IsUndefined(TARGET.GLIDEIN_ResourceName), IfThenElse(IsUndefined(TARGET.GLIDEIN_Site), \"wisc.edu\", TARGET.GLIDEIN_Site), TARGET.GLIDEIN_ResourceName)])", "DAGNodeName": "82+82", "PeriodicRelease": "error", "JobRunCount": 1, "LastRemoteHost": "slot1@e433.chtc.wisc.edu", "JobPrio": 0, "LocalSysCpu": 0.0d, "ExecutableSize": 7, "RemoteSysCpu": 173.0d, "TransferInput": "/home/xguo23/finally_2/Simulation_condor/data/82/,/home/xguo23/finally_2/Simulation_condor/data/shared/", "PeriodicHold": false, "WantRemoteIO": true, "CommittedSuspensionTime": 0, "DAGParentNodeNames": "", "CompletionDate": 1446133781, "StreamErr": false, "RecentBlockReadKbytes": 0, "WantRemoteSyscalls": false, "NumShadowStarts": 1, "MachineAttrCpus0": 1, "DAGManNodesLo
g": "/home/xguo23/finally_2/Simulation_condor/model_3/./mydag.dag.nodes.log", "Owner": "xguo23", "Requirements": "undefined", "DiskUsage": 1250000, "JobLeaseDuration": 2400, "ClusterId": 49581989, "BufferSize": 524288, "IsCHTCSubmit": true, "RecentStatsLifetimeStarter": 1200, "CommittedSlotTime": 27491.0d, "Args": "--type=Matlab --version=R2011b --cmdtorun=simu_condor --unique=82 -- 3", "Environment": "", "LastPublicClaimId": "<128.105.245.233:28601>#1443991451#13496#...", "Iwd": "/home/xguo23/finally_2/Simulation_condor/model_3/82", "QDate": 1446105374, "CurrentHosts": 0, "User": "xguo23@chtc.wisc.edu", "StreamOut": false }
-{ "GlobalJobId": "submit-3.chtc.wisc.edu#49582049.0#1446105441", "StatsLifetimeStarter": 26296, "JobStartDate": 1446106482, "SubmitEventNotes": "DAG Node: 112+112", "JobStatus": 4, "LeaveJobInQueue": false, "StartdPrincipal": "execute-side@matchsession/128.105.245.245", "WantRHEL6": true, "OnExitRemove": true, "JobCurrentStartDate": 1446106482, "CoreSize": 0, "MATCH_EXP_JOBGLIDEIN_ResourceName": "wisc.edu", "Rank": 0.0d, "ExitStatus": 0, "ResidentSetSize": 150000, "RemoteWallClockTime": 26298.0d, "WantCheckpoint": false, "In": "/dev/null", "MaxHosts": 1, "RootDir": "/", "NumRestarts": 0, "RecentBlockWriteKbytes": 0, "DiskUsage_RAW": 1216669, "EnteredCurrentStatus": 1446132780, "ResidentSetSize_RAW": 126892, "RequestDisk": 1000000, "MyType": "Job", "PeriodicRemove": false, "Cmd": "/home/xguo23/finally_2/Simulation_condor/chtcjobwrapper", "CondorVersion": "$CondorVersion: 8.5.0 Sep 16 2015 BuildID: 341710 $", "ShouldTransferFiles": "YES", "TargetType": "Machine", "MinHosts": 1, "NumCk
pts_RAW": 0, "RequestCpus": 1, "RemoteUserCpu": 26097.0d, "BlockWrites": 0, "NiceUser": false, "Out": "process.out", "ImageSize_RAW": 811948, "BytesSent": 31904.0d, "CumulativeSuspensionTime": 0, "TransferIn": false, "NumCkpts": 0, "Err": "process.err", "RecentBlockWrites": 0, "JobFinishedHookDone": 1446132780, "ProcId": 0, "ImageSize": 1000000, "JobUniverse": 5, "EncryptExecuteDirectory": false, "TransferInputSizeMB": 0, "RecentBlockReads": 0, "SpooledOutputFiles": "CURLTIME_2601607,ChtcWrapper112.out,AuditLog.112,simu_3_112.txt,harvest.log,112.out", "BlockWriteKbytes": 0, "WhenToTransferOutput": "ON_EXIT", "JobCurrentStartExecutingDate": 1446106484, "ExitBySignal": false, "LastMatchTime": 1446106482, "OnExitHold": false, "OrigMaxHosts": 1, "RequestMemory": 1000, "NumJobStarts": 1, "TerminationPending": true, "TotalSuspensions": 0, "BlockReads": 0, "DAGManJobId": 49581933, "MemoryUsage": 146, "ExitCode": 0, "JobNotification": 0, "BlockReadKbytes": 0, "NumJobMatches": 1, "LocalUserC
pu": 0.0d, "LastJobStatus": 2, "BufferBlockSize": 32768, "CommittedTime": 26298, "ExecutableSize_RAW": 6, "LastSuspensionTime": 0, "Matlab": "R2011b", "UserLog": "/home/xguo23/finally_2/Simulation_condor/model_3/112/process.log", "DAGManNodesMask": "0,1,2,4,5,7,9,10,11,12,13,16,17,24,27", "CumulativeSlotTime": 26298.0d, "LastJobLeaseRenewal": 1446132780, "MachineAttrSlotWeight0": 1, "NumSystemHolds": 0, "BytesRecvd": 285054.0d, "CondorPlatform": "$CondorPlatform: X86_64-RedHat_6.6 $", "JOBGLIDEIN_ResourceName": "$$([IfThenElse(IsUndefined(TARGET.GLIDEIN_ResourceName), IfThenElse(IsUndefined(TARGET.GLIDEIN_Site), \"wisc.edu\", TARGET.GLIDEIN_Site), TARGET.GLIDEIN_ResourceName)])", "DAGNodeName": "112+112", "PeriodicRelease": "error", "JobRunCount": 1, "LastRemoteHost": "slot1@e445.chtc.WISC.EDU", "JobPrio": 0, "LocalSysCpu": 0.0d, "ExecutableSize": 7, "RemoteSysCpu": 164.0d, "TransferInput": "/home/xguo23/finally_2/Simulation_condor/data/112/,/home/xguo23/finally_2/Simulation_condor/
data/shared/", "PeriodicHold": false, "WantRemoteIO": true, "CommittedSuspensionTime": 0, "DAGParentNodeNames": "", "CompletionDate": 1446132780, "StreamErr": false, "RecentBlockReadKbytes": 0, "WantRemoteSyscalls": false, "NumShadowStarts": 1, "MachineAttrCpus0": 1, "DAGManNodesLog": "/home/xguo23/finally_2/Simulation_condor/model_3/./mydag.dag.nodes.log", "Owner": "xguo23", "Requirements": "undefined", "DiskUsage": 1250000, "JobLeaseDuration": 2400, "ClusterId": 49582049, "BufferSize": 524288, "IsCHTCSubmit": true, "RecentStatsLifetimeStarter": 1200, "CommittedSlotTime": 26298.0d, "Args": "--type=Matlab --version=R2011b --cmdtorun=simu_condor --unique=112 -- 3", "Environment": "", "LastPublicClaimId": "<128.105.245.245:48407>#1443991450#14631#...", "Iwd": "/home/xguo23/finally_2/Simulation_condor/model_3/112", "QDate": 1446105441, "CurrentHosts": 0, "User": "xguo23@chtc.wisc.edu", "StreamOut": false }
-{ "GlobalJobId": "submit-3.chtc.wisc.edu#49582050.0#1446105441", "StatsLifetimeStarter": 27141, "JobStartDate": 1446106482, "SubmitEventNotes": "DAG Node: 301+301", "JobStatus": 4, "LeaveJobInQueue": false, "StartdPrincipal": "execute-side@matchsession/128.105.245.172", "WantRHEL6": true, "OnExitRemove": true, "JobCurrentStartDate": 1446106482, "CoreSize": 0, "MATCH_EXP_JOBGLIDEIN_ResourceName": "wisc.edu", "Rank": 0.0d, "ExitStatus": 0, "ResidentSetSize": 150000, "RemoteWallClockTime": 27143.0d, "WantCheckpoint": false, "In": "/dev/null", "MaxHosts": 1, "RootDir": "/", "NumRestarts": 0, "RecentBlockWriteKbytes": 0, "DiskUsage_RAW": 1216669, "EnteredCurrentStatus": 1446133625, "ResidentSetSize_RAW": 126464, "RequestDisk": 1000000, "MyType": "Job", "PeriodicRemove": false, "Cmd": "/home/xguo23/finally_2/Simulation_condor/chtcjobwrapper", "CondorVersion": "$CondorVersion: 8.5.0 Sep 16 2015 BuildID: 341710 $", "ShouldTransferFiles": "YES", "TargetType": "Machine", "MinHosts": 1, "NumCk
pts_RAW": 0, "RequestCpus": 1, "RemoteUserCpu": 26895.0d, "BlockWrites": 0, "NiceUser": false, "Out": "process.out", "ImageSize_RAW": 811948, "BytesSent": 31905.0d, "CumulativeSuspensionTime": 0, "TransferIn": false, "NumCkpts": 0, "Err": "process.err", "RecentBlockWrites": 0, "JobFinishedHookDone": 1446133625, "ProcId": 0, "ImageSize": 1000000, "JobUniverse": 5, "EncryptExecuteDirectory": false, "TransferInputSizeMB": 0, "RecentBlockReads": 0, "SpooledOutputFiles": "CURLTIME_2158419,ChtcWrapper301.out,AuditLog.301,simu_3_301.txt,harvest.log,301.out", "BlockWriteKbytes": 0, "WhenToTransferOutput": "ON_EXIT", "JobCurrentStartExecutingDate": 1446106484, "ExitBySignal": false, "LastMatchTime": 1446106482, "OnExitHold": false, "OrigMaxHosts": 1, "RequestMemory": 1000, "NumJobStarts": 1, "TerminationPending": true, "TotalSuspensions": 0, "BlockReads": 0, "DAGManJobId": 49581933, "MemoryUsage": 146, "ExitCode": 0, "JobNotification": 0, "BlockReadKbytes": 0, "NumJobMatches": 1, "LocalUserC
pu": 0.0d, "LastJobStatus": 2, "BufferBlockSize": 32768, "CommittedTime": 27143, "ExecutableSize_RAW": 6, "LastSuspensionTime": 0, "Matlab": "R2011b", "UserLog": "/home/xguo23/finally_2/Simulation_condor/model_3/301/process.log", "DAGManNodesMask": "0,1,2,4,5,7,9,10,11,12,13,16,17,24,27", "CumulativeSlotTime": 27143.0d, "LastJobLeaseRenewal": 1446133625, "MachineAttrSlotWeight0": 1, "NumSystemHolds": 0, "BytesRecvd": 285054.0d, "CondorPlatform": "$CondorPlatform: X86_64-RedHat_6.6 $", "JOBGLIDEIN_ResourceName": "$$([IfThenElse(IsUndefined(TARGET.GLIDEIN_ResourceName), IfThenElse(IsUndefined(TARGET.GLIDEIN_Site), \"wisc.edu\", TARGET.GLIDEIN_Site), TARGET.GLIDEIN_ResourceName)])", "DAGNodeName": "301+301", "PeriodicRelease": "error", "JobRunCount": 1, "LastRemoteHost": "slot1@e372.chtc.wisc.edu", "JobPrio": 0, "LocalSysCpu": 0.0d, "ExecutableSize": 7, "RemoteSysCpu": 201.0d, "TransferInput": "/home/xguo23/finally_2/Simulation_condor/data/301/,/home/xguo23/finally_2/Simulation_condor/
data/shared/", "PeriodicHold": false, "WantRemoteIO": true, "CommittedSuspensionTime": 0, "DAGParentNodeNames": "", "CompletionDate": 1446133625, "StreamErr": false, "RecentBlockReadKbytes": 0, "WantRemoteSyscalls": false, "NumShadowStarts": 1, "MachineAttrCpus0": 1, "DAGManNodesLog": "/home/xguo23/finally_2/Simulation_condor/model_3/./mydag.dag.nodes.log", "Owner": "xguo23", "Requirements": "undefined", "DiskUsage": 1250000, "JobLeaseDuration": 2400, "ClusterId": 49582050, "BufferSize": 524288, "IsCHTCSubmit": true, "RecentStatsLifetimeStarter": 1200, "CommittedSlotTime": 27143.0d, "Args": "--type=Matlab --version=R2011b --cmdtorun=simu_condor --unique=301 -- 3", "Environment": "", "LastPublicClaimId": "<128.105.245.172:19856>#1444760019#9307#...", "Iwd": "/home/xguo23/finally_2/Simulation_condor/model_3/301", "QDate": 1446105441, "CurrentHosts": 0, "User": "xguo23@chtc.wisc.edu", "StreamOut": false }
-{ "GlobalJobId": "submit-3.chtc.wisc.edu#49582065.0#1446105458", "StatsLifetimeStarter": 25606, "JobStartDate": 1446107042, "SubmitEventNotes": "DAG Node: 401+401", "JobStatus": 4, "LeaveJobInQueue": false, "AutoClusterId": 24, "StartdPrincipal": "execute-side@matchsession/128.105.245.206", "WantRHEL6": true, "OnExitRemove": true, "JobCurrentStartDate": 1446107042, "CoreSize": 0, "MATCH_EXP_JOBGLIDEIN_ResourceName": "wisc.edu", "ExitStatus": 0, "Rank": 0.0d, "ResidentSetSize": 150000, "RemoteWallClockTime": 25607.0d, "WantCheckpoint": false, "In": "/dev/null", "MaxHosts": 1, "RootDir": "/", "NumRestarts": 0, "RecentBlockWriteKbytes": 0, "DiskUsage_RAW": 1216669, "EnteredCurrentStatus": 1446132649, "ResidentSetSize_RAW": 126608, "RequestDisk": 1000000, "MyType": "Job", "PeriodicRemove": false, "Cmd": "/home/xguo23/finally_2/Simulation_condor/chtcjobwrapper", "CondorVersion": "$CondorVersion: 8.5.0 Sep 16 2015 BuildID: 341710 $", "ShouldTransferFiles": "YES", "TargetType": "Machine",
"MinHosts": 1, "NumCkpts_RAW": 0, "RequestCpus": 1, "RemoteUserCpu": 25478.0d, "BlockWrites": 0, "NiceUser": false, "Out": "process.out", "ImageSize_RAW": 811948, "BytesSent": 30661.0d, "CumulativeSuspensionTime": 0, "TransferIn": false, "AutoClusterAttrs": "JobUniverse,LastCheckpointPlatform,NumCkpts,ClientMachine,_condor_RequestCpus,_condor_RequestDisk,_condor_RequestGPUs,_condor_RequestMemory,RequestCpus,RequestDisk,RequestGPUs,RequestMemory,BIOCHEM,MachineLastMatchTime,ConcurrencyLimits,NiceUser,Rank,Requirements,ImageSize,MemoryRequirements,User,RemoteGroup,SubmitterGroup,SubmitterUserPrio,Group,WIDsTheme,InteractiveJob,Is_Resumable,WantFlocking,WantGlidein,Scheduler,Owner,JobStart,MemoryUsage,IsExpressQueueJob,DiskUsage,HEP_VO,IsDesktop,OSG_VO,x509userproxysubject,PassedTest,IsLocalCMSJob,IsLocalCMSSlot,IsSAMSlot,IsSAMJob,MaxDiskTempC,IsDedicated,estimated_run_hours,IsCHTCSubmit,RequiresCVMFS,DynamicSlot,PartitionableSlot,Slot1_ExpectedMachineGracefulDrainingCompletion,Slot1_J
obStarts,Slot1_SelfMonitorAge,Slot2_ExpectedMachineGracefulDrainingCompletion,Slot2_JobStarts,Slot2_SelfMonitorAge,Slot3_ExpectedMachineGracefulDrainingCompletion,Slot3_JobStarts,Slot3_SelfMonitorAge,Slot4_ExpectedMachineGracefulDrainingCompletion,Slot4_JobStarts,Slot4_SelfMonitorAge,Slot5_ExpectedMachineGracefulDrainingCompletion,Slot5_JobStarts,Slot5_SelfMonitorAge,Slot6_ExpectedMachineGracefulDrainingCompletion,Slot6_JobStarts,Slot6_SelfMonitorAge,Slot7_ExpectedMachineGracefulDrainingCompletion,Slot7_JobStarts,Slot7_SelfMonitorAge,Slot8_ExpectedMachineGracefulDrainingCompletion,Slot8_JobStarts,Slot8_SelfMonitorAge,Slot1_TotalTimeClaimedBusy,Slot1_TotalTimeUnclaimedIdle,Slot2_TotalTimeClaimedBusy,Slot2_TotalTimeUnclaimedIdle,Slot3_TotalTimeClaimedBusy,Slot3_TotalTimeUnclaimedIdle,Slot4_TotalTimeClaimedBusy,Slot4_TotalTimeUnclaimedIdle,Slot5_TotalTimeClaimedBusy,Slot5_TotalTimeUnclaimedIdle,Slot6_TotalTimeClaimedBusy,Slot6_TotalTimeUnclaimedIdle,Slot7_TotalTimeClaimedBusy,Slot7_Tot
alTimeUnclaimedIdle,Slot8_TotalTimeClaimedBusy,Slot8_TotalTimeUnclaimedIdle,Slot10_ExpectedMachineGracefulDrainingCompletion,Slot10_JobStarts,Slot10_SelfMonitorAge,Slot11_ExpectedMachineGracefulDrainingCompletion,Slot11_JobStarts,Slot11_SelfMonitorAge,Slot12_ExpectedMachineGracefulDrainingCompletion,Slot12_JobStarts,Slot12_SelfMonitorAge,Slot9_ExpectedMachineGracefulDrainingCompletion,Slot9_JobStarts,Slot9_SelfMonitorAge,Slot12_TotalTimeClaimedBusy,Slot10_TotalTimeClaimedBusy,Slot10_TotalTimeUnclaimedIdle,Slot11_TotalTimeClaimedBusy,Slot11_TotalTimeUnclaimedIdle,Slot12_TotalTimeUnclaimedIdle,Slot9_TotalTimeClaimedBusy,Slot9_TotalTimeUnclaimedIdle,Slot13_ExpectedMachineGracefulDrainingCompletion,Slot13_JobStarts,Slot13_SelfMonitorAge,Slot14_ExpectedMachineGracefulDrainingCompletion,Slot14_JobStarts,Slot14_SelfMonitorAge,Slot15_ExpectedMachineGracefulDrainingCompletion,Slot15_JobStarts,Slot15_SelfMonitorAge,Slot16_ExpectedMachineGracefulDrainingCompletion,Slot16_JobStarts,Slot16_SelfM
onitorAge,IsResumable,WHEN_TO_TRANSFER_OUTPUT,_condor_Requestadmin_mutex_1,_condor_Requestadmin_mutex_2,_condor_Requestadmin_mutex_3,_condor_Requestmachine_token,Requestadmin_mutex_1,Requestadmin_mutex_2,Requestadmin_mutex_3,Requestmachine_token,nyehle,IsBuildJob,IsMatlabBuildJob,TotalJobRunTime,NodeOnline,Slot13_TotalTimeClaimedBusy,Slot13_TotalTimeUnclaimedIdle,Slot14_TotalTimeClaimedBusy,Slot14_TotalTimeUnclaimedIdle,Slot15_TotalTimeClaimedBusy,Slot15_TotalTimeUnclaimedIdle,Slot16_TotalTimeClaimedBusy,Slot16_TotalTimeUnclaimedIdle,TmpIsFull,trResumable,RequiresCMSFrontier,Slot17_ExpectedMachineGracefulDrainingCompletion,Slot17_JobStarts,Slot17_SelfMonitorAge,Slot17_TotalTimeClaimedBusy,Slot17_TotalTimeUnclaimedIdle,Slot18_ExpectedMachineGracefulDrainingCompletion,Slot18_JobStarts,Slot18_SelfMonitorAge,Slot18_TotalTimeClaimedBusy,Slot18_TotalTimeUnclaimedIdle,Slot19_ExpectedMachineGracefulDrainingCompletion,Slot19_JobStarts,Slot19_SelfMonitorAge,Slot19_TotalTimeClaimedBusy,Slot19_
TotalTimeUnclaimedIdle,Slot20_ExpectedMachineGracefulDrainingCompletion,Slot20_JobStarts,Slot20_SelfMonitorAge,Slot20_TotalTimeClaimedBusy,Slot20_TotalTimeUnclaimedIdle,Slot21_ExpectedMachineGracefulDrainingCompletion,Slot21_JobStarts,Slot21_SelfMonitorAge,Slot21_TotalTimeClaimedBusy,Slot21_TotalTimeUnclaimedIdle,Slot22_ExpectedMachineGracefulDrainingCompletion,Slot22_JobStarts,Slot22_SelfMonitorAge,Slot22_TotalTimeClaimedBusy,Slot22_TotalTimeUnclaimedIdle,Slot23_ExpectedMachineGracefulDrainingCompletion,Slot23_JobStarts,Slot23_SelfMonitorAge,Slot23_TotalTimeClaimedBusy,Slot23_TotalTimeUnclaimedIdle,Slot24_ExpectedMachineGracefulDrainingCompletion,Slot24_JobStarts,Slot24_SelfMonitorAge,Slot24_TotalTimeClaimedBusy,Slot24_TotalTimeUnclaimedIdle,Slot25_ExpectedMachineGracefulDrainingCompletion,Slot25_JobStarts,Slot25_SelfMonitorAge,Slot25_TotalTimeClaimedBusy,Slot25_TotalTimeUnclaimedIdle,Slot26_ExpectedMachineGracefulDrainingCompletion,Slot26_JobStarts,Slot26_SelfMonitorAge,Slot26_Tot
alTimeClaimedBusy,Slot26_TotalTimeUnclaimedIdle,Slot27_ExpectedMachineGracefulDrainingCompletion,Slot27_JobStarts,Slot27_SelfMonitorAge,Slot27_TotalTimeClaimedBusy,Slot27_TotalTimeUnclaimedIdle,Slot28_ExpectedMachineGracefulDrainingCompletion,Slot28_JobStarts,Slot28_SelfMonitorAge,Slot28_TotalTimeClaimedBusy,Slot28_TotalTimeUnclaimedIdle,Slot29_ExpectedMachineGracefulDrainingCompletion,Slot29_JobStarts,Slot29_SelfMonitorAge,Slot29_TotalTimeClaimedBusy,Slot29_TotalTimeUnclaimedIdle,Slot30_ExpectedMachineGracefulDrainingCompletion,Slot30_JobStarts,Slot30_SelfMonitorAge,Slot30_TotalTimeClaimedBusy,Slot30_TotalTimeUnclaimedIdle,Slot31_ExpectedMachineGracefulDrainingCompletion,Slot31_JobStarts,Slot31_SelfMonitorAge,Slot31_TotalTimeClaimedBusy,Slot31_TotalTimeUnclaimedIdle,Slot32_ExpectedMachineGracefulDrainingCompletion,Slot32_JobStarts,Slot32_SelfMonitorAge,Slot32_TotalTimeClaimedBusy,Slot32_TotalTimeUnclaimedIdle,ResidentSetSize", "NumCkpts": 0, "Err": "process.err", "RecentBlockWrites
": 0, "JobFinishedHookDone": 1446132649, "ProcId": 0, "ImageSize": 1000000, "JobUniverse": 5, "EncryptExecuteDirectory": false, "TransferInputSizeMB": 0, "RecentBlockReads": 0, "SpooledOutputFiles": "CURLTIME_1160521,ChtcWrapper401.out,AuditLog.401,simu_3_401.txt,harvest.log,401.out", "BlockWriteKbytes": 0, "WhenToTransferOutput": "ON_EXIT", "JobCurrentStartExecutingDate": 1446107043, "ExitBySignal": false, "LastMatchTime": 1446107042, "OnExitHold": false, "OrigMaxHosts": 1, "RequestMemory": 1000, "NumJobStarts": 1, "TerminationPending": true, "TotalSuspensions": 0, "BlockReads": 0, "DAGManJobId": 49581933, "MemoryUsage": 146, "ExitCode": 0, "JobNotification": 0, "BlockReadKbytes": 0, "NumJobMatches": 1, "LocalUserCpu": 0.0d, "LastJobStatus": 2, "BufferBlockSize": 32768, "CommittedTime": 25607, "ExecutableSize_RAW": 6, "LastSuspensionTime": 0, "Matlab": "R2011b", "UserLog": "/home/xguo23/finally_2/Simulation_condor/model_3/401/process.log", "DAGManNodesMask": "0,1,2,4,5,7,9,10,11,12
,13,16,17,24,27", "CumulativeSlotTime": 25607.0d, "LastJobLeaseRenewal": 1446132649, "MachineAttrSlotWeight0": 1, "NumSystemHolds": 0, "BytesRecvd": 285054.0d, "CondorPlatform": "$CondorPlatform: X86_64-RedHat_6.6 $", "JOBGLIDEIN_ResourceName": "$$([IfThenElse(IsUndefined(TARGET.GLIDEIN_ResourceName), IfThenElse(IsUndefined(TARGET.GLIDEIN_Site), \"wisc.edu\", TARGET.GLIDEIN_Site), TARGET.GLIDEIN_ResourceName)])", "DAGNodeName": "401+401", "PeriodicRelease": "error", "JobRunCount": 1, "LastRemoteHost": "slot1@e406.chtc.wisc.edu", "JobPrio": 0, "LocalSysCpu": 0.0d, "ExecutableSize": 7, "RemoteSysCpu": 89.0d, "TransferInput": "/home/xguo23/finally_2/Simulation_condor/data/401/,/home/xguo23/finally_2/Simulation_condor/data/shared/", "PeriodicHold": false, "WantRemoteIO": true, "CommittedSuspensionTime": 0, "DAGParentNodeNames": "", "CompletionDate": 1446132649, "StreamErr": false, "RecentBlockReadKbytes": 0, "WantRemoteSyscalls": false, "NumShadowStarts": 1, "MachineAttrCpus0": 1, "DAGM
anNodesLog": "/home/xguo23/finally_2/Simulation_condor/model_3/./mydag.dag.nodes.log", "Owner": "xguo23", "Requirements": "undefined", "DiskUsage": 1250000, "JobLeaseDuration": 2400, "ClusterId": 49582065, "BufferSize": 524288, "IsCHTCSubmit": true, "RecentStatsLifetimeStarter": 1200, "CommittedSlotTime": 25607.0d, "Args": "--type=Matlab --version=R2011b --cmdtorun=simu_condor --unique=401 -- 3", "Environment": "", "LastPublicClaimId": "<128.105.245.206:27946>#1443991437#15826#...", "Iwd": "/home/xguo23/finally_2/Simulation_condor/model_3/401", "QDate": 1446105458, "CurrentHosts": 0, "User": "xguo23@chtc.wisc.edu", "StreamOut": false }
-{ "GlobalJobId": "submit-3.chtc.wisc.edu#49582094.0#1446105491", "StatsLifetimeStarter": 25168, "JobStartDate": 1446107489, "SubmitEventNotes": "DAG Node: 106+106", "JobStatus": 4, "LeaveJobInQueue": false, "StartdPrincipal": "execute-side@matchsession/128.104.55.83", "WantRHEL6": true, "OnExitRemove": true, "JobCurrentStartDate": 1446107489, "CoreSize": 0, "MATCH_EXP_JOBGLIDEIN_ResourceName": "wisc.edu", "Rank": 0.0d, "ExitStatus": 0, "ResidentSetSize": 75000, "RemoteWallClockTime": 25169.0d, "WantCheckpoint": false, "In": "/dev/null", "MaxHosts": 1, "RootDir": "/", "NumRestarts": 0, "RecentBlockWriteKbytes": 4, "DiskUsage_RAW": 1216669, "EnteredCurrentStatus": 1446132658, "ResidentSetSize_RAW": 72016, "RequestDisk": 1000000, "MyType": "Job", "PeriodicRemove": false, "Cmd": "/home/xguo23/finally_2/Simulation_condor/chtcjobwrapper", "CondorVersion": "$CondorVersion: 8.5.0 Sep 16 2015 BuildID: 341710 $", "ShouldTransferFiles": "YES", "TargetType": "Machine", "MinHosts": 1, "NumCkpts_
RAW": 0, "RequestCpus": 1, "RemoteUserCpu": 24949.0d, "BlockWrites": 1, "NiceUser": false, "Out": "process.out", "ImageSize_RAW": 119520, "BytesSent": 30486.0d, "CumulativeSuspensionTime": 0, "TransferIn": false, "NumCkpts": 0, "Err": "process.err", "RecentBlockWrites": 1, "JobFinishedHookDone": 1446132658, "ProcId": 0, "ImageSize": 125000, "JobUniverse": 5, "EncryptExecuteDirectory": false, "TransferInputSizeMB": 0, "RecentBlockReads": 86, "SpooledOutputFiles": "CURLTIME_122139,ChtcWrapper106.out,AuditLog.106,simu_3_106.txt,harvest.log,106.out", "BlockWriteKbytes": 4, "WhenToTransferOutput": "ON_EXIT", "JobCurrentStartExecutingDate": 1446107490, "ExitBySignal": false, "LastMatchTime": 1446107489, "OnExitHold": false, "OrigMaxHosts": 1, "RequestMemory": 1000, "NumJobStarts": 1, "TerminationPending": true, "TotalSuspensions": 0, "BlockReads": 665, "DAGManJobId": 49581933, "MemoryUsage": 73, "ExitCode": 0, "JobNotification": 0, "BlockReadKbytes": 26620, "NumJobMatches": 1, "LocalUserC
pu": 0.0d, "LastJobStatus": 2, "BufferBlockSize": 32768, "CommittedTime": 25169, "ExecutableSize_RAW": 6, "LastSuspensionTime": 0, "Matlab": "R2011b", "UserLog": "/home/xguo23/finally_2/Simulation_condor/model_3/106/process.log", "DAGManNodesMask": "0,1,2,4,5,7,9,10,11,12,13,16,17,24,27", "CumulativeSlotTime": 25169.0d, "LastJobLeaseRenewal": 1446132658, "MachineAttrSlotWeight0": 1, "NumSystemHolds": 0, "BytesRecvd": 285054.0d, "CondorPlatform": "$CondorPlatform: X86_64-RedHat_6.6 $", "JOBGLIDEIN_ResourceName": "$$([IfThenElse(IsUndefined(TARGET.GLIDEIN_ResourceName), IfThenElse(IsUndefined(TARGET.GLIDEIN_Site), \"wisc.edu\", TARGET.GLIDEIN_Site), TARGET.GLIDEIN_ResourceName)])", "DAGNodeName": "106+106", "PeriodicRelease": "error", "JobRunCount": 1, "LastRemoteHost": "slot1@c064.chtc.wisc.edu", "JobPrio": 0, "LocalSysCpu": 0.0d, "ExecutableSize": 7, "RemoteSysCpu": 204.0d, "TransferInput": "/home/xguo23/finally_2/Simulation_condor/data/106/,/home/xguo23/finally_2/Simulation_condor/
data/shared/", "PeriodicHold": false, "WantRemoteIO": true, "CommittedSuspensionTime": 0, "DAGParentNodeNames": "", "CompletionDate": 1446132658, "StreamErr": false, "RecentBlockReadKbytes": 960, "WantRemoteSyscalls": false, "NumShadowStarts": 1, "MachineAttrCpus0": 1, "DAGManNodesLog": "/home/xguo23/finally_2/Simulation_condor/model_3/./mydag.dag.nodes.log", "Owner": "xguo23", "Requirements": "undefined", "DiskUsage": 1250000, "JobLeaseDuration": 2400, "ClusterId": 49582094, "BufferSize": 524288, "IsCHTCSubmit": true, "RecentStatsLifetimeStarter": 1200, "CommittedSlotTime": 25169.0d, "Args": "--type=Matlab --version=R2011b --cmdtorun=simu_condor --unique=106 -- 3", "Environment": "", "LastPublicClaimId": "<128.104.55.83:25899>#1445308581#1240#...", "Iwd": "/home/xguo23/finally_2/Simulation_condor/model_3/106", "QDate": 1446105491, "CurrentHosts": 0, "User": "xguo23@chtc.wisc.edu", "StreamOut": false }
-{ "GlobalJobId": "submit-3.chtc.wisc.edu#49582098.0#1446105492", "StatsLifetimeStarter": 26020, "JobStartDate": 1446107489, "SubmitEventNotes": "DAG Node: 304+304", "JobStatus": 4, "LeaveJobInQueue": false, "StartdPrincipal": "execute-side@matchsession/128.105.245.223", "WantRHEL6": true, "OnExitRemove": true, "JobCurrentStartDate": 1446107489, "CoreSize": 0, "MATCH_EXP_JOBGLIDEIN_ResourceName": "wisc.edu", "Rank": 0.0d, "ExitStatus": 0, "ResidentSetSize": 150000, "RemoteWallClockTime": 26022.0d, "WantCheckpoint": false, "In": "/dev/null", "MaxHosts": 1, "RootDir": "/", "NumRestarts": 0, "RecentBlockWriteKbytes": 0, "DiskUsage_RAW": 1216669, "EnteredCurrentStatus": 1446133511, "ResidentSetSize_RAW": 128776, "RequestDisk": 1000000, "MyType": "Job", "PeriodicRemove": false, "Cmd": "/home/xguo23/finally_2/Simulation_condor/chtcjobwrapper", "CondorVersion": "$CondorVersion: 8.5.0 Sep 16 2015 BuildID: 341710 $", "ShouldTransferFiles": "YES", "TargetType": "Machine", "MinHosts": 1, "NumCk
pts_RAW": 0, "RequestCpus": 1, "RemoteUserCpu": 25844.0d, "BlockWrites": 0, "NiceUser": false, "Out": "process.out", "ImageSize_RAW": 811948, "BytesSent": 31801.0d, "CumulativeSuspensionTime": 0, "TransferIn": false, "NumCkpts": 0, "Err": "process.err", "RecentBlockWrites": 0, "JobFinishedHookDone": 1446133511, "ProcId": 0, "ImageSize": 1000000, "JobUniverse": 5, "EncryptExecuteDirectory": false, "TransferInputSizeMB": 0, "RecentBlockReads": 0, "SpooledOutputFiles": "CURLTIME_3651606,ChtcWrapper304.out,AuditLog.304,simu_3_304.txt,harvest.log,304.out", "BlockWriteKbytes": 0, "WhenToTransferOutput": "ON_EXIT", "JobCurrentStartExecutingDate": 1446107491, "ExitBySignal": false, "LastMatchTime": 1446107489, "OnExitHold": false, "OrigMaxHosts": 1, "RequestMemory": 1000, "NumJobStarts": 1, "TerminationPending": true, "TotalSuspensions": 0, "BlockReads": 0, "DAGManJobId": 49581933, "MemoryUsage": 146, "ExitCode": 0, "JobNotification": 0, "BlockReadKbytes": 0, "NumJobMatches": 1, "LocalUserC
pu": 0.0d, "LastJobStatus": 2, "BufferBlockSize": 32768, "CommittedTime": 26022, "ExecutableSize_RAW": 6, "LastSuspensionTime": 0, "Matlab": "R2011b", "UserLog": "/home/xguo23/finally_2/Simulation_condor/model_3/304/process.log", "DAGManNodesMask": "0,1,2,4,5,7,9,10,11,12,13,16,17,24,27", "CumulativeSlotTime": 26022.0d, "LastJobLeaseRenewal": 1446133511, "MachineAttrSlotWeight0": 1, "NumSystemHolds": 0, "BytesRecvd": 285054.0d, "CondorPlatform": "$CondorPlatform: X86_64-RedHat_6.6 $", "JOBGLIDEIN_ResourceName": "$$([IfThenElse(IsUndefined(TARGET.GLIDEIN_ResourceName), IfThenElse(IsUndefined(TARGET.GLIDEIN_Site), \"wisc.edu\", TARGET.GLIDEIN_Site), TARGET.GLIDEIN_ResourceName)])", "DAGNodeName": "304+304", "PeriodicRelease": "error", "JobRunCount": 1, "LastRemoteHost": "slot1@e423.chtc.wisc.edu", "JobPrio": 0, "LocalSysCpu": 0.0d, "ExecutableSize": 7, "RemoteSysCpu": 143.0d, "TransferInput": "/home/xguo23/finally_2/Simulation_condor/data/304/,/home/xguo23/finally_2/Simulation_condor/
data/shared/", "PeriodicHold": false, "WantRemoteIO": true, "CommittedSuspensionTime": 0, "DAGParentNodeNames": "", "CompletionDate": 1446133511, "StreamErr": false, "RecentBlockReadKbytes": 0, "WantRemoteSyscalls": false, "NumShadowStarts": 1, "MachineAttrCpus0": 1, "DAGManNodesLog": "/home/xguo23/finally_2/Simulation_condor/model_3/./mydag.dag.nodes.log", "Owner": "xguo23", "Requirements": "undefined", "DiskUsage": 1250000, "JobLeaseDuration": 2400, "ClusterId": 49582098, "BufferSize": 524288, "IsCHTCSubmit": true, "RecentStatsLifetimeStarter": 1200, "CommittedSlotTime": 26022.0d, "Args": "--type=Matlab --version=R2011b --cmdtorun=simu_condor --unique=304 -- 3", "Environment": "", "LastPublicClaimId": "<128.105.245.223:13467>#1444760039#6376#...", "Iwd": "/home/xguo23/finally_2/Simulation_condor/model_3/304", "QDate": 1446105492, "CurrentHosts": 0, "User": "xguo23@chtc.wisc.edu", "StreamOut": false }
-{ "GlobalJobId": "submit-3.chtc.wisc.edu#49582113.0#1446105509", "StatsLifetimeStarter": 26044, "JobStartDate": 1446107490, "SubmitEventNotes": "DAG Node: 206+206", "JobStatus": 4, "LeaveJobInQueue": false, "StartdPrincipal": "execute-side@matchsession/128.105.245.120", "WantRHEL6": true, "OnExitRemove": true, "JobCurrentStartDate": 1446107490, "CoreSize": 0, "MATCH_EXP_JOBGLIDEIN_ResourceName": "wisc.edu", "Rank": 0.0d, "ExitStatus": 0, "ResidentSetSize": 150000, "RemoteWallClockTime": 26045.0d, "WantCheckpoint": false, "In": "/dev/null", "MaxHosts": 1, "RootDir": "/", "NumRestarts": 0, "RecentBlockWriteKbytes": 0, "DiskUsage_RAW": 1216669, "EnteredCurrentStatus": 1446133535, "ResidentSetSize_RAW": 126460, "RequestDisk": 1000000, "MyType": "Job", "PeriodicRemove": false, "Cmd": "/home/xguo23/finally_2/Simulation_condor/chtcjobwrapper", "CondorVersion": "$CondorVersion: 8.5.0 Sep 16 2015 BuildID: 341710 $", "ShouldTransferFiles": "YES", "TargetType": "Machine", "MinHosts": 1, "NumCk
pts_RAW": 0, "RequestCpus": 1, "RemoteUserCpu": 25939.0d, "BlockWrites": 0, "NiceUser": false, "Out": "process.out", "ImageSize_RAW": 811948, "BytesSent": 30596.0d, "CumulativeSuspensionTime": 0, "TransferIn": false, "NumCkpts": 0, "Err": "process.err", "RecentBlockWrites": 0, "JobFinishedHookDone": 1446133535, "ProcId": 0, "ImageSize": 1000000, "JobUniverse": 5, "EncryptExecuteDirectory": false, "TransferInputSizeMB": 0, "RecentBlockReads": 0, "SpooledOutputFiles": "CURLTIME_522843,ChtcWrapper206.out,AuditLog.206,simu_3_206.txt,harvest.log,206.out", "BlockWriteKbytes": 0, "WhenToTransferOutput": "ON_EXIT", "JobCurrentStartExecutingDate": 1446107491, "ExitBySignal": false, "LastMatchTime": 1446107490, "OnExitHold": false, "OrigMaxHosts": 1, "RequestMemory": 1000, "NumJobStarts": 1, "TerminationPending": true, "TotalSuspensions": 0, "BlockReads": 0, "DAGManJobId": 49581933, "MemoryUsage": 146, "ExitCode": 0, "JobNotification": 0, "BlockReadKbytes": 0, "NumJobMatches": 1, "LocalUserCp
u": 0.0d, "LastJobStatus": 2, "BufferBlockSize": 32768, "CommittedTime": 26045, "ExecutableSize_RAW": 6, "LastSuspensionTime": 0, "Matlab": "R2011b", "UserLog": "/home/xguo23/finally_2/Simulation_condor/model_3/206/process.log", "DAGManNodesMask": "0,1,2,4,5,7,9,10,11,12,13,16,17,24,27", "CumulativeSlotTime": 26045.0d, "LastJobLeaseRenewal": 1446133535, "MachineAttrSlotWeight0": 1, "NumSystemHolds": 0, "BytesRecvd": 285054.0d, "CondorPlatform": "$CondorPlatform: X86_64-RedHat_6.6 $", "JOBGLIDEIN_ResourceName": "$$([IfThenElse(IsUndefined(TARGET.GLIDEIN_ResourceName), IfThenElse(IsUndefined(TARGET.GLIDEIN_Site), \"wisc.edu\", TARGET.GLIDEIN_Site), TARGET.GLIDEIN_ResourceName)])", "DAGNodeName": "206+206", "PeriodicRelease": "error", "JobRunCount": 1, "LastRemoteHost": "slot1@e320.chtc.wisc.edu", "JobPrio": 0, "LocalSysCpu": 0.0d, "ExecutableSize": 7, "RemoteSysCpu": 87.0d, "TransferInput": "/home/xguo23/finally_2/Simulation_condor/data/206/,/home/xguo23/finally_2/Simulation_condor/da
ta/shared/", "PeriodicHold": false, "WantRemoteIO": true, "CommittedSuspensionTime": 0, "DAGParentNodeNames": "", "CompletionDate": 1446133535, "StreamErr": false, "RecentBlockReadKbytes": 0, "WantRemoteSyscalls": false, "NumShadowStarts": 1, "MachineAttrCpus0": 1, "DAGManNodesLog": "/home/xguo23/finally_2/Simulation_condor/model_3/./mydag.dag.nodes.log", "Owner": "xguo23", "Requirements": "undefined", "DiskUsage": 1250000, "JobLeaseDuration": 2400, "ClusterId": 49582113, "BufferSize": 524288, "IsCHTCSubmit": true, "RecentStatsLifetimeStarter": 1200, "CommittedSlotTime": 26045.0d, "Args": "--type=Matlab --version=R2011b --cmdtorun=simu_condor --unique=206 -- 3", "Environment": "", "LastPublicClaimId": "<128.105.245.120:45185>#1443991409#14238#...", "Iwd": "/home/xguo23/finally_2/Simulation_condor/model_3/206", "QDate": 1446105509, "CurrentHosts": 0, "User": "xguo23@chtc.wisc.edu", "StreamOut": false }
-{ "GlobalJobId": "submit-3.chtc.wisc.edu#49582119.0#1446105519", "StatsLifetimeStarter": 24928, "JobStartDate": 1446107490, "SubmitEventNotes": "DAG Node: 152+152", "JobStatus": 4, "LeaveJobInQueue": false, "StartdPrincipal": "execute-side@matchsession/128.105.245.242", "WantRHEL6": true, "OnExitRemove": true, "JobCurrentStartDate": 1446107490, "CoreSize": 0, "MATCH_EXP_JOBGLIDEIN_ResourceName": "wisc.edu", "Rank": 0.0d, "ExitStatus": 0, "ResidentSetSize": 150000, "RemoteWallClockTime": 24930.0d, "WantCheckpoint": false, "In": "/dev/null", "MaxHosts": 1, "RootDir": "/", "NumRestarts": 0, "RecentBlockWriteKbytes": 0, "DiskUsage_RAW": 1216669, "EnteredCurrentStatus": 1446132420, "ResidentSetSize_RAW": 128972, "RequestDisk": 1000000, "MyType": "Job", "PeriodicRemove": false, "Cmd": "/home/xguo23/finally_2/Simulation_condor/chtcjobwrapper", "CondorVersion": "$CondorVersion: 8.5.0 Sep 16 2015 BuildID: 341710 $", "ShouldTransferFiles": "YES", "TargetType": "Machine", "MinHosts": 1, "NumCk
pts_RAW": 0, "RequestCpus": 1, "RemoteUserCpu": 24742.0d, "BlockWrites": 0, "NiceUser": false, "Out": "process.out", "ImageSize_RAW": 811948, "BytesSent": 30431.0d, "CumulativeSuspensionTime": 0, "TransferIn": false, "NumCkpts": 0, "Err": "process.err", "RecentBlockWrites": 0, "JobFinishedHookDone": 1446132420, "ProcId": 0, "ImageSize": 1000000, "JobUniverse": 5, "EncryptExecuteDirectory": false, "TransferInputSizeMB": 0, "RecentBlockReads": 0, "SpooledOutputFiles": "CURLTIME_674,ChtcWrapper152.out,AuditLog.152,simu_3_152.txt,harvest.log,152.out", "BlockWriteKbytes": 0, "WhenToTransferOutput": "ON_EXIT", "JobCurrentStartExecutingDate": 1446107491, "ExitBySignal": false, "LastMatchTime": 1446107490, "OnExitHold": false, "OrigMaxHosts": 1, "RequestMemory": 1000, "NumJobStarts": 1, "TerminationPending": true, "TotalSuspensions": 0, "BlockReads": 0, "DAGManJobId": 49581933, "MemoryUsage": 146, "ExitCode": 0, "JobNotification": 0, "BlockReadKbytes": 0, "NumJobMatches": 1, "LocalUserCpu":
0.0d, "LastJobStatus": 2, "BufferBlockSize": 32768, "CommittedTime": 24930, "ExecutableSize_RAW": 6, "LastSuspensionTime": 0, "Matlab": "R2011b", "UserLog": "/home/xguo23/finally_2/Simulation_condor/model_3/152/process.log", "DAGManNodesMask": "0,1,2,4,5,7,9,10,11,12,13,16,17,24,27", "CumulativeSlotTime": 24930.0d, "LastJobLeaseRenewal": 1446132420, "MachineAttrSlotWeight0": 1, "NumSystemHolds": 0, "BytesRecvd": 285054.0d, "CondorPlatform": "$CondorPlatform: X86_64-RedHat_6.6 $", "JOBGLIDEIN_ResourceName": "$$([IfThenElse(IsUndefined(TARGET.GLIDEIN_ResourceName), IfThenElse(IsUndefined(TARGET.GLIDEIN_Site), \"wisc.edu\", TARGET.GLIDEIN_Site), TARGET.GLIDEIN_ResourceName)])", "DAGNodeName": "152+152", "PeriodicRelease": "error", "JobRunCount": 1, "LastRemoteHost": "slot1@e442.chtc.WISC.EDU", "JobPrio": 0, "LocalSysCpu": 0.0d, "ExecutableSize": 7, "RemoteSysCpu": 156.0d, "TransferInput": "/home/xguo23/finally_2/Simulation_condor/data/152/,/home/xguo23/finally_2/Simulation_condor/data
/shared/", "PeriodicHold": false, "WantRemoteIO": true, "CommittedSuspensionTime": 0, "DAGParentNodeNames": "", "CompletionDate": 1446132420, "StreamErr": false, "RecentBlockReadKbytes": 0, "WantRemoteSyscalls": false, "NumShadowStarts": 1, "MachineAttrCpus0": 1, "DAGManNodesLog": "/home/xguo23/finally_2/Simulation_condor/model_3/./mydag.dag.nodes.log", "Owner": "xguo23", "Requirements": "undefined", "DiskUsage": 1250000, "JobLeaseDuration": 2400, "ClusterId": 49582119, "BufferSize": 524288, "IsCHTCSubmit": true, "RecentStatsLifetimeStarter": 1200, "CommittedSlotTime": 24930.0d, "Args": "--type=Matlab --version=R2011b --cmdtorun=simu_condor --unique=152 -- 3", "Environment": "", "LastPublicClaimId": "<128.105.245.242:38884>#1443991450#10374#...", "Iwd": "/home/xguo23/finally_2/Simulation_condor/model_3/152", "QDate": 1446105519, "CurrentHosts": 0, "User": "xguo23@chtc.wisc.edu", "StreamOut": false }
-{ "GlobalJobId": "submit-3.chtc.wisc.edu#49582124.0#1446105525", "StatsLifetimeStarter": 24745, "JobStartDate": 1446107685, "SubmitEventNotes": "DAG Node: 323+323", "JobStatus": 4, "LeaveJobInQueue": false, "AutoClusterId": 13, "StartdPrincipal": "execute-side@matchsession/128.104.55.89", "WantRHEL6": true, "OnExitRemove": true, "JobCurrentStartDate": 1446107685, "CoreSize": 0, "MATCH_EXP_JOBGLIDEIN_ResourceName": "wisc.edu", "ExitStatus": 0, "Rank": 0.0d, "ResidentSetSize": 75000, "RemoteWallClockTime": 24748.0d, "WantCheckpoint": false, "In": "/dev/null", "MaxHosts": 1, "RootDir": "/", "NumRestarts": 0, "RecentBlockWriteKbytes": 0, "DiskUsage_RAW": 1216669, "EnteredCurrentStatus": 1446132433, "ResidentSetSize_RAW": 71248, "RequestDisk": 1000000, "MyType": "Job", "PeriodicRemove": false, "Cmd": "/home/xguo23/finally_2/Simulation_condor/chtcjobwrapper", "CondorVersion": "$CondorVersion: 8.5.0 Sep 16 2015 BuildID: 341710 $", "ShouldTransferFiles": "YES", "TargetType": "Machine", "Min
Hosts": 1, "NumCkpts_RAW": 0, "RequestCpus": 1, "RemoteUserCpu": 21145.0d, "BlockWrites": 1, "NiceUser": false, "Out": "process.out", "ImageSize_RAW": 118000, "BytesSent": 30560.0d, "CumulativeSuspensionTime": 0, "TransferIn": false, "AutoClusterAttrs": "JobUniverse,LastCheckpointPlatform,NumCkpts,ClientMachine,_condor_RequestCpus,_condor_RequestDisk,_condor_RequestGPUs,_condor_RequestMemory,RequestCpus,RequestDisk,RequestGPUs,RequestMemory,BIOCHEM,MachineLastMatchTime,ConcurrencyLimits,NiceUser,Rank,Requirements,ImageSize,MemoryRequirements,User,RemoteGroup,SubmitterGroup,SubmitterUserPrio,Group,WIDsTheme,InteractiveJob,Is_Resumable,WantFlocking,WantGlidein,Scheduler,Owner,JobStart,MemoryUsage,IsExpressQueueJob,DiskUsage,HEP_VO,IsDesktop,OSG_VO,x509userproxysubject,PassedTest,IsLocalCMSJob,IsLocalCMSSlot,IsSAMSlot,IsSAMJob,MaxDiskTempC,IsDedicated,estimated_run_hours,IsCHTCSubmit,RequiresCVMFS,DynamicSlot,PartitionableSlot,Slot1_ExpectedMachineGracefulDrainingCompletion,Slot1_JobSt
arts,Slot1_SelfMonitorAge,Slot2_ExpectedMachineGracefulDrainingCompletion,Slot2_JobStarts,Slot2_SelfMonitorAge,Slot3_ExpectedMachineGracefulDrainingCompletion,Slot3_JobStarts,Slot3_SelfMonitorAge,Slot4_ExpectedMachineGracefulDrainingCompletion,Slot4_JobStarts,Slot4_SelfMonitorAge,Slot5_ExpectedMachineGracefulDrainingCompletion,Slot5_JobStarts,Slot5_SelfMonitorAge,Slot6_ExpectedMachineGracefulDrainingCompletion,Slot6_JobStarts,Slot6_SelfMonitorAge,Slot7_ExpectedMachineGracefulDrainingCompletion,Slot7_JobStarts,Slot7_SelfMonitorAge,Slot8_ExpectedMachineGracefulDrainingCompletion,Slot8_JobStarts,Slot8_SelfMonitorAge,Slot1_TotalTimeClaimedBusy,Slot1_TotalTimeUnclaimedIdle,Slot2_TotalTimeClaimedBusy,Slot2_TotalTimeUnclaimedIdle,Slot3_TotalTimeClaimedBusy,Slot3_TotalTimeUnclaimedIdle,Slot4_TotalTimeClaimedBusy,Slot4_TotalTimeUnclaimedIdle,Slot5_TotalTimeClaimedBusy,Slot5_TotalTimeUnclaimedIdle,Slot6_TotalTimeClaimedBusy,Slot6_TotalTimeUnclaimedIdle,Slot7_TotalTimeClaimedBusy,Slot7_TotalTi
meUnclaimedIdle,Slot8_TotalTimeClaimedBusy,Slot8_TotalTimeUnclaimedIdle,Slot10_ExpectedMachineGracefulDrainingCompletion,Slot10_JobStarts,Slot10_SelfMonitorAge,Slot11_ExpectedMachineGracefulDrainingCompletion,Slot11_JobStarts,Slot11_SelfMonitorAge,Slot12_ExpectedMachineGracefulDrainingCompletion,Slot12_JobStarts,Slot12_SelfMonitorAge,Slot9_ExpectedMachineGracefulDrainingCompletion,Slot9_JobStarts,Slot9_SelfMonitorAge,Slot12_TotalTimeClaimedBusy,Slot10_TotalTimeClaimedBusy,Slot10_TotalTimeUnclaimedIdle,Slot11_TotalTimeClaimedBusy,Slot11_TotalTimeUnclaimedIdle,Slot12_TotalTimeUnclaimedIdle,Slot9_TotalTimeClaimedBusy,Slot9_TotalTimeUnclaimedIdle,Slot13_ExpectedMachineGracefulDrainingCompletion,Slot13_JobStarts,Slot13_SelfMonitorAge,Slot14_ExpectedMachineGracefulDrainingCompletion,Slot14_JobStarts,Slot14_SelfMonitorAge,Slot15_ExpectedMachineGracefulDrainingCompletion,Slot15_JobStarts,Slot15_SelfMonitorAge,Slot16_ExpectedMachineGracefulDrainingCompletion,Slot16_JobStarts,Slot16_SelfMonit
orAge,IsResumable,WHEN_TO_TRANSFER_OUTPUT,_condor_Requestadmin_mutex_1,_condor_Requestadmin_mutex_2,_condor_Requestadmin_mutex_3,_condor_Requestmachine_token,Requestadmin_mutex_1,Requestadmin_mutex_2,Requestadmin_mutex_3,Requestmachine_token,nyehle,IsBuildJob,IsMatlabBuildJob,TotalJobRunTime,NodeOnline,Slot13_TotalTimeClaimedBusy,Slot13_TotalTimeUnclaimedIdle,Slot14_TotalTimeClaimedBusy,Slot14_TotalTimeUnclaimedIdle,Slot15_TotalTimeClaimedBusy,Slot15_TotalTimeUnclaimedIdle,Slot16_TotalTimeClaimedBusy,Slot16_TotalTimeUnclaimedIdle,TmpIsFull,trResumable,RequiresCMSFrontier,Slot17_ExpectedMachineGracefulDrainingCompletion,Slot17_JobStarts,Slot17_SelfMonitorAge,Slot17_TotalTimeClaimedBusy,Slot17_TotalTimeUnclaimedIdle,Slot18_ExpectedMachineGracefulDrainingCompletion,Slot18_JobStarts,Slot18_SelfMonitorAge,Slot18_TotalTimeClaimedBusy,Slot18_TotalTimeUnclaimedIdle,Slot19_ExpectedMachineGracefulDrainingCompletion,Slot19_JobStarts,Slot19_SelfMonitorAge,Slot19_TotalTimeClaimedBusy,Slot19_Tota
lTimeUnclaimedIdle,Slot20_ExpectedMachineGracefulDrainingCompletion,Slot20_JobStarts,Slot20_SelfMonitorAge,Slot20_TotalTimeClaimedBusy,Slot20_TotalTimeUnclaimedIdle,Slot21_ExpectedMachineGracefulDrainingCompletion,Slot21_JobStarts,Slot21_SelfMonitorAge,Slot21_TotalTimeClaimedBusy,Slot21_TotalTimeUnclaimedIdle,Slot22_ExpectedMachineGracefulDrainingCompletion,Slot22_JobStarts,Slot22_SelfMonitorAge,Slot22_TotalTimeClaimedBusy,Slot22_TotalTimeUnclaimedIdle,Slot23_ExpectedMachineGracefulDrainingCompletion,Slot23_JobStarts,Slot23_SelfMonitorAge,Slot23_TotalTimeClaimedBusy,Slot23_TotalTimeUnclaimedIdle,Slot24_ExpectedMachineGracefulDrainingCompletion,Slot24_JobStarts,Slot24_SelfMonitorAge,Slot24_TotalTimeClaimedBusy,Slot24_TotalTimeUnclaimedIdle,Slot25_ExpectedMachineGracefulDrainingCompletion,Slot25_JobStarts,Slot25_SelfMonitorAge,Slot25_TotalTimeClaimedBusy,Slot25_TotalTimeUnclaimedIdle,Slot26_ExpectedMachineGracefulDrainingCompletion,Slot26_JobStarts,Slot26_SelfMonitorAge,Slot26_TotalTi
meClaimedBusy,Slot26_TotalTimeUnclaimedIdle,Slot27_ExpectedMachineGracefulDrainingCompletion,Slot27_JobStarts,Slot27_SelfMonitorAge,Slot27_TotalTimeClaimedBusy,Slot27_TotalTimeUnclaimedIdle,Slot28_ExpectedMachineGracefulDrainingCompletion,Slot28_JobStarts,Slot28_SelfMonitorAge,Slot28_TotalTimeClaimedBusy,Slot28_TotalTimeUnclaimedIdle,Slot29_ExpectedMachineGracefulDrainingCompletion,Slot29_JobStarts,Slot29_SelfMonitorAge,Slot29_TotalTimeClaimedBusy,Slot29_TotalTimeUnclaimedIdle,Slot30_ExpectedMachineGracefulDrainingCompletion,Slot30_JobStarts,Slot30_SelfMonitorAge,Slot30_TotalTimeClaimedBusy,Slot30_TotalTimeUnclaimedIdle,Slot31_ExpectedMachineGracefulDrainingCompletion,Slot31_JobStarts,Slot31_SelfMonitorAge,Slot31_TotalTimeClaimedBusy,Slot31_TotalTimeUnclaimedIdle,Slot32_ExpectedMachineGracefulDrainingCompletion,Slot32_JobStarts,Slot32_SelfMonitorAge,Slot32_TotalTimeClaimedBusy,Slot32_TotalTimeUnclaimedIdle,ResidentSetSize", "NumCkpts": 0, "Err": "process.err", "RecentBlockWrites": 0
, "JobFinishedHookDone": 1446132434, "ProcId": 0, "ImageSize": 125000, "JobUniverse": 5, "EncryptExecuteDirectory": false, "TransferInputSizeMB": 0, "RecentBlockReads": 314, "SpooledOutputFiles": "harvest.log,CURLTIME_3853266,ChtcWrapper323.out,AuditLog.323,simu_3_323.txt,323.out", "BlockWriteKbytes": 4, "WhenToTransferOutput": "ON_EXIT", "JobCurrentStartExecutingDate": 1446107686, "ExitBySignal": false, "LastMatchTime": 1446107685, "OnExitHold": false, "OrigMaxHosts": 1, "RequestMemory": 1000, "NumJobStarts": 1, "TerminationPending": true, "TotalSuspensions": 0, "BlockReads": 1142, "DAGManJobId": 49581933, "MemoryUsage": 73, "ExitCode": 0, "JobNotification": 0, "BlockReadKbytes": 43788, "NumJobMatches": 1, "LocalUserCpu": 0.0d, "LastJobStatus": 2, "BufferBlockSize": 32768, "CommittedTime": 24748, "ExecutableSize_RAW": 6, "LastSuspensionTime": 0, "Matlab": "R2011b", "UserLog": "/home/xguo23/finally_2/Simulation_condor/model_3/323/process.log", "DAGManNodesMask": "0,1,2,4,5,7,9,10,11
,12,13,16,17,24,27", "CumulativeSlotTime": 24748.0d, "LastJobLeaseRenewal": 1446132433, "MachineAttrSlotWeight0": 1, "NumSystemHolds": 0, "BytesRecvd": 285054.0d, "CondorPlatform": "$CondorPlatform: X86_64-RedHat_6.6 $", "JOBGLIDEIN_ResourceName": "$$([IfThenElse(IsUndefined(TARGET.GLIDEIN_ResourceName), IfThenElse(IsUndefined(TARGET.GLIDEIN_Site), \"wisc.edu\", TARGET.GLIDEIN_Site), TARGET.GLIDEIN_ResourceName)])", "DAGNodeName": "323+323", "PeriodicRelease": "error", "JobRunCount": 1, "LastRemoteHost": "slot1@c070.chtc.wisc.edu", "JobPrio": 0, "LocalSysCpu": 0.0d, "ExecutableSize": 7, "RemoteSysCpu": 175.0d, "TransferInput": "/home/xguo23/finally_2/Simulation_condor/data/323/,/home/xguo23/finally_2/Simulation_condor/data/shared/", "PeriodicHold": false, "WantRemoteIO": true, "CommittedSuspensionTime": 0, "DAGParentNodeNames": "", "CompletionDate": 1446132433, "StreamErr": false, "RecentBlockReadKbytes": 4224, "WantRemoteSyscalls": false, "NumShadowStarts": 1, "MachineAttrCpus0": 1
, "DAGManNodesLog": "/home/xguo23/finally_2/Simulation_condor/model_3/./mydag.dag.nodes.log", "Owner": "xguo23", "Requirements": "undefined", "DiskUsage": 1250000, "JobLeaseDuration": 2400, "ClusterId": 49582124, "BufferSize": 524288, "IsCHTCSubmit": true, "RecentStatsLifetimeStarter": 1200, "CommittedSlotTime": 24748.0d, "Args": "--type=Matlab --version=R2011b --cmdtorun=simu_condor --unique=323 -- 3", "Environment": "", "LastPublicClaimId": "<128.104.55.89:32652>#1445371750#1302#...", "Iwd": "/home/xguo23/finally_2/Simulation_condor/model_3/323", "QDate": 1446105525, "CurrentHosts": 0, "User": "xguo23@chtc.wisc.edu", "StreamOut": false }
-{ "GlobalJobId": "submit-3.chtc.wisc.edu#49582148.0#1446105547", "StatsLifetimeStarter": 26230, "JobStartDate": 1446107686, "SubmitEventNotes": "DAG Node: 162+162", "JobStatus": 4, "LeaveJobInQueue": false, "AutoClusterId": 24, "StartdPrincipal": "execute-side@matchsession/128.105.245.170", "WantRHEL6": true, "OnExitRemove": true, "JobCurrentStartDate": 1446107686, "CoreSize": 0, "MATCH_EXP_JOBGLIDEIN_ResourceName": "wisc.edu", "ExitStatus": 0, "Rank": 0.0d, "ResidentSetSize": 150000, "RemoteWallClockTime": 26233.0d, "WantCheckpoint": false, "In": "/dev/null", "MaxHosts": 1, "RootDir": "/", "NumRestarts": 0, "RecentBlockWriteKbytes": 0, "DiskUsage_RAW": 1216669, "EnteredCurrentStatus": 1446133919, "ResidentSetSize_RAW": 126384, "RequestDisk": 1000000, "MyType": "Job", "PeriodicRemove": false, "Cmd": "/home/xguo23/finally_2/Simulation_condor/chtcjobwrapper", "CondorVersion": "$CondorVersion: 8.5.0 Sep 16 2015 BuildID: 341710 $", "ShouldTransferFiles": "YES", "TargetType": "Machine",
"MinHosts": 1, "NumCkpts_RAW": 0, "RequestCpus": 1, "RemoteUserCpu": 26088.0d, "BlockWrites": 0, "NiceUser": false, "Out": "process.out", "ImageSize_RAW": 811948, "BytesSent": 30612.0d, "CumulativeSuspensionTime": 0, "TransferIn": false, "AutoClusterAttrs": "JobUniverse,LastCheckpointPlatform,NumCkpts,ClientMachine,_condor_RequestCpus,_condor_RequestDisk,_condor_RequestGPUs,_condor_RequestMemory,RequestCpus,RequestDisk,RequestGPUs,RequestMemory,BIOCHEM,MachineLastMatchTime,ConcurrencyLimits,NiceUser,Rank,Requirements,ImageSize,MemoryRequirements,User,RemoteGroup,SubmitterGroup,SubmitterUserPrio,Group,WIDsTheme,InteractiveJob,Is_Resumable,WantFlocking,WantGlidein,Scheduler,Owner,JobStart,MemoryUsage,IsExpressQueueJob,DiskUsage,HEP_VO,IsDesktop,OSG_VO,x509userproxysubject,PassedTest,IsLocalCMSJob,IsLocalCMSSlot,IsSAMSlot,IsSAMJob,MaxDiskTempC,IsDedicated,estimated_run_hours,IsCHTCSubmit,RequiresCVMFS,DynamicSlot,PartitionableSlot,Slot1_ExpectedMachineGracefulDrainingCompletion,Slot1_J
obStarts,Slot1_SelfMonitorAge,Slot2_ExpectedMachineGracefulDrainingCompletion,Slot2_JobStarts,Slot2_SelfMonitorAge,Slot3_ExpectedMachineGracefulDrainingCompletion,Slot3_JobStarts,Slot3_SelfMonitorAge,Slot4_ExpectedMachineGracefulDrainingCompletion,Slot4_JobStarts,Slot4_SelfMonitorAge,Slot5_ExpectedMachineGracefulDrainingCompletion,Slot5_JobStarts,Slot5_SelfMonitorAge,Slot6_ExpectedMachineGracefulDrainingCompletion,Slot6_JobStarts,Slot6_SelfMonitorAge,Slot7_ExpectedMachineGracefulDrainingCompletion,Slot7_JobStarts,Slot7_SelfMonitorAge,Slot8_ExpectedMachineGracefulDrainingCompletion,Slot8_JobStarts,Slot8_SelfMonitorAge,Slot1_TotalTimeClaimedBusy,Slot1_TotalTimeUnclaimedIdle,Slot2_TotalTimeClaimedBusy,Slot2_TotalTimeUnclaimedIdle,Slot3_TotalTimeClaimedBusy,Slot3_TotalTimeUnclaimedIdle,Slot4_TotalTimeClaimedBusy,Slot4_TotalTimeUnclaimedIdle,Slot5_TotalTimeClaimedBusy,Slot5_TotalTimeUnclaimedIdle,Slot6_TotalTimeClaimedBusy,Slot6_TotalTimeUnclaimedIdle,Slot7_TotalTimeClaimedBusy,Slot7_Tot
alTimeUnclaimedIdle,Slot8_TotalTimeClaimedBusy,Slot8_TotalTimeUnclaimedIdle,Slot10_ExpectedMachineGracefulDrainingCompletion,Slot10_JobStarts,Slot10_SelfMonitorAge,Slot11_ExpectedMachineGracefulDrainingCompletion,Slot11_JobStarts,Slot11_SelfMonitorAge,Slot12_ExpectedMachineGracefulDrainingCompletion,Slot12_JobStarts,Slot12_SelfMonitorAge,Slot9_ExpectedMachineGracefulDrainingCompletion,Slot9_JobStarts,Slot9_SelfMonitorAge,Slot12_TotalTimeClaimedBusy,Slot10_TotalTimeClaimedBusy,Slot10_TotalTimeUnclaimedIdle,Slot11_TotalTimeClaimedBusy,Slot11_TotalTimeUnclaimedIdle,Slot12_TotalTimeUnclaimedIdle,Slot9_TotalTimeClaimedBusy,Slot9_TotalTimeUnclaimedIdle,Slot13_ExpectedMachineGracefulDrainingCompletion,Slot13_JobStarts,Slot13_SelfMonitorAge,Slot14_ExpectedMachineGracefulDrainingCompletion,Slot14_JobStarts,Slot14_SelfMonitorAge,Slot15_ExpectedMachineGracefulDrainingCompletion,Slot15_JobStarts,Slot15_SelfMonitorAge,Slot16_ExpectedMachineGracefulDrainingCompletion,Slot16_JobStarts,Slot16_SelfM
onitorAge,IsResumable,WHEN_TO_TRANSFER_OUTPUT,_condor_Requestadmin_mutex_1,_condor_Requestadmin_mutex_2,_condor_Requestadmin_mutex_3,_condor_Requestmachine_token,Requestadmin_mutex_1,Requestadmin_mutex_2,Requestadmin_mutex_3,Requestmachine_token,nyehle,IsBuildJob,IsMatlabBuildJob,TotalJobRunTime,NodeOnline,Slot13_TotalTimeClaimedBusy,Slot13_TotalTimeUnclaimedIdle,Slot14_TotalTimeClaimedBusy,Slot14_TotalTimeUnclaimedIdle,Slot15_TotalTimeClaimedBusy,Slot15_TotalTimeUnclaimedIdle,Slot16_TotalTimeClaimedBusy,Slot16_TotalTimeUnclaimedIdle,TmpIsFull,trResumable,RequiresCMSFrontier,Slot17_ExpectedMachineGracefulDrainingCompletion,Slot17_JobStarts,Slot17_SelfMonitorAge,Slot17_TotalTimeClaimedBusy,Slot17_TotalTimeUnclaimedIdle,Slot18_ExpectedMachineGracefulDrainingCompletion,Slot18_JobStarts,Slot18_SelfMonitorAge,Slot18_TotalTimeClaimedBusy,Slot18_TotalTimeUnclaimedIdle,Slot19_ExpectedMachineGracefulDrainingCompletion,Slot19_JobStarts,Slot19_SelfMonitorAge,Slot19_TotalTimeClaimedBusy,Slot19_
TotalTimeUnclaimedIdle,Slot20_ExpectedMachineGracefulDrainingCompletion,Slot20_JobStarts,Slot20_SelfMonitorAge,Slot20_TotalTimeClaimedBusy,Slot20_TotalTimeUnclaimedIdle,Slot21_ExpectedMachineGracefulDrainingCompletion,Slot21_JobStarts,Slot21_SelfMonitorAge,Slot21_TotalTimeClaimedBusy,Slot21_TotalTimeUnclaimedIdle,Slot22_ExpectedMachineGracefulDrainingCompletion,Slot22_JobStarts,Slot22_SelfMonitorAge,Slot22_TotalTimeClaimedBusy,Slot22_TotalTimeUnclaimedIdle,Slot23_ExpectedMachineGracefulDrainingCompletion,Slot23_JobStarts,Slot23_SelfMonitorAge,Slot23_TotalTimeClaimedBusy,Slot23_TotalTimeUnclaimedIdle,Slot24_ExpectedMachineGracefulDrainingCompletion,Slot24_JobStarts,Slot24_SelfMonitorAge,Slot24_TotalTimeClaimedBusy,Slot24_TotalTimeUnclaimedIdle,Slot25_ExpectedMachineGracefulDrainingCompletion,Slot25_JobStarts,Slot25_SelfMonitorAge,Slot25_TotalTimeClaimedBusy,Slot25_TotalTimeUnclaimedIdle,Slot26_ExpectedMachineGracefulDrainingCompletion,Slot26_JobStarts,Slot26_SelfMonitorAge,Slot26_Tot
alTimeClaimedBusy,Slot26_TotalTimeUnclaimedIdle,Slot27_ExpectedMachineGracefulDrainingCompletion,Slot27_JobStarts,Slot27_SelfMonitorAge,Slot27_TotalTimeClaimedBusy,Slot27_TotalTimeUnclaimedIdle,Slot28_ExpectedMachineGracefulDrainingCompletion,Slot28_JobStarts,Slot28_SelfMonitorAge,Slot28_TotalTimeClaimedBusy,Slot28_TotalTimeUnclaimedIdle,Slot29_ExpectedMachineGracefulDrainingCompletion,Slot29_JobStarts,Slot29_SelfMonitorAge,Slot29_TotalTimeClaimedBusy,Slot29_TotalTimeUnclaimedIdle,Slot30_ExpectedMachineGracefulDrainingCompletion,Slot30_JobStarts,Slot30_SelfMonitorAge,Slot30_TotalTimeClaimedBusy,Slot30_TotalTimeUnclaimedIdle,Slot31_ExpectedMachineGracefulDrainingCompletion,Slot31_JobStarts,Slot31_SelfMonitorAge,Slot31_TotalTimeClaimedBusy,Slot31_TotalTimeUnclaimedIdle,Slot32_ExpectedMachineGracefulDrainingCompletion,Slot32_JobStarts,Slot32_SelfMonitorAge,Slot32_TotalTimeClaimedBusy,Slot32_TotalTimeUnclaimedIdle,ResidentSetSize", "NumCkpts": 0, "Err": "process.err", "RecentBlockWrites
": 0, "JobFinishedHookDone": 1446133919, "ProcId": 0, "ImageSize": 1000000, "JobUniverse": 5, "EncryptExecuteDirectory": false, "TransferInputSizeMB": 0, "RecentBlockReads": 0, "SpooledOutputFiles": "CURLTIME_1114551,ChtcWrapper162.out,AuditLog.162,simu_3_162.txt,harvest.log,162.out", "BlockWriteKbytes": 0, "WhenToTransferOutput": "ON_EXIT", "JobCurrentStartExecutingDate": 1446107688, "ExitBySignal": false, "LastMatchTime": 1446107686, "OnExitHold": false, "OrigMaxHosts": 1, "RequestMemory": 1000, "NumJobStarts": 1, "TerminationPending": true, "TotalSuspensions": 0, "BlockReads": 0, "DAGManJobId": 49581933, "MemoryUsage": 146, "ExitCode": 0, "JobNotification": 0, "BlockReadKbytes": 0, "NumJobMatches": 1, "LocalUserCpu": 0.0d, "LastJobStatus": 2, "BufferBlockSize": 32768, "CommittedTime": 26233, "ExecutableSize_RAW": 6, "LastSuspensionTime": 0, "Matlab": "R2011b", "UserLog": "/home/xguo23/finally_2/Simulation_condor/model_3/162/process.log", "DAGManNodesMask": "0,1,2,4,5,7,9,10,11,12
,13,16,17,24,27", "CumulativeSlotTime": 26233.0d, "LastJobLeaseRenewal": 1446133919, "MachineAttrSlotWeight0": 1, "NumSystemHolds": 0, "BytesRecvd": 285054.0d, "CondorPlatform": "$CondorPlatform: X86_64-RedHat_6.6 $", "JOBGLIDEIN_ResourceName": "$$([IfThenElse(IsUndefined(TARGET.GLIDEIN_ResourceName), IfThenElse(IsUndefined(TARGET.GLIDEIN_Site), \"wisc.edu\", TARGET.GLIDEIN_Site), TARGET.GLIDEIN_ResourceName)])", "DAGNodeName": "162+162", "PeriodicRelease": "error", "JobRunCount": 1, "LastRemoteHost": "slot1@e370.chtc.WISC.EDU", "JobPrio": 0, "LocalSysCpu": 0.0d, "ExecutableSize": 7, "RemoteSysCpu": 96.0d, "TransferInput": "/home/xguo23/finally_2/Simulation_condor/data/162/,/home/xguo23/finally_2/Simulation_condor/data/shared/", "PeriodicHold": false, "WantRemoteIO": true, "CommittedSuspensionTime": 0, "DAGParentNodeNames": "", "CompletionDate": 1446133919, "StreamErr": false, "RecentBlockReadKbytes": 0, "WantRemoteSyscalls": false, "NumShadowStarts": 1, "MachineAttrCpus0": 1, "DAGM
anNodesLog": "/home/xguo23/finally_2/Simulation_condor/model_3/./mydag.dag.nodes.log", "Owner": "xguo23", "Requirements": "undefined", "DiskUsage": 1250000, "JobLeaseDuration": 2400, "ClusterId": 49582148, "BufferSize": 524288, "IsCHTCSubmit": true, "RecentStatsLifetimeStarter": 1200, "CommittedSlotTime": 26233.0d, "Args": "--type=Matlab --version=R2011b --cmdtorun=simu_condor --unique=162 -- 3", "Environment": "", "LastPublicClaimId": "<128.105.245.170:9482>#1443991414#13008#...", "Iwd": "/home/xguo23/finally_2/Simulation_condor/model_3/162", "QDate": 1446105547, "CurrentHosts": 0, "User": "xguo23@chtc.wisc.edu", "StreamOut": false }
-{ "GlobalJobId": "submit-3.chtc.wisc.edu#49582154.0#1446105553", "StatsLifetimeStarter": 25874, "JobStartDate": 1446107686, "SubmitEventNotes": "DAG Node: 333+333", "JobStatus": 4, "LeaveJobInQueue": false, "StartdPrincipal": "execute-side@matchsession/128.105.245.120", "WantRHEL6": true, "OnExitRemove": true, "JobCurrentStartDate": 1446107686, "CoreSize": 0, "MATCH_EXP_JOBGLIDEIN_ResourceName": "wisc.edu", "Rank": 0.0d, "ExitStatus": 0, "ResidentSetSize": 150000, "RemoteWallClockTime": 25876.0d, "WantCheckpoint": false, "In": "/dev/null", "MaxHosts": 1, "RootDir": "/", "NumRestarts": 0, "RecentBlockWriteKbytes": 0, "DiskUsage_RAW": 1216669, "EnteredCurrentStatus": 1446133562, "ResidentSetSize_RAW": 125740, "RequestDisk": 1000000, "MyType": "Job", "PeriodicRemove": false, "Cmd": "/home/xguo23/finally_2/Simulation_condor/chtcjobwrapper", "CondorVersion": "$CondorVersion: 8.5.0 Sep 16 2015 BuildID: 341710 $", "ShouldTransferFiles": "YES", "TargetType": "Machine", "MinHosts": 1, "NumCk
pts_RAW": 0, "RequestCpus": 1, "RemoteUserCpu": 25692.0d, "BlockWrites": 0, "NiceUser": false, "Out": "process.out", "ImageSize_RAW": 811948, "BytesSent": 30542.0d, "CumulativeSuspensionTime": 0, "TransferIn": false, "NumCkpts": 0, "Err": "process.err", "RecentBlockWrites": 0, "JobFinishedHookDone": 1446133562, "ProcId": 0, "ImageSize": 1000000, "JobUniverse": 5, "EncryptExecuteDirectory": false, "TransferInputSizeMB": 0, "RecentBlockReads": 0, "SpooledOutputFiles": "CURLTIME_523030,ChtcWrapper333.out,AuditLog.333,simu_3_333.txt,harvest.log,333.out", "BlockWriteKbytes": 0, "WhenToTransferOutput": "ON_EXIT", "JobCurrentStartExecutingDate": 1446107688, "ExitBySignal": false, "LastMatchTime": 1446107686, "OnExitHold": false, "OrigMaxHosts": 1, "RequestMemory": 1000, "NumJobStarts": 1, "TerminationPending": true, "TotalSuspensions": 0, "BlockReads": 0, "DAGManJobId": 49581933, "MemoryUsage": 146, "ExitCode": 0, "JobNotification": 0, "BlockReadKbytes": 0, "NumJobMatches": 1, "LocalUserCp
u": 0.0d, "LastJobStatus": 2, "BufferBlockSize": 32768, "CommittedTime": 25876, "ExecutableSize_RAW": 6, "LastSuspensionTime": 0, "Matlab": "R2011b", "UserLog": "/home/xguo23/finally_2/Simulation_condor/model_3/333/process.log", "DAGManNodesMask": "0,1,2,4,5,7,9,10,11,12,13,16,17,24,27", "CumulativeSlotTime": 25876.0d, "LastJobLeaseRenewal": 1446133562, "MachineAttrSlotWeight0": 1, "NumSystemHolds": 0, "BytesRecvd": 285054.0d, "CondorPlatform": "$CondorPlatform: X86_64-RedHat_6.6 $", "JOBGLIDEIN_ResourceName": "$$([IfThenElse(IsUndefined(TARGET.GLIDEIN_ResourceName), IfThenElse(IsUndefined(TARGET.GLIDEIN_Site), \"wisc.edu\", TARGET.GLIDEIN_Site), TARGET.GLIDEIN_ResourceName)])", "DAGNodeName": "333+333", "PeriodicRelease": "error", "JobRunCount": 1, "LastRemoteHost": "slot1@e320.chtc.wisc.edu", "JobPrio": 0, "LocalSysCpu": 0.0d, "ExecutableSize": 7, "RemoteSysCpu": 157.0d, "TransferInput": "/home/xguo23/finally_2/Simulation_condor/data/333/,/home/xguo23/finally_2/Simulation_condor/d
ata/shared/", "PeriodicHold": false, "WantRemoteIO": true, "CommittedSuspensionTime": 0, "DAGParentNodeNames": "", "CompletionDate": 1446133562, "StreamErr": false, "RecentBlockReadKbytes": 0, "WantRemoteSyscalls": false, "NumShadowStarts": 1, "MachineAttrCpus0": 1, "DAGManNodesLog": "/home/xguo23/finally_2/Simulation_condor/model_3/./mydag.dag.nodes.log", "Owner": "xguo23", "Requirements": "undefined", "DiskUsage": 1250000, "JobLeaseDuration": 2400, "ClusterId": 49582154, "BufferSize": 524288, "IsCHTCSubmit": true, "RecentStatsLifetimeStarter": 1200, "CommittedSlotTime": 25876.0d, "Args": "--type=Matlab --version=R2011b --cmdtorun=simu_condor --unique=333 -- 3", "Environment": "", "LastPublicClaimId": "<128.105.245.120:45185>#1443991409#14242#...", "Iwd": "/home/xguo23/finally_2/Simulation_condor/model_3/333", "QDate": 1446105553, "CurrentHosts": 0, "User": "xguo23@chtc.wisc.edu", "StreamOut": false }
-{ "GlobalJobId": "submit-3.chtc.wisc.edu#49582177.0#1446105581", "StatsLifetimeStarter": 25025, "JobStartDate": 1446108665, "SubmitEventNotes": "DAG Node: 145+145", "JobStatus": 4, "LeaveJobInQueue": false, "StartdPrincipal": "execute-side@matchsession/128.104.55.57", "WantRHEL6": true, "OnExitRemove": true, "JobCurrentStartDate": 1446108665, "CoreSize": 0, "MATCH_EXP_JOBGLIDEIN_ResourceName": "wisc.edu", "Rank": 0.0d, "ExitStatus": 0, "ResidentSetSize": 75000, "RemoteWallClockTime": 25026.0d, "WantCheckpoint": false, "In": "/dev/null", "MaxHosts": 1, "RootDir": "/", "NumRestarts": 0, "RecentBlockWriteKbytes": 4, "DiskUsage_RAW": 1216669, "EnteredCurrentStatus": 1446133691, "ResidentSetSize_RAW": 73308, "RequestDisk": 1000000, "MyType": "Job", "PeriodicRemove": false, "Cmd": "/home/xguo23/finally_2/Simulation_condor/chtcjobwrapper", "CondorVersion": "$CondorVersion: 8.5.0 Sep 16 2015 BuildID: 341710 $", "ShouldTransferFiles": "YES", "TargetType": "Machine", "MinHosts": 1, "NumCkpts_
RAW": 0, "RequestCpus": 1, "RemoteUserCpu": 24770.0d, "BlockWrites": 1, "NiceUser": false, "Out": "process.out", "ImageSize_RAW": 120972, "BytesSent": 28290.0d, "CumulativeSuspensionTime": 0, "TransferIn": false, "NumCkpts": 0, "Err": "process.err", "RecentBlockWrites": 1, "JobFinishedHookDone": 1446133691, "ProcId": 0, "ImageSize": 125000, "JobUniverse": 5, "EncryptExecuteDirectory": false, "TransferInputSizeMB": 0, "RecentBlockReads": 146, "SpooledOutputFiles": "CURLTIME_4179033,ChtcWrapper145.out,AuditLog.145,simu_3_145.txt,harvest.log,145.out", "BlockWriteKbytes": 4, "WhenToTransferOutput": "ON_EXIT", "JobCurrentStartExecutingDate": 1446108666, "ExitBySignal": false, "LastMatchTime": 1446108665, "OnExitHold": false, "OrigMaxHosts": 1, "RequestMemory": 1000, "NumJobStarts": 1, "TerminationPending": true, "TotalSuspensions": 0, "BlockReads": 796, "DAGManJobId": 49581933, "MemoryUsage": 73, "ExitCode": 0, "JobNotification": 0, "BlockReadKbytes": 28476, "NumJobMatches": 1, "LocalUse
rCpu": 0.0d, "LastJobStatus": 2, "BufferBlockSize": 32768, "CommittedTime": 25026, "ExecutableSize_RAW": 6, "LastSuspensionTime": 0, "Matlab": "R2011b", "UserLog": "/home/xguo23/finally_2/Simulation_condor/model_3/145/process.log", "DAGManNodesMask": "0,1,2,4,5,7,9,10,11,12,13,16,17,24,27", "CumulativeSlotTime": 25026.0d, "LastJobLeaseRenewal": 1446133691, "MachineAttrSlotWeight0": 1, "NumSystemHolds": 0, "BytesRecvd": 285054.0d, "CondorPlatform": "$CondorPlatform: X86_64-RedHat_6.6 $", "JOBGLIDEIN_ResourceName": "$$([IfThenElse(IsUndefined(TARGET.GLIDEIN_ResourceName), IfThenElse(IsUndefined(TARGET.GLIDEIN_Site), \"wisc.edu\", TARGET.GLIDEIN_Site), TARGET.GLIDEIN_ResourceName)])", "DAGNodeName": "145+145", "PeriodicRelease": "error", "JobRunCount": 1, "LastRemoteHost": "slot1@c038.chtc.wisc.edu", "JobPrio": 0, "LocalSysCpu": 0.0d, "ExecutableSize": 7, "RemoteSysCpu": 217.0d, "TransferInput": "/home/xguo23/finally_2/Simulation_condor/data/145/,/home/xguo23/finally_2/Simulation_condo
r/data/shared/", "PeriodicHold": false, "WantRemoteIO": true, "CommittedSuspensionTime": 0, "DAGParentNodeNames": "", "CompletionDate": 1446133691, "StreamErr": false, "RecentBlockReadKbytes": 1932, "WantRemoteSyscalls": false, "NumShadowStarts": 1, "MachineAttrCpus0": 1, "DAGManNodesLog": "/home/xguo23/finally_2/Simulation_condor/model_3/./mydag.dag.nodes.log", "Owner": "xguo23", "Requirements": "undefined", "DiskUsage": 1250000, "JobLeaseDuration": 2400, "ClusterId": 49582177, "BufferSize": 524288, "IsCHTCSubmit": true, "RecentStatsLifetimeStarter": 1200, "CommittedSlotTime": 25026.0d, "Args": "--type=Matlab --version=R2011b --cmdtorun=simu_condor --unique=145 -- 3", "Environment": "", "LastPublicClaimId": "<128.104.55.57:49793>#1445322694#1541#...", "Iwd": "/home/xguo23/finally_2/Simulation_condor/model_3/145", "QDate": 1446105581, "CurrentHosts": 0, "User": "xguo23@chtc.wisc.edu", "StreamOut": false }
-{ "GlobalJobId": "submit-3.chtc.wisc.edu#49582178.0#1446105581", "StatsLifetimeStarter": 24871, "JobStartDate": 1446108666, "SubmitEventNotes": "DAG Node: 154+154", "JobStatus": 4, "LeaveJobInQueue": false, "StartdPrincipal": "execute-side@matchsession/128.105.245.158", "WantRHEL6": true, "OnExitRemove": true, "JobCurrentStartDate": 1446108666, "CoreSize": 0, "MATCH_EXP_JOBGLIDEIN_ResourceName": "wisc.edu", "Rank": 0.0d, "ExitStatus": 0, "ResidentSetSize": 150000, "RemoteWallClockTime": 24874.0d, "WantCheckpoint": false, "In": "/dev/null", "MaxHosts": 1, "RootDir": "/", "NumRestarts": 0, "RecentBlockWriteKbytes": 0, "DiskUsage_RAW": 1216669, "EnteredCurrentStatus": 1446133540, "ResidentSetSize_RAW": 125792, "RequestDisk": 1000000, "MyType": "Job", "PeriodicRemove": false, "Cmd": "/home/xguo23/finally_2/Simulation_condor/chtcjobwrapper", "CondorVersion": "$CondorVersion: 8.5.0 Sep 16 2015 BuildID: 341710 $", "ShouldTransferFiles": "YES", "TargetType": "Machine", "MinHosts": 1, "NumCk
pts_RAW": 0, "RequestCpus": 1, "RemoteUserCpu": 24626.0d, "BlockWrites": 0, "NiceUser": false, "Out": "process.out", "ImageSize_RAW": 811948, "BytesSent": 30559.0d, "CumulativeSuspensionTime": 0, "TransferIn": false, "NumCkpts": 0, "Err": "process.err", "RecentBlockWrites": 0, "JobFinishedHookDone": 1446133540, "ProcId": 0, "ImageSize": 1000000, "JobUniverse": 5, "EncryptExecuteDirectory": false, "TransferInputSizeMB": 0, "RecentBlockReads": 0, "SpooledOutputFiles": "CURLTIME_1382128,ChtcWrapper154.out,AuditLog.154,simu_3_154.txt,harvest.log,154.out", "BlockWriteKbytes": 0, "WhenToTransferOutput": "ON_EXIT", "JobCurrentStartExecutingDate": 1446108668, "ExitBySignal": false, "LastMatchTime": 1446108666, "OnExitHold": false, "OrigMaxHosts": 1, "RequestMemory": 1000, "NumJobStarts": 1, "TerminationPending": true, "TotalSuspensions": 0, "BlockReads": 0, "DAGManJobId": 49581933, "MemoryUsage": 146, "ExitCode": 0, "JobNotification": 0, "BlockReadKbytes": 0, "NumJobMatches": 1, "LocalUserC
pu": 0.0d, "LastJobStatus": 2, "BufferBlockSize": 32768, "CommittedTime": 24874, "ExecutableSize_RAW": 6, "LastSuspensionTime": 0, "Matlab": "R2011b", "UserLog": "/home/xguo23/finally_2/Simulation_condor/model_3/154/process.log", "DAGManNodesMask": "0,1,2,4,5,7,9,10,11,12,13,16,17,24,27", "CumulativeSlotTime": 24874.0d, "LastJobLeaseRenewal": 1446133540, "MachineAttrSlotWeight0": 1, "NumSystemHolds": 0, "BytesRecvd": 285054.0d, "CondorPlatform": "$CondorPlatform: X86_64-RedHat_6.6 $", "JOBGLIDEIN_ResourceName": "$$([IfThenElse(IsUndefined(TARGET.GLIDEIN_ResourceName), IfThenElse(IsUndefined(TARGET.GLIDEIN_Site), \"wisc.edu\", TARGET.GLIDEIN_Site), TARGET.GLIDEIN_ResourceName)])", "DAGNodeName": "154+154", "PeriodicRelease": "error", "JobRunCount": 1, "LastRemoteHost": "slot1@e358.chtc.wisc.edu", "JobPrio": 0, "LocalSysCpu": 0.0d, "E
<TRUNCATED>
[08/13] incubator-asterixdb git commit: Improve Error Handling in
Local Directory Feeds
Posted by am...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketServerInputStreamProviderFactory.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketServerInputStreamProviderFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketServerInputStreamProviderFactory.java
deleted file mode 100644
index a301c1a..0000000
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketServerInputStreamProviderFactory.java
+++ /dev/null
@@ -1,141 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.external.input.stream.factory;
-
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.ServerSocket;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.Set;
-
-import org.apache.asterix.common.exceptions.AsterixException;
-import org.apache.asterix.external.api.IInputStreamProvider;
-import org.apache.asterix.external.api.IInputStreamProviderFactory;
-import org.apache.asterix.external.input.stream.provider.SocketServerInputStreamProvider;
-import org.apache.asterix.external.util.ExternalDataConstants;
-import org.apache.asterix.om.util.AsterixRuntimeUtil;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
-import org.apache.hyracks.algebricks.common.utils.Pair;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-
-public class SocketServerInputStreamProviderFactory implements IInputStreamProviderFactory {
-
- private static final long serialVersionUID = 1L;
- private List<Pair<String, Integer>> sockets;
- private Mode mode = Mode.IP;
-
- public static enum Mode {
- NC,
- IP
- }
-
- @Override
- public void configure(Map<String, String> configuration) throws AsterixException {
- try {
- sockets = new ArrayList<Pair<String, Integer>>();
- String modeValue = configuration.get(ExternalDataConstants.KEY_MODE);
- if (modeValue != null) {
- mode = Mode.valueOf(modeValue.trim().toUpperCase());
- }
- String socketsValue = configuration.get(ExternalDataConstants.KEY_SOCKETS);
- if (socketsValue == null) {
- throw new IllegalArgumentException(
- "\'sockets\' parameter not specified as part of adapter configuration");
- }
- Map<InetAddress, Set<String>> ncMap;
- ncMap = AsterixRuntimeUtil.getNodeControllerMap();
- List<String> ncs = AsterixRuntimeUtil.getAllNodeControllers();
- String[] socketsArray = socketsValue.split(",");
- Random random = new Random();
- for (String socket : socketsArray) {
- String[] socketTokens = socket.split(":");
- String host = socketTokens[0].trim();
- int port = Integer.parseInt(socketTokens[1].trim());
- Pair<String, Integer> p = null;
- switch (mode) {
- case IP:
- Set<String> ncsOnIp = ncMap.get(InetAddress.getByName(host));
- if ((ncsOnIp == null) || ncsOnIp.isEmpty()) {
- throw new IllegalArgumentException("Invalid host " + host
- + " as it is not part of the AsterixDB cluster. Valid choices are "
- + StringUtils.join(ncMap.keySet(), ", "));
- }
- String[] ncArray = ncsOnIp.toArray(new String[] {});
- String nc = ncArray[random.nextInt(ncArray.length)];
- p = new Pair<String, Integer>(nc, port);
- break;
-
- case NC:
- p = new Pair<String, Integer>(host, port);
- if (!ncs.contains(host)) {
- throw new IllegalArgumentException("Invalid NC " + host
- + " as it is not part of the AsterixDB cluster. Valid choices are "
- + StringUtils.join(ncs, ", "));
-
- }
- break;
- }
- sockets.add(p);
- }
- } catch (Exception e) {
- throw new AsterixException(e);
- }
- }
-
- @Override
- public synchronized IInputStreamProvider createInputStreamProvider(IHyracksTaskContext ctx, int partition)
- throws HyracksDataException {
- try {
- Pair<String, Integer> socket = sockets.get(partition);
- ServerSocket server;
- server = new ServerSocket(socket.second);
- return new SocketServerInputStreamProvider(server);
- } catch (IOException e) {
- throw new HyracksDataException(e);
- }
- }
-
- @Override
- public AlgebricksAbsolutePartitionConstraint getPartitionConstraint() {
- List<String> locations = new ArrayList<String>();
- for (Pair<String, Integer> socket : sockets) {
- locations.add(socket.first);
- }
- return new AlgebricksAbsolutePartitionConstraint(locations.toArray(new String[] {}));
- }
-
- public List<Pair<String, Integer>> getSockets() {
- return sockets;
- }
-
- @Override
- public DataSourceType getDataSourceType() {
- return DataSourceType.STREAM;
- }
-
- @Override
- public boolean isIndexible() {
- return false;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/TwitterFirehoseStreamFactory.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/TwitterFirehoseStreamFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/TwitterFirehoseStreamFactory.java
new file mode 100644
index 0000000..8052822
--- /dev/null
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/TwitterFirehoseStreamFactory.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.input.stream.factory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.asterix.external.api.AsterixInputStream;
+import org.apache.asterix.external.api.IInputStreamFactory;
+import org.apache.asterix.external.input.stream.TwitterFirehoseInputStream;
+import org.apache.asterix.om.util.AsterixClusterProperties;
+import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+/**
+ * Factory class for creating @see{TwitterFirehoseFeedAdapter}. The adapter
+ * simulates a twitter firehose with tweets being "pushed" into Asterix at a
+ * configurable rate measured in terms of TPS (tweets/second). The stream of
+ * tweets lasts for a configurable duration (measured in seconds).
+ */
+public class TwitterFirehoseStreamFactory implements IInputStreamFactory {
+
+ private static final long serialVersionUID = 1L;
+
+ /**
+ * Degree of parallelism for feed ingestion activity. Defaults to 1. This
+ * determines the count constraint for the ingestion operator.
+ **/
+ private static final String KEY_INGESTION_CARDINALITY = "ingestion-cardinality";
+
+ /**
+ * The absolute locations where ingestion operator instances will be placed.
+ **/
+ private static final String KEY_INGESTION_LOCATIONS = "ingestion-location";
+
+ private Map<String, String> configuration;
+
+ @Override
+ public AlgebricksAbsolutePartitionConstraint getPartitionConstraint() {
+ String ingestionCardinalityParam = configuration.get(KEY_INGESTION_CARDINALITY);
+ String ingestionLocationParam = configuration.get(KEY_INGESTION_LOCATIONS);
+ String[] locations = null;
+ if (ingestionLocationParam != null) {
+ locations = ingestionLocationParam.split(",");
+ }
+ int count = locations != null ? locations.length : 1;
+ if (ingestionCardinalityParam != null) {
+ count = Integer.parseInt(ingestionCardinalityParam);
+ }
+
+ List<String> chosenLocations = new ArrayList<String>();
+ String[] availableLocations = locations != null ? locations
+ : AsterixClusterProperties.INSTANCE.getParticipantNodes().toArray(new String[] {});
+ for (int i = 0, k = 0; i < count; i++, k = (k + 1) % availableLocations.length) {
+ chosenLocations.add(availableLocations[k]);
+ }
+ return new AlgebricksAbsolutePartitionConstraint(chosenLocations.toArray(new String[] {}));
+ }
+
+ @Override
+ public DataSourceType getDataSourceType() {
+ return DataSourceType.STREAM;
+ }
+
+ @Override
+ public void configure(Map<String, String> configuration) {
+ this.configuration = configuration;
+ }
+
+ @Override
+ public boolean isIndexible() {
+ return false;
+ }
+
+ @Override
+ public AsterixInputStream createInputStream(IHyracksTaskContext ctx, int partition) throws HyracksDataException {
+ try {
+ return new TwitterFirehoseInputStream(configuration, ctx, partition);
+ } catch (IOException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/TwitterFirehoseStreamProviderFactory.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/TwitterFirehoseStreamProviderFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/TwitterFirehoseStreamProviderFactory.java
deleted file mode 100644
index 7b09ade..0000000
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/TwitterFirehoseStreamProviderFactory.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.external.input.stream.factory;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.asterix.external.api.IInputStreamProvider;
-import org.apache.asterix.external.api.IInputStreamProviderFactory;
-import org.apache.asterix.external.input.stream.provider.TwitterFirehoseInputStreamProvider;
-import org.apache.asterix.om.util.AsterixClusterProperties;
-import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-
-/**
- * Factory class for creating @see{TwitterFirehoseFeedAdapter}. The adapter
- * simulates a twitter firehose with tweets being "pushed" into Asterix at a
- * configurable rate measured in terms of TPS (tweets/second). The stream of
- * tweets lasts for a configurable duration (measured in seconds).
- */
-public class TwitterFirehoseStreamProviderFactory implements IInputStreamProviderFactory {
-
- private static final long serialVersionUID = 1L;
-
- /**
- * Degree of parallelism for feed ingestion activity. Defaults to 1. This
- * determines the count constraint for the ingestion operator.
- **/
- private static final String KEY_INGESTION_CARDINALITY = "ingestion-cardinality";
-
- /**
- * The absolute locations where ingestion operator instances will be placed.
- **/
- private static final String KEY_INGESTION_LOCATIONS = "ingestion-location";
-
- private Map<String, String> configuration;
-
- @Override
- public AlgebricksAbsolutePartitionConstraint getPartitionConstraint() {
- String ingestionCardinalityParam = configuration.get(KEY_INGESTION_CARDINALITY);
- String ingestionLocationParam = configuration.get(KEY_INGESTION_LOCATIONS);
- String[] locations = null;
- if (ingestionLocationParam != null) {
- locations = ingestionLocationParam.split(",");
- }
- int count = locations != null ? locations.length : 1;
- if (ingestionCardinalityParam != null) {
- count = Integer.parseInt(ingestionCardinalityParam);
- }
-
- List<String> chosenLocations = new ArrayList<String>();
- String[] availableLocations = locations != null ? locations
- : AsterixClusterProperties.INSTANCE.getParticipantNodes().toArray(new String[] {});
- for (int i = 0, k = 0; i < count; i++, k = (k + 1) % availableLocations.length) {
- chosenLocations.add(availableLocations[k]);
- }
- return new AlgebricksAbsolutePartitionConstraint(chosenLocations.toArray(new String[] {}));
- }
-
- @Override
- public DataSourceType getDataSourceType() {
- return DataSourceType.STREAM;
- }
-
- @Override
- public void configure(Map<String, String> configuration) {
- this.configuration = configuration;
- }
-
- @Override
- public boolean isIndexible() {
- return false;
- }
-
- @Override
- public IInputStreamProvider createInputStreamProvider(IHyracksTaskContext ctx, int partition)
- throws HyracksDataException {
- return new TwitterFirehoseInputStreamProvider(configuration, ctx, partition);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/HDFSInputStreamProvider.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/HDFSInputStreamProvider.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/HDFSInputStreamProvider.java
deleted file mode 100644
index e1ab331..0000000
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/HDFSInputStreamProvider.java
+++ /dev/null
@@ -1,127 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.external.input.stream.provider;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.asterix.external.api.IInputStreamProvider;
-import org.apache.asterix.external.dataflow.AbstractFeedDataFlowController;
-import org.apache.asterix.external.indexing.ExternalFile;
-import org.apache.asterix.external.input.record.reader.hdfs.HDFSRecordReader;
-import org.apache.asterix.external.input.stream.AInputStream;
-import org.apache.asterix.external.provider.ExternalIndexerProvider;
-import org.apache.asterix.external.util.ExternalDataConstants;
-import org.apache.asterix.external.util.FeedLogManager;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.InputSplit;
-import org.apache.hadoop.mapred.JobConf;
-
-public class HDFSInputStreamProvider<K> extends HDFSRecordReader<K, Text> implements IInputStreamProvider {
-
- public HDFSInputStreamProvider(boolean read[], InputSplit[] inputSplits, String[] readSchedule, String nodeName,
- JobConf conf, Map<String, String> configuration, List<ExternalFile> snapshot) throws Exception {
- super(read, inputSplits, readSchedule, nodeName, conf, snapshot,
- snapshot == null ? null : ExternalIndexerProvider.getIndexer(configuration));
- value = new Text();
- if (snapshot != null) {
- if (currentSplitIndex < snapshot.size()) {
- indexer.reset(this);
- }
- }
- }
-
- @Override
- public AInputStream getInputStream() {
- return new HDFSInputStream();
- }
-
- private class HDFSInputStream extends AInputStream {
- int pos = 0;
-
- @Override
- public int read() throws IOException {
- if (value.getLength() < pos) {
- if (!readMore()) {
- return -1;
- }
- } else if (value.getLength() == pos) {
- pos++;
- return ExternalDataConstants.BYTE_LF;
- }
- return value.getBytes()[pos++];
- }
-
- private int readRecord(byte[] buffer, int offset, int len) {
- int actualLength = value.getLength() + 1;
- if ((actualLength - pos) > len) {
- //copy partial record
- System.arraycopy(value.getBytes(), pos, buffer, offset, len);
- pos += len;
- return len;
- } else {
- int numBytes = value.getLength() - pos;
- System.arraycopy(value.getBytes(), pos, buffer, offset, numBytes);
- buffer[offset + numBytes] = ExternalDataConstants.LF;
- pos += numBytes;
- numBytes++;
- return numBytes;
- }
- }
-
- @Override
- public int read(byte[] buffer, int offset, int len) throws IOException {
- if (value.getLength() > pos) {
- return readRecord(buffer, offset, len);
- }
- if (!readMore()) {
- return -1;
- }
- return readRecord(buffer, offset, len);
- }
-
- private boolean readMore() throws IOException {
- try {
- pos = 0;
- return HDFSInputStreamProvider.this.hasNext();
- } catch (Exception e) {
- throw new IOException(e);
- }
- }
-
- @Override
- public boolean skipError() throws Exception {
- return true;
- }
-
- @Override
- public boolean stop() throws Exception {
- return false;
- }
-
- @Override
- public void setFeedLogManager(FeedLogManager logManager) {
- }
-
- @Override
- public void setController(AbstractFeedDataFlowController controller) {
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/LocalFSInputStreamProvider.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/LocalFSInputStreamProvider.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/LocalFSInputStreamProvider.java
deleted file mode 100644
index fbe6035..0000000
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/LocalFSInputStreamProvider.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.external.input.stream.provider;
-
-import java.nio.file.Path;
-import java.util.Map;
-
-import org.apache.asterix.external.api.IInputStreamProvider;
-import org.apache.asterix.external.input.stream.AInputStream;
-import org.apache.asterix.external.input.stream.LocalFileSystemInputStream;
-import org.apache.asterix.external.util.FeedLogManager;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.dataflow.std.file.FileSplit;
-
-public class LocalFSInputStreamProvider implements IInputStreamProvider {
-
- private final String expression;
- private final boolean isFeed;
- private final Path path;
- private FeedLogManager feedLogManager;
-
- public LocalFSInputStreamProvider(final FileSplit[] fileSplits, final IHyracksTaskContext ctx,
- final Map<String, String> configuration, final int partition, final String expression,
- final boolean isFeed) {
- this.expression = expression;
- this.isFeed = isFeed;
- this.path = fileSplits[partition].getLocalFile().getFile().toPath();
- }
-
- @Override
- public AInputStream getInputStream() throws HyracksDataException {
- final LocalFileSystemInputStream stream = new LocalFileSystemInputStream(path, expression, isFeed);
- stream.setFeedLogManager(feedLogManager);
- return stream;
- }
-
- @Override
- public void setFeedLogManager(final FeedLogManager feedLogManager) {
- this.feedLogManager = feedLogManager;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/SocketClientInputStreamProvider.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/SocketClientInputStreamProvider.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/SocketClientInputStreamProvider.java
deleted file mode 100644
index f842638..0000000
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/SocketClientInputStreamProvider.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.external.input.stream.provider;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.Socket;
-
-import org.apache.asterix.external.api.IInputStreamProvider;
-import org.apache.asterix.external.dataflow.AbstractFeedDataFlowController;
-import org.apache.asterix.external.input.stream.AInputStream;
-import org.apache.asterix.external.util.FeedLogManager;
-import org.apache.hyracks.algebricks.common.utils.Pair;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.log4j.Logger;
-
-public class SocketClientInputStreamProvider implements IInputStreamProvider {
-
- private static final Logger LOGGER = Logger.getLogger(SocketClientInputStreamProvider.class.getName());
- private final Socket socket;
-
- public SocketClientInputStreamProvider(Pair<String, Integer> ipAndPort) throws HyracksDataException {
- try {
- socket = new Socket(ipAndPort.first, ipAndPort.second);
- } catch (IOException e) {
- LOGGER.error(
- "Problem in creating socket against host " + ipAndPort.first + " on the port " + ipAndPort.second,
- e);
- throw new HyracksDataException(e);
- }
- }
-
- @Override
- public AInputStream getInputStream() throws HyracksDataException {
- InputStream in;
- try {
- in = socket.getInputStream();
- } catch (IOException e) {
- throw new HyracksDataException(e);
- }
- return new AInputStream() {
- @Override
- public int read() throws IOException {
- throw new IOException("method not supported. use read(byte[] buffer, int offset, int length) instead");
- }
-
- @Override
- public int read(byte[] buffer, int offset, int length) throws IOException {
- return in.read(buffer, offset, length);
- }
-
- @Override
- public boolean stop() throws Exception {
- if (!socket.isClosed()) {
- try {
- in.close();
- } finally {
- socket.close();
- }
- }
- return true;
- }
-
- @Override
- public boolean skipError() throws Exception {
- return false;
- }
-
- @Override
- public void setFeedLogManager(FeedLogManager logManager) {
- }
-
- @Override
- public void setController(AbstractFeedDataFlowController controller) {
- }
- };
- }
-
- @Override
- public void setFeedLogManager(FeedLogManager feedLogManager) {
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/SocketServerInputStreamProvider.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/SocketServerInputStreamProvider.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/SocketServerInputStreamProvider.java
deleted file mode 100644
index 64f0342..0000000
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/SocketServerInputStreamProvider.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.external.input.stream.provider;
-
-import java.net.ServerSocket;
-
-import org.apache.asterix.external.api.IInputStreamProvider;
-import org.apache.asterix.external.input.stream.AInputStream;
-import org.apache.asterix.external.input.stream.SocketServerInputStream;
-import org.apache.asterix.external.util.FeedLogManager;
-
-public class SocketServerInputStreamProvider implements IInputStreamProvider {
- private final ServerSocket server;
-
- public SocketServerInputStreamProvider(ServerSocket server) {
- this.server = server;
- }
-
- @Override
- public AInputStream getInputStream() {
- return new SocketServerInputStream(server);
- }
-
- @Override
- public void setFeedLogManager(FeedLogManager feedLogManager) {
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/TwitterFirehoseInputStreamProvider.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/TwitterFirehoseInputStreamProvider.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/TwitterFirehoseInputStreamProvider.java
deleted file mode 100644
index a979262..0000000
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/provider/TwitterFirehoseInputStreamProvider.java
+++ /dev/null
@@ -1,202 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.external.input.stream.provider;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.io.PipedInputStream;
-import java.io.PipedOutputStream;
-import java.util.Map;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import org.apache.asterix.external.api.IInputStreamProvider;
-import org.apache.asterix.external.dataflow.AbstractFeedDataFlowController;
-import org.apache.asterix.external.input.stream.AInputStream;
-import org.apache.asterix.external.util.FeedLogManager;
-import org.apache.asterix.external.util.TweetGenerator;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-
-public class TwitterFirehoseInputStreamProvider implements IInputStreamProvider {
-
- private static final Logger LOGGER = Logger.getLogger(TwitterFirehoseInputStreamProvider.class.getName());
-
- private final ExecutorService executorService;
-
- private final PipedOutputStream outputStream;
-
- private final PipedInputStream inputStream;
-
- private final TwitterServer twitterServer;
-
- public TwitterFirehoseInputStreamProvider(Map<String, String> configuration, IHyracksTaskContext ctx, int partition)
- throws HyracksDataException {
- try {
- executorService = Executors.newCachedThreadPool();
- outputStream = new PipedOutputStream();
- inputStream = new PipedInputStream(outputStream);
- twitterServer = new TwitterServer(configuration, partition, outputStream, executorService, inputStream);
- } catch (IOException e) {
- throw new HyracksDataException(e);
- }
- }
-
- @Override
- public AInputStream getInputStream() {
- return twitterServer;
- }
-
- private static class TwitterServer extends AInputStream {
- private final DataProvider dataProvider;
- private final ExecutorService executorService;
- private final InputStream in;
- private boolean started;
-
- public TwitterServer(Map<String, String> configuration, int partition, OutputStream os,
- ExecutorService executorService, InputStream in) {
- dataProvider = new DataProvider(configuration, partition, os);
- this.executorService = executorService;
- this.in = in;
- this.started = false;
- }
-
- @Override
- public boolean stop() throws IOException {
- dataProvider.stop();
- return true;
- }
-
- public synchronized void start() {
- if (!started) {
- executorService.execute(dataProvider);
- started = true;
- }
- }
-
- @Override
- public boolean skipError() throws Exception {
- return false;
- }
-
- @Override
- public int read() throws IOException {
- if (!started) {
- start();
- }
- return in.read();
- }
-
- @Override
- public int read(byte b[], int off, int len) throws IOException {
- if (!started) {
- start();
- started = true;
- }
- return in.read(b, off, len);
- }
-
- @Override
- public void setFeedLogManager(FeedLogManager logManager) {
- }
-
- @Override
- public void setController(AbstractFeedDataFlowController controller) {
- }
- }
-
- private static class DataProvider implements Runnable {
-
- public static final String KEY_MODE = "mode";
-
- private final TweetGenerator tweetGenerator;
- private boolean continuePush = true;
- private int batchSize;
- private final Mode mode;
- private final OutputStream os;
-
- public static enum Mode {
- AGGRESSIVE,
- CONTROLLED
- }
-
- public DataProvider(Map<String, String> configuration, int partition, OutputStream os) {
- this.tweetGenerator = new TweetGenerator(configuration, partition);
- this.tweetGenerator.registerSubscriber(os);
- this.os = os;
- mode = configuration.get(KEY_MODE) != null ? Mode.valueOf(configuration.get(KEY_MODE).toUpperCase())
- : Mode.AGGRESSIVE;
- switch (mode) {
- case CONTROLLED:
- String tpsValue = configuration.get(TweetGenerator.KEY_TPS);
- if (tpsValue == null) {
- throw new IllegalArgumentException("TPS value not configured. use tps=<value>");
- }
- batchSize = Integer.parseInt(tpsValue);
- break;
- case AGGRESSIVE:
- batchSize = 5000;
- break;
- }
- }
-
- @Override
- public void run() {
- boolean moreData = true;
- long startBatch;
- long endBatch;
- while (true) {
- try {
- while (moreData && continuePush) {
- switch (mode) {
- case AGGRESSIVE:
- moreData = tweetGenerator.generateNextBatch(batchSize);
- break;
- case CONTROLLED:
- startBatch = System.currentTimeMillis();
- moreData = tweetGenerator.generateNextBatch(batchSize);
- endBatch = System.currentTimeMillis();
- if ((endBatch - startBatch) < 1000) {
- Thread.sleep(1000 - (endBatch - startBatch));
- }
- break;
- }
- }
- os.close();
- break;
- } catch (Exception e) {
- if (LOGGER.isLoggable(Level.WARNING)) {
- LOGGER.warning("Exception in adapter " + e.getMessage());
- }
- }
- }
- }
-
- public void stop() {
- continuePush = false;
- }
- }
-
- @Override
- public void setFeedLogManager(FeedLogManager feedLogManager) {
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedCollectOperatorDescriptor.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedCollectOperatorDescriptor.java b/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedCollectOperatorDescriptor.java
index 8475e45..9485b77 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedCollectOperatorDescriptor.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedCollectOperatorDescriptor.java
@@ -19,11 +19,9 @@
package org.apache.asterix.external.operators;
import java.util.Map;
-import java.util.logging.Level;
import java.util.logging.Logger;
import org.apache.asterix.common.api.IAsterixAppRuntimeContext;
-import org.apache.asterix.external.feed.api.IFeedLifecycleListener.ConnectionLocation;
import org.apache.asterix.external.feed.api.IFeedManager;
import org.apache.asterix.external.feed.api.IFeedRuntime.FeedRuntimeType;
import org.apache.asterix.external.feed.api.IFeedSubscriptionManager;
@@ -52,7 +50,7 @@ public class FeedCollectOperatorDescriptor extends AbstractSingleActivityOperato
private static final long serialVersionUID = 1L;
private static final Logger LOGGER = Logger.getLogger(FeedCollectOperatorDescriptor.class.getName());
- /** The type associated with the ADM data output from the feed adaptor */
+ /** The type associated with the ADM data output from (the feed adapter OR the compute operator) */
private final IAType outputType;
/** unique identifier for a feed instance. */
@@ -67,12 +65,12 @@ public class FeedCollectOperatorDescriptor extends AbstractSingleActivityOperato
/** The source feed from which the feed derives its data from. **/
private final FeedId sourceFeedId;
- /** The subscription location at which the recipient feed receives tuples from the source feed **/
- private final ConnectionLocation subscriptionLocation;
+ /** The subscription location at which the recipient feed receives tuples from the source feed {SOURCE_FEED_INTAKE_STAGE , SOURCE_FEED_COMPUTE_STAGE} **/
+ private final FeedRuntimeType subscriptionLocation;
public FeedCollectOperatorDescriptor(JobSpecification spec, FeedConnectionId feedConnectionId, FeedId sourceFeedId,
ARecordType atype, RecordDescriptor rDesc, Map<String, String> feedPolicyProperties,
- ConnectionLocation subscriptionLocation) {
+ FeedRuntimeType subscriptionLocation) {
super(spec, 0, 1);
recordDescriptors[0] = rDesc;
this.outputType = atype;
@@ -85,45 +83,25 @@ public class FeedCollectOperatorDescriptor extends AbstractSingleActivityOperato
@Override
public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
IRecordDescriptorProvider recordDescProvider, final int partition, int nPartitions)
- throws HyracksDataException {
+ throws HyracksDataException {
IAsterixAppRuntimeContext runtimeCtx = (IAsterixAppRuntimeContext) ctx.getJobletContext()
.getApplicationContext().getApplicationObject();
this.subscriptionManager = ((IFeedManager) runtimeCtx.getFeedManager()).getFeedSubscriptionManager();
ISubscribableRuntime sourceRuntime = null;
- IOperatorNodePushable nodePushable = null;
+ SubscribableFeedRuntimeId feedSubscribableRuntimeId = new SubscribableFeedRuntimeId(sourceFeedId,
+ subscriptionLocation, partition);
switch (subscriptionLocation) {
- case SOURCE_FEED_INTAKE_STAGE:
- try {
- SubscribableFeedRuntimeId feedSubscribableRuntimeId = new SubscribableFeedRuntimeId(sourceFeedId,
- FeedRuntimeType.INTAKE, partition);
- sourceRuntime = getIntakeRuntime(feedSubscribableRuntimeId);
- if (sourceRuntime == null) {
- throw new HyracksDataException(
- "Source intake task not found for source feed id " + sourceFeedId);
- }
- nodePushable = new FeedCollectOperatorNodePushable(ctx, sourceFeedId, connectionId,
- feedPolicyProperties, partition, nPartitions, sourceRuntime);
-
- } catch (Exception exception) {
- if (LOGGER.isLoggable(Level.SEVERE)) {
- LOGGER.severe("Initialization of the feed adaptor failed with exception " + exception);
- }
- throw new HyracksDataException("Initialization of the feed adapter failed", exception);
- }
+ case INTAKE:
+ sourceRuntime = getIntakeRuntime(feedSubscribableRuntimeId);
break;
- case SOURCE_FEED_COMPUTE_STAGE:
- SubscribableFeedRuntimeId feedSubscribableRuntimeId = new SubscribableFeedRuntimeId(sourceFeedId,
- FeedRuntimeType.COMPUTE, partition);
+ case COMPUTE:
sourceRuntime = subscriptionManager.getSubscribableRuntime(feedSubscribableRuntimeId);
- if (sourceRuntime == null) {
- throw new HyracksDataException("Source compute task not found for source feed id " + sourceFeedId
- + " " + FeedRuntimeType.COMPUTE + "[" + partition + "]");
- }
- nodePushable = new FeedCollectOperatorNodePushable(ctx, sourceFeedId, connectionId,
- feedPolicyProperties, partition, nPartitions, sourceRuntime);
break;
+ default:
+ throw new HyracksDataException("Can't subscirbe to FeedRuntime with Type: " + subscriptionLocation);
}
- return nodePushable;
+ return new FeedCollectOperatorNodePushable(ctx, sourceFeedId, connectionId, feedPolicyProperties, partition,
+ nPartitions, sourceRuntime);
}
public FeedConnectionId getFeedConnectionId() {
@@ -150,7 +128,7 @@ public class FeedCollectOperatorDescriptor extends AbstractSingleActivityOperato
return (IngestionRuntime) subscriptionManager.getSubscribableRuntime(subscribableRuntimeId);
}
- public ConnectionLocation getSubscriptionLocation() {
+ public FeedRuntimeType getSubscriptionLocation() {
return subscriptionLocation;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedCollectOperatorNodePushable.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedCollectOperatorNodePushable.java b/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedCollectOperatorNodePushable.java
index 7901f03..87e1edb 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedCollectOperatorNodePushable.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedCollectOperatorNodePushable.java
@@ -46,10 +46,9 @@ import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
import org.apache.hyracks.dataflow.std.base.AbstractUnaryOutputSourceOperatorNodePushable;
/**
- * The runtime for @see{FeedIntakeOperationDescriptor}
+ * The first operator in a collect job in a feed.
*/
public class FeedCollectOperatorNodePushable extends AbstractUnaryOutputSourceOperatorNodePushable {
-
private static Logger LOGGER = Logger.getLogger(FeedCollectOperatorNodePushable.class.getName());
private final int partition;
@@ -74,19 +73,16 @@ public class FeedCollectOperatorNodePushable extends AbstractUnaryOutputSourceOp
this.connectionId = feedConnectionId;
this.sourceRuntime = sourceRuntime;
this.feedPolicy = feedPolicy;
- policyAccessor = new FeedPolicyAccessor(feedPolicy);
- IAsterixAppRuntimeContext runtimeCtx = (IAsterixAppRuntimeContext) ctx.getJobletContext()
- .getApplicationContext().getApplicationObject();
- this.feedManager = (IFeedManager) runtimeCtx.getFeedManager();
+ this.policyAccessor = new FeedPolicyAccessor(feedPolicy);
+ this.feedManager = (IFeedManager) ((IAsterixAppRuntimeContext) ctx.getJobletContext().getApplicationContext()
+ .getApplicationObject()).getFeedManager();
}
@Override
public void initialize() throws HyracksDataException {
try {
outputRecordDescriptor = recordDesc;
- FeedRuntimeType sourceRuntimeType = ((SubscribableFeedRuntimeId) sourceRuntime.getRuntimeId())
- .getFeedRuntimeType();
- switch (sourceRuntimeType) {
+ switch (((SubscribableFeedRuntimeId) sourceRuntime.getRuntimeId()).getFeedRuntimeType()) {
case INTAKE:
handleCompleteConnection();
break;
@@ -94,7 +90,8 @@ public class FeedCollectOperatorNodePushable extends AbstractUnaryOutputSourceOp
handlePartialConnection();
break;
default:
- throw new IllegalStateException("Invalid source type " + sourceRuntimeType);
+ throw new IllegalStateException("Invalid source type "
+ + ((SubscribableFeedRuntimeId) sourceRuntime.getRuntimeId()).getFeedRuntimeType());
}
State state = collectRuntime.waitTillCollectionOver();
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/parser/AbstractDataParser.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/parser/AbstractDataParser.java b/asterix-external-data/src/main/java/org/apache/asterix/external/parser/AbstractDataParser.java
index cfd1b9c..2517166 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/parser/AbstractDataParser.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/parser/AbstractDataParser.java
@@ -114,7 +114,7 @@ public abstract class AbstractDataParser implements IDataParser {
.getSerializerDeserializer(BuiltinType.ADOUBLE);
@SuppressWarnings("unchecked")
protected ISerializerDeserializer<AString> stringSerde = AqlSerializerDeserializerProvider.INSTANCE
- .getSerializerDeserializer(BuiltinType.ASTRING);
+ .getAStringSerializerDeserializer();
@SuppressWarnings("unchecked")
protected ISerializerDeserializer<ABinary> binarySerde = AqlSerializerDeserializerProvider.INSTANCE
.getSerializerDeserializer(BuiltinType.ABINARY);
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/provider/DataflowControllerProvider.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/provider/DataflowControllerProvider.java b/asterix-external-data/src/main/java/org/apache/asterix/external/provider/DataflowControllerProvider.java
index 159ea73..d362201 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/provider/DataflowControllerProvider.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/provider/DataflowControllerProvider.java
@@ -22,12 +22,12 @@ import java.io.IOException;
import java.util.Map;
import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.external.api.AsterixInputStream;
import org.apache.asterix.external.api.IDataFlowController;
import org.apache.asterix.external.api.IDataParserFactory;
import org.apache.asterix.external.api.IExternalDataSourceFactory;
import org.apache.asterix.external.api.IIndexingDatasource;
-import org.apache.asterix.external.api.IInputStreamProvider;
-import org.apache.asterix.external.api.IInputStreamProviderFactory;
+import org.apache.asterix.external.api.IInputStreamFactory;
import org.apache.asterix.external.api.IRecordDataParser;
import org.apache.asterix.external.api.IRecordDataParserFactory;
import org.apache.asterix.external.api.IRecordReader;
@@ -44,7 +44,6 @@ import org.apache.asterix.external.dataflow.FeedWithMetaDataFlowController;
import org.apache.asterix.external.dataflow.IndexingDataFlowController;
import org.apache.asterix.external.dataflow.RecordDataFlowController;
import org.apache.asterix.external.dataflow.StreamDataFlowController;
-import org.apache.asterix.external.input.stream.AInputStream;
import org.apache.asterix.external.parser.RecordWithMetadataParser;
import org.apache.asterix.external.util.DataflowUtils;
import org.apache.asterix.external.util.ExternalDataUtils;
@@ -106,17 +105,15 @@ public class DataflowControllerProvider {
recordReader, 1);
}
case STREAM:
- IInputStreamProviderFactory streamProviderFactory = (IInputStreamProviderFactory) dataSourceFactory;
- IInputStreamProvider streamProvider = streamProviderFactory.createInputStreamProvider(ctx,
- partition);
+ IInputStreamFactory streamFactory = (IInputStreamFactory) dataSourceFactory;
+ AsterixInputStream stream = streamFactory.createInputStream(ctx, partition);
IStreamDataParserFactory streamParserFactory = (IStreamDataParserFactory) dataParserFactory;
IStreamDataParser streamParser = streamParserFactory.createInputStreamParser(ctx, partition);
- AInputStream inputStream = streamProvider.getInputStream();
- streamParser.setInputStream(inputStream);
+ streamParser.setInputStream(stream);
if (isFeed) {
return new FeedStreamDataFlowController(ctx,
(FeedTupleForwarder) DataflowUtils.getTupleForwarder(configuration, feedLogManager),
- feedLogManager, FeedUtils.getNumOfFields(configuration), streamParser, inputStream);
+ feedLogManager, FeedUtils.getNumOfFields(configuration), streamParser, stream);
} else {
return new StreamDataFlowController(ctx, DataflowUtils.getTupleForwarder(configuration, null),
streamParser);
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/provider/DatasourceFactoryProvider.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/provider/DatasourceFactoryProvider.java b/asterix-external-data/src/main/java/org/apache/asterix/external/provider/DatasourceFactoryProvider.java
index e9307e5..f8d64e0 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/provider/DatasourceFactoryProvider.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/provider/DatasourceFactoryProvider.java
@@ -23,7 +23,7 @@ import java.util.Map;
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.external.api.IExternalDataSourceFactory;
import org.apache.asterix.external.api.IExternalDataSourceFactory.DataSourceType;
-import org.apache.asterix.external.api.IInputStreamProviderFactory;
+import org.apache.asterix.external.api.IInputStreamFactory;
import org.apache.asterix.external.api.IRecordReaderFactory;
import org.apache.asterix.external.input.HDFSDataSourceFactory;
import org.apache.asterix.external.input.record.reader.RecordWithPKTestReaderFactory;
@@ -33,9 +33,9 @@ import org.apache.asterix.external.input.record.reader.stream.EmptyLineSeparated
import org.apache.asterix.external.input.record.reader.stream.LineRecordReaderFactory;
import org.apache.asterix.external.input.record.reader.stream.SemiStructuredRecordReaderFactory;
import org.apache.asterix.external.input.record.reader.twitter.TwitterRecordReaderFactory;
-import org.apache.asterix.external.input.stream.factory.LocalFSInputStreamProviderFactory;
-import org.apache.asterix.external.input.stream.factory.SocketServerInputStreamProviderFactory;
-import org.apache.asterix.external.input.stream.factory.TwitterFirehoseStreamProviderFactory;
+import org.apache.asterix.external.input.stream.factory.LocalFSInputStreamFactory;
+import org.apache.asterix.external.input.stream.factory.SocketServerInputStreamFactory;
+import org.apache.asterix.external.input.stream.factory.TwitterFirehoseStreamFactory;
import org.apache.asterix.external.util.ExternalDataConstants;
import org.apache.asterix.external.util.ExternalDataUtils;
@@ -53,9 +53,9 @@ public class DatasourceFactoryProvider {
}
}
- public static IInputStreamProviderFactory getInputStreamFactory(String streamSource,
+ public static IInputStreamFactory getInputStreamFactory(String streamSource,
Map<String, String> configuration) throws AsterixException {
- IInputStreamProviderFactory streamSourceFactory;
+ IInputStreamFactory streamSourceFactory;
if (ExternalDataUtils.isExternal(streamSource)) {
String dataverse = ExternalDataUtils.getDataverse(configuration);
streamSourceFactory = ExternalDataUtils.createExternalInputStreamFactory(dataverse, streamSource);
@@ -65,17 +65,17 @@ public class DatasourceFactoryProvider {
streamSourceFactory = new HDFSDataSourceFactory();
break;
case ExternalDataConstants.STREAM_LOCAL_FILESYSTEM:
- streamSourceFactory = new LocalFSInputStreamProviderFactory();
+ streamSourceFactory = new LocalFSInputStreamFactory();
break;
case ExternalDataConstants.STREAM_SOCKET:
case ExternalDataConstants.ALIAS_SOCKET_ADAPTER:
- streamSourceFactory = new SocketServerInputStreamProviderFactory();
+ streamSourceFactory = new SocketServerInputStreamFactory();
break;
case ExternalDataConstants.STREAM_SOCKET_CLIENT:
- streamSourceFactory = new SocketServerInputStreamProviderFactory();
+ streamSourceFactory = new SocketServerInputStreamFactory();
break;
case ExternalDataConstants.ALIAS_TWITTER_FIREHOSE_ADAPTER:
- streamSourceFactory = new TwitterFirehoseStreamProviderFactory();
+ streamSourceFactory = new TwitterFirehoseStreamFactory();
break;
default:
throw new AsterixException("unknown input stream factory");
@@ -90,7 +90,7 @@ public class DatasourceFactoryProvider {
return ExternalDataUtils.createExternalRecordReaderFactory(configuration);
}
String parser = configuration.get(ExternalDataConstants.KEY_PARSER);
- IInputStreamProviderFactory inputStreamFactory;
+ IInputStreamFactory inputStreamFactory;
switch (parser) {
case ExternalDataConstants.FORMAT_ADM:
case ExternalDataConstants.FORMAT_JSON:
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java b/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
index 0e68698..a02152b 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
@@ -206,8 +206,9 @@ public class ExternalDataConstants {
* Size default values
*/
public static final int DEFAULT_BUFFER_SIZE = 4096;
- public static final int DEFAULT_BUFFER_INCREMENT = 4096;
+ public static final int DEFAULT_BUFFER_INCREMENT = 2048;
public static final int DEFAULT_QUEUE_SIZE = 64;
+ public static final int MAX_RECORD_SIZE = 32000000;
/**
* Expected parameter values
@@ -226,4 +227,6 @@ public class ExternalDataConstants {
public static final String FORMAT_CSV = "csv";
public static final String TEST_RECORD_WITH_PK = "test-record-with-pk";
+ public static final String ERROR_LARGE_RECORD = "Record is too large";
+ public static final String ERROR_PARSE_RECORD = "Parser failed to parse record";
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java b/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
index 32139f1..42fe8bf 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
@@ -25,7 +25,7 @@ import java.util.Map;
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.external.api.IDataParserFactory;
import org.apache.asterix.external.api.IExternalDataSourceFactory.DataSourceType;
-import org.apache.asterix.external.api.IInputStreamProviderFactory;
+import org.apache.asterix.external.api.IInputStreamFactory;
import org.apache.asterix.external.api.IRecordReaderFactory;
import org.apache.asterix.external.library.ExternalLibraryManager;
import org.apache.asterix.om.types.ARecordType;
@@ -116,13 +116,13 @@ public class ExternalDataUtils {
return aString.trim().split(FeedConstants.NamingConstants.LIBRARY_NAME_SEPARATOR)[1];
}
- public static IInputStreamProviderFactory createExternalInputStreamFactory(String dataverse, String stream)
+ public static IInputStreamFactory createExternalInputStreamFactory(String dataverse, String stream)
throws AsterixException {
try {
String libraryName = getLibraryName(stream);
String className = getExternalClassName(stream);
ClassLoader classLoader = getClassLoader(dataverse, libraryName);
- return ((IInputStreamProviderFactory) (classLoader.loadClass(className).newInstance()));
+ return ((IInputStreamFactory) (classLoader.loadClass(className).newInstance()));
} catch (InstantiationException | IllegalAccessException | ClassNotFoundException e) {
throw new AsterixException("Failed to create stream factory", e);
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/util/FileSystemWatcher.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/util/FileSystemWatcher.java b/asterix-external-data/src/main/java/org/apache/asterix/external/util/FileSystemWatcher.java
index 386a8cf..4eec348 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/util/FileSystemWatcher.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/util/FileSystemWatcher.java
@@ -42,7 +42,7 @@ import org.apache.log4j.Logger;
public class FileSystemWatcher {
private static final Logger LOGGER = Logger.getLogger(FileSystemWatcher.class.getName());
- private final WatchService watcher;
+ private WatchService watcher;
private final HashMap<WatchKey, Path> keys;
private final LinkedList<File> files = new LinkedList<File>();
private Iterator<File> it;
@@ -53,17 +53,14 @@ public class FileSystemWatcher {
private boolean done;
private File current;
private AbstractFeedDataFlowController controller;
+ private final LinkedList<Path> dirs;
- public FileSystemWatcher(Path inputResource, String expression, boolean isFeed) throws HyracksDataException {
- try {
- this.watcher = isFeed ? FileSystems.getDefault().newWatchService() : null;
- this.keys = isFeed ? new HashMap<WatchKey, Path>() : null;
- this.expression = expression;
- this.path = inputResource;
- this.isFeed = isFeed;
- } catch (IOException e) {
- throw new HyracksDataException(e);
- }
+ public FileSystemWatcher(Path inputResource, String expression, boolean isFeed) {
+ this.keys = isFeed ? new HashMap<WatchKey, Path>() : null;
+ this.expression = expression;
+ this.path = inputResource;
+ this.isFeed = isFeed;
+ this.dirs = new LinkedList<Path>();
}
public void setFeedLogManager(FeedLogManager feedLogManager) {
@@ -72,11 +69,19 @@ public class FileSystemWatcher {
public void init() throws HyracksDataException {
try {
- LinkedList<Path> dirs = null;
- dirs = new LinkedList<Path>();
+ dirs.clear();
LocalFileSystemUtils.traverse(files, path.toFile(), expression, dirs);
it = files.iterator();
if (isFeed) {
+ keys.clear();
+ if (watcher != null) {
+ try {
+ watcher.close();
+ } catch (IOException e) {
+ LOGGER.warn("Failed to close watcher service", e);
+ }
+ }
+ watcher = FileSystems.getDefault().newWatchService();
for (Path path : dirs) {
register(path);
}
@@ -125,7 +130,7 @@ public class FileSystemWatcher {
return (WatchEvent<T>) event;
}
- private void handleEvents(WatchKey key) {
+ private void handleEvents(WatchKey key) throws IOException {
// get dir associated with the key
Path dir = keys.get(key);
if (dir == null) {
@@ -143,7 +148,10 @@ public class FileSystemWatcher {
if (LOGGER.isEnabledFor(Level.WARN)) {
LOGGER.warn("Overflow event. Some events might have been missed");
}
- continue;
+ // need to read and validate all files.
+ //TODO: use btrees for all logs
+ init();
+ return;
}
// Context for directory entry event is the file name of entry
@@ -172,6 +180,7 @@ public class FileSystemWatcher {
if (!done) {
if (watcher != null) {
watcher.close();
+ watcher = null;
}
if (logManager != null) {
if (current != null) {
@@ -205,7 +214,7 @@ public class FileSystemWatcher {
return false;
}
- public boolean hasNext() throws HyracksDataException {
+ public boolean hasNext() throws IOException {
if (it.hasNext()) {
return true;
}
@@ -222,6 +231,7 @@ public class FileSystemWatcher {
while (key != null) {
handleEvents(key);
if (endOfEvents(key)) {
+ close();
return false;
}
key = watcher.poll();
@@ -237,12 +247,18 @@ public class FileSystemWatcher {
if (LOGGER.isEnabledFor(Level.WARN)) {
LOGGER.warn("Feed Closed");
}
- return false;
+ if (watcher == null) {
+ return false;
+ }
+ continue;
} catch (ClosedWatchServiceException e) {
if (LOGGER.isEnabledFor(Level.WARN)) {
LOGGER.warn("The watcher has exited");
}
- return false;
+ if (watcher == null) {
+ return false;
+ }
+ continue;
}
handleEvents(key);
if (endOfEvents(key)) {
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/util/HDFSUtils.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/util/HDFSUtils.java b/asterix-external-data/src/main/java/org/apache/asterix/external/util/HDFSUtils.java
index 84ccc35..936ada5 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/util/HDFSUtils.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/util/HDFSUtils.java
@@ -28,7 +28,7 @@ import org.apache.asterix.common.config.DatasetConfig.ExternalFilePendingOp;
import org.apache.asterix.external.indexing.ExternalFile;
import org.apache.asterix.external.indexing.IndexingScheduler;
import org.apache.asterix.external.indexing.RecordId.RecordIdType;
-import org.apache.asterix.external.input.stream.provider.HDFSInputStreamProvider;
+import org.apache.asterix.external.input.stream.HDFSInputStream;
import org.apache.asterix.om.util.AsterixAppContextInfo;
import org.apache.asterix.om.util.AsterixClusterProperties;
import org.apache.hadoop.fs.BlockLocation;
@@ -186,7 +186,7 @@ public class HDFSUtils {
conf.set(ExternalDataConstants.KEY_HADOOP_FILESYSTEM_URI,
configuration.get(ExternalDataConstants.KEY_HDFS_URL).trim());
conf.set(ExternalDataConstants.KEY_HADOOP_FILESYSTEM_CLASS, ExternalDataConstants.CLASS_NAME_HDFS_FILESYSTEM);
- conf.setClassLoader(HDFSInputStreamProvider.class.getClassLoader());
+ conf.setClassLoader(HDFSInputStream.class.getClassLoader());
conf.set(ExternalDataConstants.KEY_HADOOP_INPUT_DIR, configuration.get(ExternalDataConstants.KEY_PATH).trim());
conf.set(ExternalDataConstants.KEY_HADOOP_INPUT_FORMAT, formatClassName);
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/AttributeReference.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/AttributeReference.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/AttributeReference.java
index 04fe6ec..da2cebd 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/AttributeReference.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/AttributeReference.java
@@ -18,41 +18,42 @@
*/
package org.apache.asterix.external.classad;
+import org.apache.asterix.external.classad.object.pool.ClassAdObjectPool;
import org.apache.asterix.om.base.AMutableInt32;
import org.apache.commons.lang3.mutable.MutableBoolean;
import org.apache.hyracks.api.exceptions.HyracksDataException;
public class AttributeReference extends ExprTree {
- private ExprTree expr;
+ private final ExprTreeHolder expr;
private boolean absolute;
- private AMutableCharArrayString attributeStr;
- private ClassAd current = new ClassAd(false, false);
- private ExprList adList = new ExprList();
- private Value val = new Value();
- private MutableBoolean rVal = new MutableBoolean(false);
+ private final AMutableCharArrayString attributeStr;
+ private final ExprList adList;
+ private final Value val;
private AttributeReference tempAttrRef;
- private EvalState tstate = new EvalState();
+ private final EvalState tstate;
+ private MutableBoolean rVal = new MutableBoolean(false);
+ private ClassAd current;
public ExprTree getExpr() {
return expr;
}
public void setExpr(ExprTree expr) {
- this.expr = expr == null ? null : expr.self();
+ this.expr.setInnerTree(expr.self());
}
- public AttributeReference() {
- expr = null;
- attributeStr = null;
+ public AttributeReference(ClassAdObjectPool objectPool) {
+ super(objectPool);
+ this.val = new Value(objectPool);
+ this.current = new ClassAd(this.objectPool);
+ this.tstate = new EvalState(this.objectPool);
+ this.adList = new ExprList(this.objectPool);
+ this.attributeStr = new AMutableCharArrayString();
+ this.expr = new ExprTreeHolder(objectPool);
absolute = false;
}
- /// Copy Constructor
- public AttributeReference(AttributeReference ref) throws HyracksDataException {
- copyFrom(ref);
- }
-
/// Assignment operator
@Override
public boolean equals(Object o) {
@@ -69,8 +70,9 @@ public class AttributeReference extends ExprTree {
return NodeKind.ATTRREF_NODE;
}
- public static AttributeReference createAttributeReference(ExprTree expr, AMutableCharArrayString attrName) {
- return createAttributeReference(expr, attrName, false);
+ public static AttributeReference createAttributeReference(ExprTree expr, AMutableCharArrayString attrName,
+ ClassAdObjectPool objectPool) {
+ return createAttributeReference(expr, attrName, false, objectPool);
}
/**
@@ -80,7 +82,7 @@ public class AttributeReference extends ExprTree {
*/
@Override
public ExprTree copy() throws HyracksDataException {
- AttributeReference newTree = new AttributeReference();
+ AttributeReference newTree = objectPool.attrRefPool.get();
newTree.copyFrom(this);
return newTree;
}
@@ -94,14 +96,8 @@ public class AttributeReference extends ExprTree {
* @throws HyracksDataException
*/
public boolean copyFrom(AttributeReference ref) throws HyracksDataException {
- if (attributeStr == null) {
- attributeStr = new AMutableCharArrayString(ref.attributeStr);
- } else {
- attributeStr.setValue(ref.attributeStr);
- }
- if (ref.expr != null) {
- expr = ref.expr.copy();
- }
+ attributeStr.setValue(ref.attributeStr);
+ expr.setInnerTree(ref.expr);
super.copyFrom(ref);
this.absolute = ref.absolute;
return true;
@@ -127,7 +123,7 @@ public class AttributeReference extends ExprTree {
if (absolute != other_ref.absolute || !attributeStr.equals(other_ref.attributeStr)) {
is_same = false;
} else if ((expr == null && other_ref.expr == null) || (expr.equals(other_ref.expr))
- || (expr != null && other_ref.expr != null && ((AttributeReference) expr).sameAs(other_ref.expr))) {
+ || (expr != null && other_ref.expr != null && expr.sameAs(other_ref.expr))) {
// Will this check result in infinite recursion? How do I stop it?
is_same = true;
} else {
@@ -137,18 +133,9 @@ public class AttributeReference extends ExprTree {
return is_same;
}
- // a private ctor for use in significant expr identification
- private AttributeReference(ExprTree tree, AMutableCharArrayString attrname, boolean absolut) {
- attributeStr = attrname;
- expr = tree == null ? null : tree.self();
- absolute = absolut;
- }
-
@Override
public void privateSetParentScope(ClassAd parent) {
- if (expr != null) {
- expr.setParentScope(parent);
- }
+ expr.setParentScope(parent);
}
public void getComponents(ExprTreeHolder tree, AMutableCharArrayString attr, MutableBoolean abs)
@@ -161,7 +148,7 @@ public class AttributeReference extends ExprTree {
public EvalResult findExpr(EvalState state, ExprTreeHolder tree, ExprTreeHolder sig, boolean wantSig)
throws HyracksDataException {
// establish starting point for search
- if (expr == null) {
+ if (expr.getTree() == null) {
// "attr" and ".attr"
current = absolute ? state.getRootAd() : state.getCurAd();
if (absolute && (current == null)) { // NAC - circularity so no root
@@ -186,7 +173,7 @@ public class AttributeReference extends ExprTree {
}
if (val.isListValue()) {
- ExprList eList = new ExprList();
+ ExprList eList = objectPool.exprListPool.get();
//
// iterate through exprList and apply attribute reference
// to each exprTree
@@ -195,12 +182,12 @@ public class AttributeReference extends ExprTree {
return (EvalResult.EVAL_FAIL);
} else {
if (tempAttrRef == null) {
- tempAttrRef = new AttributeReference();
+ tempAttrRef = objectPool.attrRefPool.get();
} else {
tempAttrRef.reset();
}
createAttributeReference(currExpr.copy(), attributeStr, false, tempAttrRef);
- val.clear();
+ val.setUndefinedValue();
// Create new EvalState, within this scope, because
// attrRef is only temporary, so we do not want to
// cache the evaluated result in the outer state object.
@@ -212,8 +199,8 @@ public class AttributeReference extends ExprTree {
return (EvalResult.EVAL_FAIL);
}
- ClassAd evaledAd = new ClassAd();
- ExprList evaledList = new ExprList();
+ ClassAd evaledAd = objectPool.classAdPool.get();
+ ExprList evaledList = objectPool.exprListPool.get();
if (val.isClassAdValue(evaledAd)) {
eList.add(evaledAd);
continue;
@@ -221,12 +208,13 @@ public class AttributeReference extends ExprTree {
eList.add(evaledList.copy());
continue;
} else {
- eList.add(Literal.createLiteral(val));
+ eList.add(Literal.createLiteral(val, objectPool));
}
}
}
- tree.setInnerTree(ExprList.createExprList(eList));
- ClassAd newRoot = new ClassAd();
+
+ tree.setInnerTree(ExprList.createExprList(eList, objectPool));
+ ClassAd newRoot = objectPool.classAdPool.get();
tree.setParentScope(newRoot);
return EvalResult.EVAL_OK;
}
@@ -242,8 +230,15 @@ public class AttributeReference extends ExprTree {
if (current == null) {
return EvalResult.EVAL_UNDEF;
}
- int rc = current.lookupInScope(attributeStr.toString(), tree, state);
- if (expr == null && !absolute && rc == EvalResult.EVAL_UNDEF.ordinal() && current.getAlternateScope() != null) {
+ int rc = 0;
+ try {
+ rc = current.lookupInScope(attributeStr.toString(), tree, state);
+ } catch (Throwable th) {
+ th.printStackTrace();
+ throw th;
+ }
+ if (expr.getTree() == null && !absolute && rc == EvalResult.EVAL_UNDEF.ordinal()
+ && current.getAlternateScope() != null) {
rc = current.getAlternateScope().lookupInScope(attributeStr.toString(), tree, state);
}
return EvalResult.values()[rc];
@@ -251,9 +246,10 @@ public class AttributeReference extends ExprTree {
@Override
public boolean publicEvaluate(EvalState state, Value val) throws HyracksDataException {
- ExprTreeHolder tree = new ExprTreeHolder();
- ExprTreeHolder dummy = new ExprTreeHolder();
- ClassAd curAd = new ClassAd(state.getCurAd());
+ ExprTreeHolder tree = objectPool.mutableExprPool.get();
+ ExprTreeHolder dummy = objectPool.mutableExprPool.get();
+ ClassAd curAd = objectPool.classAdPool.get();
+ curAd.copyFrom(state.getCurAd());
boolean rval;
// find the expression and the evalstate
switch (findExpr(state, tree, dummy, false)) {
@@ -286,10 +282,13 @@ public class AttributeReference extends ExprTree {
@Override
public boolean privateEvaluate(EvalState state, Value val, ExprTreeHolder sig) throws HyracksDataException {
- ExprTreeHolder tree = new ExprTreeHolder();
- ExprTreeHolder exprSig = new ExprTreeHolder();
- ClassAd curAd = new ClassAd(state.getCurAd());
- MutableBoolean rval = new MutableBoolean(true);
+ ExprTreeHolder tree = objectPool.mutableExprPool.get();
+ ExprTreeHolder exprSig = objectPool.mutableExprPool.get();
+ ClassAd curAd = objectPool.classAdPool.get();
+ curAd.copyFrom(state.getCurAd());
+ MutableBoolean rval = objectPool.boolPool.get();
+ rval.setValue(true);
+
switch (findExpr(state, tree, exprSig, true)) {
case EVAL_FAIL:
rval.setValue(false);
@@ -314,7 +313,9 @@ public class AttributeReference extends ExprTree {
default:
throw new HyracksDataException("ClassAd: Should not reach here");
}
- sig.setInnerTree((new AttributeReference(exprSig, attributeStr, absolute)));
+ AttributeReference newAttrRef = objectPool.attrRefPool.get();
+ newAttrRef.setValue(exprSig, attributeStr, absolute);
+ sig.setInnerTree(newAttrRef);
state.getCurAd().setValue(curAd);
return rval.booleanValue();
}
@@ -322,8 +323,8 @@ public class AttributeReference extends ExprTree {
@Override
public boolean privateFlatten(EvalState state, Value val, ExprTreeHolder ntree, AMutableInt32 op)
throws HyracksDataException {
- ExprTreeHolder tree = new ExprTreeHolder();
- ExprTreeHolder dummy = new ExprTreeHolder();
+ ExprTreeHolder tree = objectPool.mutableExprPool.get();
+ ExprTreeHolder dummy = objectPool.mutableExprPool.get();
ClassAd curAd;
boolean rval;
ntree.setInnerTree(null); // Just to be safe... wenger 2003-12-11.
@@ -338,8 +339,8 @@ public class AttributeReference extends ExprTree {
return true;
case EVAL_UNDEF:
if (expr != null && state.isFlattenAndInline()) {
- ExprTreeHolder expr_ntree = new ExprTreeHolder();
- Value expr_val = new Value();
+ ExprTreeHolder expr_ntree = objectPool.mutableExprPool.get();
+ Value expr_val = objectPool.valuePool.get();
if (state.getDepthRemaining() <= 0) {
val.setErrorValue();
state.getCurAd().setValue(curAd);
@@ -349,7 +350,7 @@ public class AttributeReference extends ExprTree {
rval = expr.publicFlatten(state, expr_val, expr_ntree);
state.incrementDepth();
if (rval && expr_ntree.getInnerTree() != null) {
- ntree.setInnerTree(createAttributeReference(expr_ntree, attributeStr));
+ ntree.setInnerTree(createAttributeReference(expr_ntree, attributeStr, objectPool));
if (ntree.getInnerTree() != null) {
state.getCurAd().setValue(curAd);
return true;
@@ -412,14 +413,16 @@ public class AttributeReference extends ExprTree {
* expr is not NULL, default value is false;
*/
public static AttributeReference createAttributeReference(ExprTree tree, AMutableCharArrayString attrStr,
- boolean absolut) {
- return (new AttributeReference(tree, attrStr, absolut));
+ boolean absolut, ClassAdObjectPool objectPool) {
+ AttributeReference attrRef = objectPool.attrRefPool.get();
+ attrRef.setValue(tree, attrStr, absolut);
+ return attrRef;
}
public void setValue(ExprTree tree, AMutableCharArrayString attrStr, boolean absolut) {
this.absolute = absolut;
- this.attributeStr = attrStr;
- this.expr = tree == null ? null : tree.self();
+ this.attributeStr.copyValue(attrStr.getValue(), attrStr.size());
+ this.expr.setInnerTree(tree == null ? null : tree.self());
}
public static void createAttributeReference(ExprTree tree, AMutableCharArrayString attrStr, boolean absolut,
@@ -429,8 +432,8 @@ public class AttributeReference extends ExprTree {
@Override
public boolean privateEvaluate(EvalState state, Value val) throws HyracksDataException {
- ExprTreeHolder tree = new ExprTreeHolder();
- ExprTreeHolder dummy = new ExprTreeHolder();
+ ExprTreeHolder tree = objectPool.mutableExprPool.get();
+ ExprTreeHolder dummy = objectPool.mutableExprPool.get();
ClassAd curAd;
boolean rval;
@@ -467,8 +470,12 @@ public class AttributeReference extends ExprTree {
@Override
public void reset() {
- if (expr != null) {
- expr.reset();
- }
+ expr.reset();
+ val.reset();
+ current.reset();
+ tstate.reset();
+ adList.reset();
+ attributeStr.reset();
+ absolute = false;
}
}
[09/13] incubator-asterixdb git commit: Improve Error Handling in
Local Directory Feeds
Posted by am...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/LineRecordReaderFactory.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/LineRecordReaderFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/LineRecordReaderFactory.java
index 68f10f6..4d44001 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/LineRecordReaderFactory.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/LineRecordReaderFactory.java
@@ -18,9 +18,9 @@
*/
package org.apache.asterix.external.input.record.reader.stream;
+import org.apache.asterix.external.api.AsterixInputStream;
import org.apache.asterix.external.api.IExternalIndexer;
import org.apache.asterix.external.api.IRecordReader;
-import org.apache.asterix.external.input.stream.AInputStream;
import org.apache.asterix.external.util.ExternalDataConstants;
import org.apache.asterix.external.util.ExternalDataUtils;
import org.apache.hyracks.algebricks.common.utils.Pair;
@@ -36,7 +36,7 @@ public class LineRecordReaderFactory extends AbstractStreamRecordReaderFactory<c
throws HyracksDataException {
String quoteString = configuration.get(ExternalDataConstants.KEY_QUOTE);
boolean hasHeader = ExternalDataUtils.hasHeader(configuration);
- Pair<AInputStream, IExternalIndexer> streamAndIndexer = getStreamAndIndexer(ctx, partition);
+ Pair<AsterixInputStream, IExternalIndexer> streamAndIndexer = getStreamAndIndexer(ctx, partition);
if (quoteString != null) {
return new QuotedLineRecordReader(hasHeader, streamAndIndexer.first, streamAndIndexer.second, quoteString);
} else {
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/QuotedLineRecordReader.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/QuotedLineRecordReader.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/QuotedLineRecordReader.java
index 6266aa2..abd2952 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/QuotedLineRecordReader.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/QuotedLineRecordReader.java
@@ -20,8 +20,8 @@ package org.apache.asterix.external.input.record.reader.stream;
import java.io.IOException;
+import org.apache.asterix.external.api.AsterixInputStream;
import org.apache.asterix.external.api.IExternalIndexer;
-import org.apache.asterix.external.input.stream.AInputStream;
import org.apache.asterix.external.util.ExternalDataConstants;
import org.apache.asterix.external.util.ExternalDataExceptionUtils;
import org.apache.hyracks.api.exceptions.HyracksDataException;
@@ -32,8 +32,8 @@ public class QuotedLineRecordReader extends LineRecordReader {
private boolean prevCharEscape;
private boolean inQuote;
- public QuotedLineRecordReader(final boolean hasHeader, final AInputStream stream, final IExternalIndexer indexer,
- final String quoteString) throws HyracksDataException {
+ public QuotedLineRecordReader(final boolean hasHeader, final AsterixInputStream stream,
+ final IExternalIndexer indexer, final String quoteString) throws HyracksDataException {
super(hasHeader, stream, indexer);
if ((quoteString == null) || (quoteString.length() != 1)) {
throw new HyracksDataException(ExternalDataExceptionUtils.incorrectParameterMessage(
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/SemiStructuredRecordReader.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/SemiStructuredRecordReader.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/SemiStructuredRecordReader.java
index 678dd03..7339bfd 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/SemiStructuredRecordReader.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/SemiStructuredRecordReader.java
@@ -21,8 +21,8 @@ package org.apache.asterix.external.input.record.reader.stream;
import java.io.IOException;
import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.external.api.AsterixInputStream;
import org.apache.asterix.external.api.IExternalIndexer;
-import org.apache.asterix.external.input.stream.AInputStream;
import org.apache.asterix.external.util.ExternalDataConstants;
import org.apache.asterix.external.util.ExternalDataExceptionUtils;
@@ -35,7 +35,7 @@ public class SemiStructuredRecordReader extends AbstractStreamRecordReader {
private char recordEnd;
private int recordNumber = 0;
- public SemiStructuredRecordReader(AInputStream stream, IExternalIndexer indexer, String recStartString,
+ public SemiStructuredRecordReader(AsterixInputStream stream, IExternalIndexer indexer, String recStartString,
String recEndString) throws AsterixException {
super(stream, indexer);
// set record opening char
@@ -100,9 +100,7 @@ public class SemiStructuredRecordReader extends AbstractStreamRecordReader {
&& inputBuffer[bufferPosn] != ExternalDataConstants.LF
&& inputBuffer[bufferPosn] != ExternalDataConstants.CR) {
// corrupted file. clear the buffer and stop reading
- if (!reader.skipError()) {
- reader.close();
- }
+ reader.reset();
bufferPosn = bufferLength = 0;
throw new IOException("Malformed input stream");
}
@@ -139,7 +137,13 @@ public class SemiStructuredRecordReader extends AbstractStreamRecordReader {
int appendLength = bufferPosn - startPosn;
if (appendLength > 0) {
- record.append(inputBuffer, startPosn, appendLength);
+ try {
+ record.append(inputBuffer, startPosn, appendLength);
+ } catch (IOException e) {
+ reader.reset();
+ bufferPosn = bufferLength = 0;
+ throw new IOException("Malformed input stream");
+ }
}
} while (!hasFinished);
record.endRecord();
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/SemiStructuredRecordReaderFactory.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/SemiStructuredRecordReaderFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/SemiStructuredRecordReaderFactory.java
index 206ae50..0f50204 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/SemiStructuredRecordReaderFactory.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/SemiStructuredRecordReaderFactory.java
@@ -19,9 +19,9 @@
package org.apache.asterix.external.input.record.reader.stream;
import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.external.api.AsterixInputStream;
import org.apache.asterix.external.api.IExternalIndexer;
import org.apache.asterix.external.api.IRecordReader;
-import org.apache.asterix.external.input.stream.AInputStream;
import org.apache.asterix.external.util.ExternalDataConstants;
import org.apache.hyracks.algebricks.common.utils.Pair;
import org.apache.hyracks.api.context.IHyracksTaskContext;
@@ -34,7 +34,7 @@ public class SemiStructuredRecordReaderFactory extends AbstractStreamRecordReade
@Override
public IRecordReader<? extends char[]> createRecordReader(IHyracksTaskContext ctx, int partition)
throws HyracksDataException {
- Pair<AInputStream, IExternalIndexer> streamAndIndexer = getStreamAndIndexer(ctx, partition);
+ Pair<AsterixInputStream, IExternalIndexer> streamAndIndexer = getStreamAndIndexer(ctx, partition);
try {
return new SemiStructuredRecordReader(streamAndIndexer.first, streamAndIndexer.second,
configuration.get(ExternalDataConstants.KEY_RECORD_START),
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPullRecordReader.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPullRecordReader.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPullRecordReader.java
index be9ce06..e31325a 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPullRecordReader.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPullRecordReader.java
@@ -21,9 +21,9 @@ package org.apache.asterix.external.input.record.reader.twitter;
import java.io.IOException;
import java.util.List;
-import org.apache.asterix.external.api.IDataFlowController;
import org.apache.asterix.external.api.IRawRecord;
import org.apache.asterix.external.api.IRecordReader;
+import org.apache.asterix.external.dataflow.AbstractFeedDataFlowController;
import org.apache.asterix.external.input.record.GenericRecord;
import org.apache.asterix.external.util.FeedLogManager;
import org.apache.hyracks.api.exceptions.HyracksDataException;
@@ -96,6 +96,11 @@ public class TwitterPullRecordReader implements IRecordReader<Status> {
}
@Override
- public void setController(IDataFlowController controller) {
+ public void setController(AbstractFeedDataFlowController controller) {
+ }
+
+ @Override
+ public boolean handleException(Throwable th) {
+ return false;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPushRecordReader.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPushRecordReader.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPushRecordReader.java
index 64695b5..f04cdb9 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPushRecordReader.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPushRecordReader.java
@@ -21,9 +21,9 @@ package org.apache.asterix.external.input.record.reader.twitter;
import java.io.IOException;
import java.util.concurrent.LinkedBlockingQueue;
-import org.apache.asterix.external.api.IDataFlowController;
import org.apache.asterix.external.api.IRawRecord;
import org.apache.asterix.external.api.IRecordReader;
+import org.apache.asterix.external.dataflow.AbstractFeedDataFlowController;
import org.apache.asterix.external.input.record.GenericRecord;
import org.apache.asterix.external.util.FeedLogManager;
@@ -131,6 +131,11 @@ public class TwitterPushRecordReader implements IRecordReader<Status> {
}
@Override
- public void setController(IDataFlowController controller) {
+ public void setController(AbstractFeedDataFlowController controller) {
+ }
+
+ @Override
+ public boolean handleException(Throwable th) {
+ return false;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AInputStream.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AInputStream.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AInputStream.java
deleted file mode 100644
index b78f96d..0000000
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AInputStream.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.external.input.stream;
-
-import java.io.InputStream;
-
-import org.apache.asterix.external.dataflow.AbstractFeedDataFlowController;
-import org.apache.asterix.external.util.FeedLogManager;
-
-public abstract class AInputStream extends InputStream {
- public abstract boolean skipError() throws Exception;
-
- public abstract boolean stop() throws Exception;
-
- // TODO: Find a better way to send notifications
- public abstract void setController(AbstractFeedDataFlowController controller);
-
- // TODO: Find a better way to send notifications
- public abstract void setFeedLogManager(FeedLogManager logManager);
-}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AInputStreamReader.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AInputStreamReader.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AInputStreamReader.java
deleted file mode 100644
index bf85330..0000000
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AInputStreamReader.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.external.input.stream;
-
-import java.io.IOException;
-import java.io.Reader;
-import java.nio.ByteBuffer;
-import java.nio.CharBuffer;
-import java.nio.charset.CharsetDecoder;
-import java.nio.charset.StandardCharsets;
-
-import org.apache.asterix.external.dataflow.AbstractFeedDataFlowController;
-import org.apache.asterix.external.util.ExternalDataConstants;
-import org.apache.asterix.external.util.FeedLogManager;
-
-public class AInputStreamReader extends Reader {
- private AInputStream in;
- private byte[] bytes = new byte[ExternalDataConstants.DEFAULT_BUFFER_SIZE];
- private ByteBuffer byteBuffer = ByteBuffer.wrap(bytes);
- private CharBuffer charBuffer = CharBuffer.allocate(ExternalDataConstants.DEFAULT_BUFFER_SIZE);
- private CharsetDecoder decoder;
- private boolean done = false;
-
- public AInputStreamReader(AInputStream in) {
- this.in = in;
- this.decoder = StandardCharsets.UTF_8.newDecoder();
- this.byteBuffer.flip();
- }
-
- public boolean skipError() throws Exception {
- return in.skipError();
- }
-
- public void stop() throws IOException {
- try {
- in.stop();
- } catch (Exception e) {
- throw new IOException(e);
- }
- }
-
- public void setController(AbstractFeedDataFlowController controller) {
- in.setController(controller);
- }
-
- public void setFeedLogManager(FeedLogManager feedLogManager) {
- in.setFeedLogManager(feedLogManager);
- }
-
- @Override
- public int read(char cbuf[]) throws IOException {
- return read(cbuf, 0, cbuf.length);
- }
-
- @Override
- public int read(char cbuf[], int offset, int length) throws IOException {
- if (done) {
- return -1;
- }
- int len = 0;
- charBuffer.clear();
- while (charBuffer.position() == 0) {
- if (byteBuffer.hasRemaining()) {
- decoder.decode(byteBuffer, charBuffer, false);
- System.arraycopy(charBuffer.array(), 0, cbuf, offset, charBuffer.position());
- if (charBuffer.position() > 0) {
- return charBuffer.position();
- } else {
- // need to read more data
- System.arraycopy(bytes, byteBuffer.position(), bytes, 0, byteBuffer.remaining());
- byteBuffer.position(byteBuffer.remaining());
- while (len == 0) {
- len = in.read(bytes, byteBuffer.position(), bytes.length - byteBuffer.position());
- }
- }
- } else {
- byteBuffer.clear();
- while (len == 0) {
- len = in.read(bytes, 0, bytes.length);
- }
- }
- if (len == -1) {
- done = true;
- return len;
- }
- byteBuffer.position(len);
- byteBuffer.flip();
- decoder.decode(byteBuffer, charBuffer, false);
- System.arraycopy(charBuffer.array(), 0, cbuf, offset, charBuffer.position());
- }
- return charBuffer.position();
- }
-
- @Override
- public void close() throws IOException {
- in.close();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AsterixInputStreamReader.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AsterixInputStreamReader.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AsterixInputStreamReader.java
new file mode 100644
index 0000000..b1ef892
--- /dev/null
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AsterixInputStreamReader.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.input.stream;
+
+import java.io.IOException;
+import java.io.Reader;
+import java.nio.ByteBuffer;
+import java.nio.CharBuffer;
+import java.nio.charset.CharsetDecoder;
+import java.nio.charset.StandardCharsets;
+
+import org.apache.asterix.external.api.AsterixInputStream;
+import org.apache.asterix.external.dataflow.AbstractFeedDataFlowController;
+import org.apache.asterix.external.util.ExternalDataConstants;
+import org.apache.asterix.external.util.FeedLogManager;
+
+public class AsterixInputStreamReader extends Reader {
+ private AsterixInputStream in;
+ private byte[] bytes = new byte[ExternalDataConstants.DEFAULT_BUFFER_SIZE];
+ private ByteBuffer byteBuffer = ByteBuffer.wrap(bytes);
+ private CharBuffer charBuffer = CharBuffer.allocate(ExternalDataConstants.DEFAULT_BUFFER_SIZE);
+ private CharsetDecoder decoder;
+ private boolean done = false;
+
+ public AsterixInputStreamReader(AsterixInputStream in) {
+ this.in = in;
+ this.decoder = StandardCharsets.UTF_8.newDecoder();
+ this.byteBuffer.flip();
+ }
+
+ public void stop() throws IOException {
+ try {
+ in.stop();
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+ }
+
+ public void setController(AbstractFeedDataFlowController controller) {
+ in.setController(controller);
+ }
+
+ public void setFeedLogManager(FeedLogManager feedLogManager) {
+ in.setFeedLogManager(feedLogManager);
+ }
+
+ @Override
+ public int read(char cbuf[]) throws IOException {
+ return read(cbuf, 0, cbuf.length);
+ }
+
+ @Override
+ public int read(char cbuf[], int offset, int length) throws IOException {
+ if (done) {
+ return -1;
+ }
+ int len = 0;
+ charBuffer.clear();
+ while (charBuffer.position() == 0) {
+ if (byteBuffer.hasRemaining()) {
+ decoder.decode(byteBuffer, charBuffer, false);
+ System.arraycopy(charBuffer.array(), 0, cbuf, offset, charBuffer.position());
+ if (charBuffer.position() > 0) {
+ return charBuffer.position();
+ } else {
+ // need to read more data
+ System.arraycopy(bytes, byteBuffer.position(), bytes, 0, byteBuffer.remaining());
+ byteBuffer.position(byteBuffer.remaining());
+ while (len == 0) {
+ len = in.read(bytes, byteBuffer.position(), bytes.length - byteBuffer.position());
+ }
+ }
+ } else {
+ byteBuffer.clear();
+ while (len == 0) {
+ len = in.read(bytes, 0, bytes.length);
+ }
+ }
+ if (len == -1) {
+ done = true;
+ return len;
+ }
+ byteBuffer.position(len);
+ byteBuffer.flip();
+ decoder.decode(byteBuffer, charBuffer, false);
+ System.arraycopy(charBuffer.array(), 0, cbuf, offset, charBuffer.position());
+ }
+ return charBuffer.position();
+ }
+
+ @Override
+ public void close() throws IOException {
+ in.close();
+ }
+
+ public boolean handleException(Throwable th) {
+ return in.handleException(th);
+ }
+
+ @Override
+ public void reset() throws IOException {
+ byteBuffer.clear();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/BasicInputStream.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/BasicInputStream.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/BasicInputStream.java
index 176f5f4..e4eeba3 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/BasicInputStream.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/BasicInputStream.java
@@ -21,10 +21,9 @@ package org.apache.asterix.external.input.stream;
import java.io.IOException;
import java.io.InputStream;
-import org.apache.asterix.external.dataflow.AbstractFeedDataFlowController;
-import org.apache.asterix.external.util.FeedLogManager;
+import org.apache.asterix.external.api.AsterixInputStream;
-public class BasicInputStream extends AInputStream {
+public class BasicInputStream extends AsterixInputStream {
private final InputStream in;
public BasicInputStream(InputStream in) {
@@ -78,20 +77,12 @@ public class BasicInputStream extends AInputStream {
}
@Override
- public boolean skipError() {
- return false;
- }
-
- @Override
public boolean stop() throws Exception {
return false;
}
@Override
- public void setFeedLogManager(FeedLogManager logManager) {
- }
-
- @Override
- public void setController(AbstractFeedDataFlowController controller) {
+ public boolean handleException(Throwable th) {
+ return false;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/HDFSInputStream.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/HDFSInputStream.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/HDFSInputStream.java
new file mode 100644
index 0000000..063b8fa
--- /dev/null
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/HDFSInputStream.java
@@ -0,0 +1,237 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.input.stream;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.external.api.AsterixInputStream;
+import org.apache.asterix.external.api.IExternalIndexer;
+import org.apache.asterix.external.api.IIndexingDatasource;
+import org.apache.asterix.external.indexing.ExternalFile;
+import org.apache.asterix.external.input.record.reader.hdfs.EmptyRecordReader;
+import org.apache.asterix.external.provider.ExternalIndexerProvider;
+import org.apache.asterix.external.util.ExternalDataConstants;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.mapred.FileSplit;
+import org.apache.hadoop.mapred.InputFormat;
+import org.apache.hadoop.mapred.InputSplit;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.RecordReader;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+public class HDFSInputStream extends AsterixInputStream implements IIndexingDatasource {
+
+ private RecordReader<Object, Text> reader;
+ private Text value = null;
+ private Object key = null;
+ private int currentSplitIndex = 0;
+ private boolean read[];
+ private InputFormat<?, Text> inputFormat;
+ private InputSplit[] inputSplits;
+ private String[] readSchedule;
+ private String nodeName;
+ private JobConf conf;
+ // Indexing variables
+ private final IExternalIndexer indexer;
+ private final List<ExternalFile> snapshot;
+ private final FileSystem hdfs;
+ private int pos = 0;
+
+ @SuppressWarnings("unchecked")
+ public HDFSInputStream(boolean read[], InputSplit[] inputSplits, String[] readSchedule, String nodeName,
+ JobConf conf, Map<String, String> configuration, List<ExternalFile> snapshot)
+ throws IOException, AsterixException {
+ this.read = read;
+ this.inputSplits = inputSplits;
+ this.readSchedule = readSchedule;
+ this.nodeName = nodeName;
+ this.conf = conf;
+ this.inputFormat = conf.getInputFormat();
+ this.reader = new EmptyRecordReader<Object, Text>();
+ this.snapshot = snapshot;
+ this.hdfs = FileSystem.get(conf);
+ nextInputSplit();
+ this.value = new Text();
+ if (snapshot != null) {
+ this.indexer = ExternalIndexerProvider.getIndexer(configuration);
+ if (currentSplitIndex < snapshot.size()) {
+ indexer.reset(this);
+ }
+ } else {
+ this.indexer = null;
+ }
+ }
+
+ @Override
+ public int read() throws IOException {
+ if (value.getLength() < pos) {
+ if (!readMore()) {
+ return -1;
+ }
+ } else if (value.getLength() == pos) {
+ pos++;
+ return ExternalDataConstants.BYTE_LF;
+ }
+ return value.getBytes()[pos++];
+ }
+
+ private int readRecord(byte[] buffer, int offset, int len) {
+ int actualLength = value.getLength() + 1;
+ if ((actualLength - pos) > len) {
+ //copy partial record
+ System.arraycopy(value.getBytes(), pos, buffer, offset, len);
+ pos += len;
+ return len;
+ } else {
+ int numBytes = value.getLength() - pos;
+ System.arraycopy(value.getBytes(), pos, buffer, offset, numBytes);
+ buffer[offset + numBytes] = ExternalDataConstants.LF;
+ pos += numBytes;
+ numBytes++;
+ return numBytes;
+ }
+ }
+
+ @Override
+ public int read(byte[] buffer, int offset, int len) throws IOException {
+ if (value.getLength() > pos) {
+ return readRecord(buffer, offset, len);
+ }
+ if (!readMore()) {
+ return -1;
+ }
+ return readRecord(buffer, offset, len);
+ }
+
+ private boolean readMore() throws IOException {
+ try {
+ pos = 0;
+ return HDFSInputStream.this.hasNext();
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+ }
+
+ @Override
+ public boolean stop() throws Exception {
+ return false;
+ }
+
+ @Override
+ public boolean handleException(Throwable th) {
+ return false;
+ }
+
+ @Override
+ public void close() throws IOException {
+ reader.close();
+ }
+
+ private boolean hasNext() throws Exception {
+ if (reader.next(key, value)) {
+ return true;
+ }
+ while (nextInputSplit()) {
+ if (reader.next(key, value)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private boolean nextInputSplit() throws IOException {
+ for (; currentSplitIndex < inputSplits.length; currentSplitIndex++) {
+ /**
+ * read all the partitions scheduled to the current node
+ */
+ if (readSchedule[currentSplitIndex].equals(nodeName)) {
+ /**
+ * pick an unread split to read synchronize among
+ * simultaneous partitions in the same machine
+ */
+ synchronized (read) {
+ if (read[currentSplitIndex] == false) {
+ read[currentSplitIndex] = true;
+ } else {
+ continue;
+ }
+ }
+ if (snapshot != null) {
+ String fileName = ((FileSplit) (inputSplits[currentSplitIndex])).getPath().toUri().getPath();
+ FileStatus fileStatus = hdfs.getFileStatus(new Path(fileName));
+ // Skip if not the same file stored in the files snapshot
+ if (fileStatus.getModificationTime() != snapshot.get(currentSplitIndex).getLastModefiedTime()
+ .getTime()) {
+ continue;
+ }
+ }
+
+ reader.close();
+ reader = getRecordReader(currentSplitIndex);
+ return true;
+ }
+ }
+ return false;
+ }
+
+ @SuppressWarnings("unchecked")
+ private RecordReader<Object, Text> getRecordReader(int splitIndex) throws IOException {
+ reader = (RecordReader<Object, Text>) inputFormat.getRecordReader(inputSplits[splitIndex], conf, Reporter.NULL);
+ if (key == null) {
+ key = reader.createKey();
+ value = reader.createValue();
+ }
+ if (indexer != null) {
+ try {
+ indexer.reset(this);
+ } catch (Exception e) {
+ throw new HyracksDataException(e);
+ }
+ }
+ return reader;
+ }
+
+ @Override
+ public IExternalIndexer getIndexer() {
+ return indexer;
+ }
+
+ @Override
+ public List<ExternalFile> getSnapshot() {
+ return snapshot;
+ }
+
+ @Override
+ public int getCurrentSplitIndex() {
+ return currentSplitIndex;
+ }
+
+ @Override
+ public RecordReader<?, ? extends Writable> getReader() {
+ return reader;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/LocalFSInputStream.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/LocalFSInputStream.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/LocalFSInputStream.java
new file mode 100644
index 0000000..00c1eb7
--- /dev/null
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/LocalFSInputStream.java
@@ -0,0 +1,182 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.input.stream;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.Map;
+
+import org.apache.asterix.external.api.AsterixInputStream;
+import org.apache.asterix.external.dataflow.AbstractFeedDataFlowController;
+import org.apache.asterix.external.util.ExternalDataConstants;
+import org.apache.asterix.external.util.FeedLogManager;
+import org.apache.asterix.external.util.FileSystemWatcher;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.dataflow.std.file.FileSplit;
+import org.apache.log4j.Logger;
+
+public class LocalFSInputStream extends AsterixInputStream {
+
+ private static final Logger LOGGER = Logger.getLogger(LocalFSInputStream.class.getName());
+ private final Path path;
+ private final FileSystemWatcher watcher;
+ private FileInputStream in;
+ private byte lastByte;
+ private File currentFile;
+
+ public LocalFSInputStream(final FileSplit[] fileSplits, final IHyracksTaskContext ctx,
+ final Map<String, String> configuration, final int partition, final String expression, final boolean isFeed)
+ throws IOException {
+ this.path = fileSplits[partition].getLocalFile().getFile().toPath();
+ this.watcher = new FileSystemWatcher(path, expression, isFeed);
+ this.watcher.init();
+ }
+
+ @Override
+ public void setFeedLogManager(FeedLogManager logManager) {
+ super.setFeedLogManager(logManager);
+ watcher.setFeedLogManager(logManager);
+ }
+
+ @Override
+ public void setController(AbstractFeedDataFlowController controller) {
+ super.setController(controller);
+ watcher.setController(controller);
+ }
+
+ @Override
+ public void close() throws IOException {
+ IOException ioe = null;
+ if (in != null) {
+ try {
+ closeFile();
+ } catch (Exception e) {
+ ioe = new IOException(e);
+ }
+ }
+ try {
+ watcher.close();
+ } catch (Exception e) {
+ if (ioe == null) {
+ throw e;
+ }
+ ioe.addSuppressed(e);
+ throw ioe;
+ }
+ }
+
+ private void closeFile() throws IOException {
+ if (in != null) {
+ try {
+ in.close();
+ } finally {
+ in = null;
+ currentFile = null;
+ }
+ }
+ }
+
+ /**
+ * Closes the current input stream and opens the next one, if any.
+ */
+ private boolean advance() throws IOException {
+ closeFile();
+ if (watcher.hasNext()) {
+ currentFile = watcher.next();
+ in = new FileInputStream(currentFile);
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public int read() throws IOException {
+ throw new HyracksDataException(
+ "read() is not supported with this stream. use read(byte[] b, int off, int len)");
+ }
+
+ @Override
+ public int read(byte[] b, int off, int len) throws IOException {
+ if (in == null) {
+ if (!advance()) {
+ return -1;
+ }
+ }
+ int result = in.read(b, off, len);
+ while ((result < 0) && advance()) {
+ // return a new line at the end of every file <--Might create problems for some cases
+ // depending on the parser implementation-->
+ if ((lastByte != ExternalDataConstants.BYTE_LF) && (lastByte != ExternalDataConstants.BYTE_LF)) {
+ lastByte = ExternalDataConstants.BYTE_LF;
+ b[off] = ExternalDataConstants.BYTE_LF;
+ return 1;
+ }
+ // recursive call
+ result = in.read(b, off, len);
+ }
+ if (result > 0) {
+ lastByte = b[(off + result) - 1];
+ }
+ return result;
+ }
+
+ @Override
+ public boolean stop() throws Exception {
+ watcher.close();
+ return true;
+ }
+
+ @Override
+ public boolean handleException(Throwable th) {
+ if (in == null) {
+ return false;
+ }
+ if (th instanceof IOException) {
+ // TODO: Change from string check to exception type
+ if (th.getCause().getMessage().contains("Malformed input stream")) {
+ if (currentFile != null) {
+ try {
+ logManager.logRecord(currentFile.getAbsolutePath(), "Corrupted input file");
+ } catch (IOException e) {
+ LOGGER.warn("Filed to write to feed log file", e);
+ }
+ LOGGER.warn("Corrupted input file: " + currentFile.getAbsolutePath());
+ }
+ try {
+ advance();
+ return true;
+ } catch (Exception e) {
+ return false;
+ }
+ } else {
+ try {
+ watcher.init();
+ } catch (IOException e) {
+ LOGGER.warn("Failed to initialize watcher during failure recovery", e);
+ return false;
+ }
+ }
+ return true;
+ }
+ return false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/LocalFileSystemInputStream.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/LocalFileSystemInputStream.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/LocalFileSystemInputStream.java
deleted file mode 100644
index dc6a130..0000000
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/LocalFileSystemInputStream.java
+++ /dev/null
@@ -1,137 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.external.input.stream;
-
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.nio.file.Path;
-
-import org.apache.asterix.external.dataflow.AbstractFeedDataFlowController;
-import org.apache.asterix.external.util.ExternalDataConstants;
-import org.apache.asterix.external.util.FeedLogManager;
-import org.apache.asterix.external.util.FileSystemWatcher;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-
-public class LocalFileSystemInputStream extends AInputStream {
- private final FileSystemWatcher watcher;
- private FileInputStream in;
- private byte lastByte;
-
- public LocalFileSystemInputStream(Path inputResource, String expression, boolean isFeed)
- throws HyracksDataException {
- this.watcher = new FileSystemWatcher(inputResource, expression, isFeed);
- watcher.init();
- }
-
- @Override
- public void setFeedLogManager(FeedLogManager logManager) {
- watcher.setFeedLogManager(logManager);
- }
-
- @Override
- public void setController(AbstractFeedDataFlowController controller) {
- watcher.setController(controller);
- }
-
- @Override
- public void close() throws IOException {
- IOException ioe = null;
- if (in != null) {
- try {
- closeFile();
- } catch (Exception e) {
- ioe = new IOException(e);
- }
- }
- try {
- watcher.close();
- } catch (Exception e) {
- if (ioe == null) {
- throw e;
- }
- ioe.addSuppressed(e);
- throw ioe;
- }
- }
-
- private void closeFile() throws IOException {
- if (in != null) {
- try {
- in.close();
- } finally {
- in = null;
- }
- }
- }
-
- /**
- * Closes the current input stream and opens the next one, if any.
- */
- private boolean advance() throws IOException {
- closeFile();
- if (watcher.hasNext()) {
- in = new FileInputStream(watcher.next());
- return true;
- }
- return false;
- }
-
- @Override
- public int read() throws IOException {
- throw new HyracksDataException(
- "read() is not supported with this stream. use read(byte[] b, int off, int len)");
- }
-
- @Override
- public int read(byte[] b, int off, int len) throws IOException {
- if (in == null) {
- if (!advance()) {
- return -1;
- }
- }
- int result = in.read(b, off, len);
- while ((result < 0) && advance()) {
- // return a new line at the end of every file <--Might create problems for some cases
- // depending on the parser implementation-->
- if ((lastByte != ExternalDataConstants.BYTE_LF) && (lastByte != ExternalDataConstants.BYTE_LF)) {
- lastByte = ExternalDataConstants.BYTE_LF;
- b[off] = ExternalDataConstants.BYTE_LF;
- return 1;
- }
- // recursive call
- result = in.read(b, off, len);
- }
- if (result > 0) {
- lastByte = b[(off + result) - 1];
- }
- return result;
- }
-
- @Override
- public boolean skipError() throws Exception {
- advance();
- return true;
- }
-
- @Override
- public boolean stop() throws Exception {
- watcher.close();
- return true;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/SocketClientInputStream.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/SocketClientInputStream.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/SocketClientInputStream.java
new file mode 100644
index 0000000..4321bcd
--- /dev/null
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/SocketClientInputStream.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.input.stream;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.Socket;
+import java.net.UnknownHostException;
+
+import org.apache.asterix.external.api.AsterixInputStream;
+import org.apache.hyracks.algebricks.common.utils.Pair;
+
+public class SocketClientInputStream extends AsterixInputStream {
+
+ private final Socket socket;
+ private InputStream in;
+
+ public SocketClientInputStream(Pair<String, Integer> address) throws UnknownHostException, IOException {
+ this.socket = new Socket(address.first, address.second);
+ this.in = socket.getInputStream();
+ }
+
+ @Override
+ public int read() throws IOException {
+ return in.read();
+ }
+
+ @Override
+ public int read(byte[] buffer, int offset, int length) throws IOException {
+ return in.read(buffer, offset, length);
+ }
+
+ @Override
+ public boolean stop() throws Exception {
+ if (!socket.isClosed()) {
+ socket.close();
+ }
+ return true;
+ }
+
+ @Override
+ public boolean handleException(Throwable th) {
+ return false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/SocketServerInputStream.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/SocketServerInputStream.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/SocketServerInputStream.java
index 1c33709..ce1d893 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/SocketServerInputStream.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/SocketServerInputStream.java
@@ -23,16 +23,16 @@ import java.io.InputStream;
import java.net.ServerSocket;
import java.net.Socket;
-import org.apache.asterix.external.dataflow.AbstractFeedDataFlowController;
+import org.apache.asterix.external.api.AsterixInputStream;
import org.apache.asterix.external.util.ExternalDataExceptionUtils;
-import org.apache.asterix.external.util.FeedLogManager;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.log4j.Logger;
-public class SocketServerInputStream extends AInputStream {
+public class SocketServerInputStream extends AsterixInputStream {
+ private static final Logger LOGGER = Logger.getLogger(SocketServerInputStream.class.getName());
private ServerSocket server;
private Socket socket;
private InputStream connectionStream;
- private AbstractFeedDataFlowController controller;
public SocketServerInputStream(ServerSocket server) {
this.server = server;
@@ -56,12 +56,6 @@ public class SocketServerInputStream extends AInputStream {
}
@Override
- public boolean skipError() throws Exception {
- accept();
- return true;
- }
-
- @Override
public int read(byte b[]) throws IOException {
return read(b, 0, b.length);
}
@@ -160,11 +154,13 @@ public class SocketServerInputStream extends AInputStream {
}
@Override
- public void setFeedLogManager(FeedLogManager logManager) {
- }
-
- @Override
- public void setController(AbstractFeedDataFlowController controller) {
- this.controller = controller;
+ public boolean handleException(Throwable th) {
+ try {
+ accept();
+ } catch (IOException e) {
+ LOGGER.warn("Failed accepting more connections", e);
+ return false;
+ }
+ return true;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/TwitterFirehoseInputStream.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/TwitterFirehoseInputStream.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/TwitterFirehoseInputStream.java
new file mode 100644
index 0000000..e2afd7b
--- /dev/null
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/TwitterFirehoseInputStream.java
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.input.stream;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.PipedInputStream;
+import java.io.PipedOutputStream;
+import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import org.apache.asterix.external.api.AsterixInputStream;
+import org.apache.asterix.external.util.TweetGenerator;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+
+public class TwitterFirehoseInputStream extends AsterixInputStream {
+
+ private static final Logger LOGGER = Logger.getLogger(TwitterFirehoseInputStream.class.getName());
+ private final ExecutorService executorService;
+ private final PipedOutputStream outputStream;
+ private final PipedInputStream inputStream;
+ private final DataProvider dataProvider;
+ private boolean started;
+
+ public TwitterFirehoseInputStream(Map<String, String> configuration, IHyracksTaskContext ctx, int partition)
+ throws IOException {
+ executorService = Executors.newCachedThreadPool();
+ outputStream = new PipedOutputStream();
+ inputStream = new PipedInputStream(outputStream);
+ dataProvider = new DataProvider(configuration, partition, outputStream);
+ started = false;
+ }
+
+ @Override
+ public boolean stop() throws IOException {
+ dataProvider.stop();
+ return true;
+ }
+
+ public synchronized void start() {
+ if (!started) {
+ executorService.execute(dataProvider);
+ started = true;
+ }
+ }
+
+ @Override
+ public int read() throws IOException {
+ if (!started) {
+ start();
+ }
+ return inputStream.read();
+ }
+
+ @Override
+ public int read(byte b[], int off, int len) throws IOException {
+ if (!started) {
+ start();
+ }
+ return inputStream.read(b, off, len);
+ }
+
+ @Override
+ public boolean handleException(Throwable th) {
+ return false;
+ }
+
+ private static class DataProvider implements Runnable {
+
+ public static final String KEY_MODE = "mode";
+
+ private final TweetGenerator tweetGenerator;
+ private boolean continuePush = true;
+ private int batchSize;
+ private final Mode mode;
+ private final OutputStream os;
+
+ public static enum Mode {
+ AGGRESSIVE,
+ CONTROLLED
+ }
+
+ public DataProvider(Map<String, String> configuration, int partition, OutputStream os) {
+ this.tweetGenerator = new TweetGenerator(configuration, partition);
+ this.tweetGenerator.registerSubscriber(os);
+ this.os = os;
+ mode = configuration.get(KEY_MODE) != null ? Mode.valueOf(configuration.get(KEY_MODE).toUpperCase())
+ : Mode.AGGRESSIVE;
+ switch (mode) {
+ case CONTROLLED:
+ String tpsValue = configuration.get(TweetGenerator.KEY_TPS);
+ if (tpsValue == null) {
+ throw new IllegalArgumentException("TPS value not configured. use tps=<value>");
+ }
+ batchSize = Integer.parseInt(tpsValue);
+ break;
+ case AGGRESSIVE:
+ batchSize = 5000;
+ break;
+ }
+ }
+
+ @Override
+ public void run() {
+ boolean moreData = true;
+ long startBatch;
+ long endBatch;
+ while (true) {
+ try {
+ while (moreData && continuePush) {
+ switch (mode) {
+ case AGGRESSIVE:
+ moreData = tweetGenerator.generateNextBatch(batchSize);
+ break;
+ case CONTROLLED:
+ startBatch = System.currentTimeMillis();
+ moreData = tweetGenerator.generateNextBatch(batchSize);
+ endBatch = System.currentTimeMillis();
+ if ((endBatch - startBatch) < 1000) {
+ Thread.sleep(1000 - (endBatch - startBatch));
+ }
+ break;
+ }
+ }
+ os.close();
+ break;
+ } catch (Exception e) {
+ if (LOGGER.isLoggable(Level.WARNING)) {
+ LOGGER.warning("Exception in adapter " + e.getMessage());
+ }
+ }
+ }
+ }
+
+ public void stop() {
+ continuePush = false;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/LocalFSInputStreamFactory.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/LocalFSInputStreamFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/LocalFSInputStreamFactory.java
new file mode 100644
index 0000000..85d0e41
--- /dev/null
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/LocalFSInputStreamFactory.java
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.input.stream.factory;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Map;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.external.api.AsterixInputStream;
+import org.apache.asterix.external.api.IInputStreamFactory;
+import org.apache.asterix.external.api.INodeResolver;
+import org.apache.asterix.external.api.INodeResolverFactory;
+import org.apache.asterix.external.input.stream.LocalFSInputStream;
+import org.apache.asterix.external.util.ExternalDataConstants;
+import org.apache.asterix.external.util.ExternalDataUtils;
+import org.apache.asterix.external.util.FeedUtils;
+import org.apache.asterix.external.util.NodeResolverFactory;
+import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.dataflow.std.file.FileSplit;
+
+public class LocalFSInputStreamFactory implements IInputStreamFactory {
+
+ private static final long serialVersionUID = 1L;
+
+ protected static final INodeResolver DEFAULT_NODE_RESOLVER = new NodeResolverFactory().createNodeResolver();
+ protected static final Logger LOGGER = Logger.getLogger(LocalFSInputStreamFactory.class.getName());
+ protected static INodeResolver nodeResolver;
+ protected Map<String, String> configuration;
+ protected FileSplit[] inputFileSplits;
+ protected FileSplit[] feedLogFileSplits; // paths where instances of this feed can use as log storage
+ protected boolean isFeed;
+ protected String expression;
+ // transient fields (They don't need to be serialized and transferred)
+ private transient AlgebricksAbsolutePartitionConstraint constraints;
+
+ @Override
+ public AsterixInputStream createInputStream(IHyracksTaskContext ctx, int partition) throws HyracksDataException {
+ try {
+ return new LocalFSInputStream(inputFileSplits, ctx, configuration, partition, expression, isFeed);
+ } catch (IOException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+
+ @Override
+ public DataSourceType getDataSourceType() {
+ return DataSourceType.STREAM;
+ }
+
+ @Override
+ public boolean isIndexible() {
+ return false;
+ }
+
+ @Override
+ public void configure(Map<String, String> configuration) throws AsterixException {
+ this.configuration = configuration;
+ String[] splits = configuration.get(ExternalDataConstants.KEY_PATH).split(",");
+ configureFileSplits(splits);
+ configurePartitionConstraint();
+ this.isFeed = ExternalDataUtils.isFeed(configuration) && ExternalDataUtils.keepDataSourceOpen(configuration);
+ if (isFeed) {
+ feedLogFileSplits = FeedUtils.splitsForAdapter(ExternalDataUtils.getDataverse(configuration),
+ ExternalDataUtils.getFeedName(configuration), constraints);
+ }
+ this.expression = configuration.get(ExternalDataConstants.KEY_EXPRESSION);
+ }
+
+ @Override
+ public AlgebricksAbsolutePartitionConstraint getPartitionConstraint() {
+ return constraints;
+ }
+
+ private void configureFileSplits(String[] splits) throws AsterixException {
+ if (inputFileSplits == null) {
+ inputFileSplits = new FileSplit[splits.length];
+ String nodeName;
+ String nodeLocalPath;
+ int count = 0;
+ String trimmedValue;
+ for (String splitPath : splits) {
+ trimmedValue = splitPath.trim();
+ if (!trimmedValue.contains("://")) {
+ throw new AsterixException(
+ "Invalid path: " + splitPath + "\nUsage- path=\"Host://Absolute File Path\"");
+ }
+ nodeName = trimmedValue.split(":")[0];
+ nodeLocalPath = trimmedValue.split("://")[1];
+ FileSplit fileSplit = new FileSplit(nodeName, new FileReference(new File(nodeLocalPath)));
+ inputFileSplits[count++] = fileSplit;
+ }
+ }
+ }
+
+ private void configurePartitionConstraint() throws AsterixException {
+ String[] locs = new String[inputFileSplits.length];
+ String location;
+ for (int i = 0; i < inputFileSplits.length; i++) {
+ location = getNodeResolver().resolveNode(inputFileSplits[i].getNodeName());
+ locs[i] = location;
+ }
+ constraints = new AlgebricksAbsolutePartitionConstraint(locs);
+ }
+
+ protected INodeResolver getNodeResolver() {
+ if (nodeResolver == null) {
+ synchronized (DEFAULT_NODE_RESOLVER) {
+ if (nodeResolver == null) {
+ nodeResolver = initializeNodeResolver();
+ }
+ }
+ }
+ return nodeResolver;
+ }
+
+ private static INodeResolver initializeNodeResolver() {
+ INodeResolver nodeResolver = null;
+ String configuredNodeResolverFactory = System.getProperty(ExternalDataConstants.NODE_RESOLVER_FACTORY_PROPERTY);
+ if (configuredNodeResolverFactory != null) {
+ try {
+ nodeResolver = ((INodeResolverFactory) (Class.forName(configuredNodeResolverFactory).newInstance()))
+ .createNodeResolver();
+
+ } catch (Exception e) {
+ if (LOGGER.isLoggable(Level.WARNING)) {
+ LOGGER.log(Level.WARNING, "Unable to create node resolver from the configured classname "
+ + configuredNodeResolverFactory + "\n" + e.getMessage());
+ }
+ nodeResolver = DEFAULT_NODE_RESOLVER;
+ }
+ } else {
+ nodeResolver = DEFAULT_NODE_RESOLVER;
+ }
+ return nodeResolver;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/LocalFSInputStreamProviderFactory.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/LocalFSInputStreamProviderFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/LocalFSInputStreamProviderFactory.java
deleted file mode 100644
index 54ee780..0000000
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/LocalFSInputStreamProviderFactory.java
+++ /dev/null
@@ -1,152 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.external.input.stream.factory;
-
-import java.io.File;
-import java.util.Map;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import org.apache.asterix.common.exceptions.AsterixException;
-import org.apache.asterix.external.api.IInputStreamProvider;
-import org.apache.asterix.external.api.IInputStreamProviderFactory;
-import org.apache.asterix.external.api.INodeResolver;
-import org.apache.asterix.external.api.INodeResolverFactory;
-import org.apache.asterix.external.input.stream.provider.LocalFSInputStreamProvider;
-import org.apache.asterix.external.util.ExternalDataConstants;
-import org.apache.asterix.external.util.ExternalDataUtils;
-import org.apache.asterix.external.util.FeedUtils;
-import org.apache.asterix.external.util.NodeResolverFactory;
-import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.api.io.FileReference;
-import org.apache.hyracks.dataflow.std.file.FileSplit;
-
-public class LocalFSInputStreamProviderFactory implements IInputStreamProviderFactory {
-
- private static final long serialVersionUID = 1L;
-
- protected static final INodeResolver DEFAULT_NODE_RESOLVER = new NodeResolverFactory().createNodeResolver();
- protected static final Logger LOGGER = Logger.getLogger(LocalFSInputStreamProviderFactory.class.getName());
- protected static INodeResolver nodeResolver;
- protected Map<String, String> configuration;
- protected FileSplit[] inputFileSplits;
- protected FileSplit[] feedLogFileSplits; // paths where instances of this feed can use as log storage
- protected boolean isFeed;
- protected String expression;
- // transient fields (They don't need to be serialized and transferred)
- private transient AlgebricksAbsolutePartitionConstraint constraints;
-
- @Override
- public IInputStreamProvider createInputStreamProvider(IHyracksTaskContext ctx, int partition) {
- return new LocalFSInputStreamProvider(inputFileSplits, ctx, configuration, partition, expression, isFeed);
- }
-
- @Override
- public DataSourceType getDataSourceType() {
- return DataSourceType.STREAM;
- }
-
- @Override
- public boolean isIndexible() {
- return false;
- }
-
- @Override
- public void configure(Map<String, String> configuration) throws AsterixException {
- this.configuration = configuration;
- String[] splits = configuration.get(ExternalDataConstants.KEY_PATH).split(",");
- configureFileSplits(splits);
- configurePartitionConstraint();
- this.isFeed = ExternalDataUtils.isFeed(configuration) && ExternalDataUtils.keepDataSourceOpen(configuration);
- if (isFeed) {
- feedLogFileSplits = FeedUtils.splitsForAdapter(ExternalDataUtils.getDataverse(configuration),
- ExternalDataUtils.getFeedName(configuration), constraints);
- }
- this.expression = configuration.get(ExternalDataConstants.KEY_EXPRESSION);
- }
-
- @Override
- public AlgebricksAbsolutePartitionConstraint getPartitionConstraint() {
- return constraints;
- }
-
- private void configureFileSplits(String[] splits) throws AsterixException {
- if (inputFileSplits == null) {
- inputFileSplits = new FileSplit[splits.length];
- String nodeName;
- String nodeLocalPath;
- int count = 0;
- String trimmedValue;
- for (String splitPath : splits) {
- trimmedValue = splitPath.trim();
- if (!trimmedValue.contains("://")) {
- throw new AsterixException(
- "Invalid path: " + splitPath + "\nUsage- path=\"Host://Absolute File Path\"");
- }
- nodeName = trimmedValue.split(":")[0];
- nodeLocalPath = trimmedValue.split("://")[1];
- FileSplit fileSplit = new FileSplit(nodeName, new FileReference(new File(nodeLocalPath)));
- inputFileSplits[count++] = fileSplit;
- }
- }
- }
-
- private void configurePartitionConstraint() throws AsterixException {
- String[] locs = new String[inputFileSplits.length];
- String location;
- for (int i = 0; i < inputFileSplits.length; i++) {
- location = getNodeResolver().resolveNode(inputFileSplits[i].getNodeName());
- locs[i] = location;
- }
- constraints = new AlgebricksAbsolutePartitionConstraint(locs);
- }
-
- protected INodeResolver getNodeResolver() {
- if (nodeResolver == null) {
- synchronized (DEFAULT_NODE_RESOLVER) {
- if (nodeResolver == null) {
- nodeResolver = initializeNodeResolver();
- }
- }
- }
- return nodeResolver;
- }
-
- private static INodeResolver initializeNodeResolver() {
- INodeResolver nodeResolver = null;
- String configuredNodeResolverFactory = System.getProperty(ExternalDataConstants.NODE_RESOLVER_FACTORY_PROPERTY);
- if (configuredNodeResolverFactory != null) {
- try {
- nodeResolver = ((INodeResolverFactory) (Class.forName(configuredNodeResolverFactory).newInstance()))
- .createNodeResolver();
-
- } catch (Exception e) {
- if (LOGGER.isLoggable(Level.WARNING)) {
- LOGGER.log(Level.WARNING, "Unable to create node resolver from the configured classname "
- + configuredNodeResolverFactory + "\n" + e.getMessage());
- }
- nodeResolver = DEFAULT_NODE_RESOLVER;
- }
- } else {
- nodeResolver = DEFAULT_NODE_RESOLVER;
- }
- return nodeResolver;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketClientInputStreamFactory.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketClientInputStreamFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketClientInputStreamFactory.java
new file mode 100644
index 0000000..1eb760e
--- /dev/null
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketClientInputStreamFactory.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.input.stream.factory;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.external.api.AsterixInputStream;
+import org.apache.asterix.external.api.IExternalDataSourceFactory;
+import org.apache.asterix.external.api.IInputStreamFactory;
+import org.apache.asterix.external.input.stream.SocketClientInputStream;
+import org.apache.asterix.external.util.ExternalDataConstants;
+import org.apache.http.impl.conn.SystemDefaultDnsResolver;
+import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
+import org.apache.hyracks.algebricks.common.utils.Pair;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+public class SocketClientInputStreamFactory implements IInputStreamFactory {
+
+ private static final long serialVersionUID = 1L;
+ private transient AlgebricksAbsolutePartitionConstraint clusterLocations;
+ private List<Pair<String, Integer>> sockets;
+
+ @Override
+ public AlgebricksAbsolutePartitionConstraint getPartitionConstraint() {
+ clusterLocations = IExternalDataSourceFactory.getPartitionConstraints(clusterLocations, sockets.size());
+ return clusterLocations;
+ }
+
+ @Override
+ public void configure(Map<String, String> configuration) throws AsterixException {
+ try {
+ this.sockets = new ArrayList<Pair<String, Integer>>();
+ String socketsValue = configuration.get(ExternalDataConstants.KEY_SOCKETS);
+ if (socketsValue == null) {
+ throw new IllegalArgumentException(
+ "\'sockets\' parameter not specified as part of adapter configuration");
+ }
+ String[] socketsArray = socketsValue.split(",");
+ for (String socket : socketsArray) {
+ String[] socketTokens = socket.split(":");
+ String host = socketTokens[0].trim();
+ int port = Integer.parseInt(socketTokens[1].trim());
+ InetAddress[] resolved;
+ resolved = SystemDefaultDnsResolver.INSTANCE.resolve(host);
+ Pair<String, Integer> p = new Pair<String, Integer>(resolved[0].getHostAddress(), port);
+ sockets.add(p);
+ }
+ } catch (UnknownHostException e) {
+ throw new AsterixException(e);
+ }
+ }
+
+ @Override
+ public AsterixInputStream createInputStream(IHyracksTaskContext ctx, int partition) throws HyracksDataException {
+ try {
+ return new SocketClientInputStream(sockets.get(partition));
+ } catch (IOException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketClientInputStreamProviderFactory.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketClientInputStreamProviderFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketClientInputStreamProviderFactory.java
deleted file mode 100644
index 5e84123..0000000
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketClientInputStreamProviderFactory.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.external.input.stream.factory;
-
-import java.net.InetAddress;
-import java.net.UnknownHostException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.asterix.common.exceptions.AsterixException;
-import org.apache.asterix.external.api.IExternalDataSourceFactory;
-import org.apache.asterix.external.api.IInputStreamProvider;
-import org.apache.asterix.external.api.IInputStreamProviderFactory;
-import org.apache.asterix.external.input.stream.provider.SocketClientInputStreamProvider;
-import org.apache.asterix.external.util.ExternalDataConstants;
-import org.apache.http.impl.conn.SystemDefaultDnsResolver;
-import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
-import org.apache.hyracks.algebricks.common.utils.Pair;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-
-public class SocketClientInputStreamProviderFactory implements IInputStreamProviderFactory {
-
- private static final long serialVersionUID = 1L;
- private transient AlgebricksAbsolutePartitionConstraint clusterLocations;
- private List<Pair<String, Integer>> sockets;
-
- @Override
- public AlgebricksAbsolutePartitionConstraint getPartitionConstraint() {
- clusterLocations = IExternalDataSourceFactory.getPartitionConstraints(clusterLocations, sockets.size());
- return clusterLocations;
- }
-
- @Override
- public void configure(Map<String, String> configuration) throws AsterixException {
- try {
- this.sockets = new ArrayList<Pair<String, Integer>>();
- String socketsValue = configuration.get(ExternalDataConstants.KEY_SOCKETS);
- if (socketsValue == null) {
- throw new IllegalArgumentException(
- "\'sockets\' parameter not specified as part of adapter configuration");
- }
- String[] socketsArray = socketsValue.split(",");
- for (String socket : socketsArray) {
- String[] socketTokens = socket.split(":");
- String host = socketTokens[0].trim();
- int port = Integer.parseInt(socketTokens[1].trim());
- InetAddress[] resolved;
- resolved = SystemDefaultDnsResolver.INSTANCE.resolve(host);
- Pair<String, Integer> p = new Pair<String, Integer>(resolved[0].getHostAddress(), port);
- sockets.add(p);
- }
- } catch (UnknownHostException e) {
- throw new AsterixException(e);
- }
- }
-
- @Override
- public IInputStreamProvider createInputStreamProvider(IHyracksTaskContext ctx, int partition)
- throws HyracksDataException {
- return new SocketClientInputStreamProvider(sockets.get(partition));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketServerInputStreamFactory.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketServerInputStreamFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketServerInputStreamFactory.java
new file mode 100644
index 0000000..f63b895
--- /dev/null
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketServerInputStreamFactory.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.input.stream.factory;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.ServerSocket;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
+
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.external.api.AsterixInputStream;
+import org.apache.asterix.external.api.IInputStreamFactory;
+import org.apache.asterix.external.input.stream.SocketServerInputStream;
+import org.apache.asterix.external.util.ExternalDataConstants;
+import org.apache.asterix.om.util.AsterixRuntimeUtil;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
+import org.apache.hyracks.algebricks.common.utils.Pair;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+public class SocketServerInputStreamFactory implements IInputStreamFactory {
+
+ private static final long serialVersionUID = 1L;
+ private List<Pair<String, Integer>> sockets;
+ private Mode mode = Mode.IP;
+
+ public static enum Mode {
+ NC,
+ IP
+ }
+
+ @Override
+ public void configure(Map<String, String> configuration) throws AsterixException {
+ try {
+ sockets = new ArrayList<Pair<String, Integer>>();
+ String modeValue = configuration.get(ExternalDataConstants.KEY_MODE);
+ if (modeValue != null) {
+ mode = Mode.valueOf(modeValue.trim().toUpperCase());
+ }
+ String socketsValue = configuration.get(ExternalDataConstants.KEY_SOCKETS);
+ if (socketsValue == null) {
+ throw new IllegalArgumentException(
+ "\'sockets\' parameter not specified as part of adapter configuration");
+ }
+ Map<InetAddress, Set<String>> ncMap;
+ ncMap = AsterixRuntimeUtil.getNodeControllerMap();
+ List<String> ncs = AsterixRuntimeUtil.getAllNodeControllers();
+ String[] socketsArray = socketsValue.split(",");
+ Random random = new Random();
+ for (String socket : socketsArray) {
+ String[] socketTokens = socket.split(":");
+ String host = socketTokens[0].trim();
+ int port = Integer.parseInt(socketTokens[1].trim());
+ Pair<String, Integer> p = null;
+ switch (mode) {
+ case IP:
+ Set<String> ncsOnIp = ncMap.get(InetAddress.getByName(host));
+ if ((ncsOnIp == null) || ncsOnIp.isEmpty()) {
+ throw new IllegalArgumentException("Invalid host " + host
+ + " as it is not part of the AsterixDB cluster. Valid choices are "
+ + StringUtils.join(ncMap.keySet(), ", "));
+ }
+ String[] ncArray = ncsOnIp.toArray(new String[] {});
+ String nc = ncArray[random.nextInt(ncArray.length)];
+ p = new Pair<String, Integer>(nc, port);
+ break;
+
+ case NC:
+ p = new Pair<String, Integer>(host, port);
+ if (!ncs.contains(host)) {
+ throw new IllegalArgumentException("Invalid NC " + host
+ + " as it is not part of the AsterixDB cluster. Valid choices are "
+ + StringUtils.join(ncs, ", "));
+
+ }
+ break;
+ }
+ sockets.add(p);
+ }
+ } catch (Exception e) {
+ throw new AsterixException(e);
+ }
+ }
+
+ @Override
+ public synchronized AsterixInputStream createInputStream(IHyracksTaskContext ctx, int partition)
+ throws HyracksDataException {
+ try {
+ Pair<String, Integer> socket = sockets.get(partition);
+ ServerSocket server;
+ server = new ServerSocket(socket.second);
+ return new SocketServerInputStream(server);
+ } catch (IOException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+
+ @Override
+ public AlgebricksAbsolutePartitionConstraint getPartitionConstraint() {
+ List<String> locations = new ArrayList<String>();
+ for (Pair<String, Integer> socket : sockets) {
+ locations.add(socket.first);
+ }
+ return new AlgebricksAbsolutePartitionConstraint(locations.toArray(new String[] {}));
+ }
+
+ public List<Pair<String, Integer>> getSockets() {
+ return sockets;
+ }
+
+ @Override
+ public DataSourceType getDataSourceType() {
+ return DataSourceType.STREAM;
+ }
+
+ @Override
+ public boolean isIndexible() {
+ return false;
+ }
+}
[11/13] incubator-asterixdb git commit: Improve Error Handling in
Local Directory Feeds
Posted by am...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-app/src/test/resources/runtimets/results/feeds/feed-with-external-parser/feed-with-external-parser.1.adm
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/results/feeds/feed-with-external-parser/feed-with-external-parser.1.adm b/asterix-app/src/test/resources/runtimets/results/feeds/feed-with-external-parser/feed-with-external-parser.1.adm
index 02535c5..edc05f2 100644
--- a/asterix-app/src/test/resources/runtimets/results/feeds/feed-with-external-parser/feed-with-external-parser.1.adm
+++ b/asterix-app/src/test/resources/runtimets/results/feeds/feed-with-external-parser/feed-with-external-parser.1.adm
@@ -1,99 +1,99 @@
-{ "GlobalJobId": "submit-3.chtc.wisc.edu#48968872.0#1445354636", "StatsLifetimeStarter": 572059, "JobStartDate": 1445362267, "SubmitEventNotes": "DAG Node: fabp4-0002+fabp4-0002", "JobStatus": 4, "LeaveJobInQueue": false, "WantGlidein": true, "StartdPrincipal": "execute-side@matchsession/128.104.119.175", "WantRHEL6": true, "OnExitRemove": true, "JobCurrentStartDate": 1445561276, "CoreSize": 0, "MATCH_EXP_JOBGLIDEIN_ResourceName": "wisc.edu", "Rank": 0.0d, "ExitStatus": 0, "ResidentSetSize": 125000, "ScheddBday": 1445383086, "RemoteWallClockTime": 769511.0d, "WantCheckpoint": false, "In": "/dev/null", "LastVacateTime": 1445546251, "MaxHosts": 1, "RootDir": "/", "NumRestarts": 0, "RecentBlockWriteKbytes": 0, "DiskUsage_RAW": 32543, "EnteredCurrentStatus": 1446133322, "ResidentSetSize_RAW": 100432, "RequestDisk": 1000000, "MyType": "Job", "PeriodicRemove": false, "Cmd": "/home/ssericksen/dude-14-xdock/ChtcRun/chtcjobwrapper", "CondorVersion": "$CondorVersion: 8.5.0 Sep 16 2015 BuildID
: 341710 $", "ShouldTransferFiles": "YES", "TargetType": "Machine", "MinHosts": 1, "NumCkpts_RAW": 0, "RequestCpus": 1, "RemoteUserCpu": 571737.0d, "BlockWrites": 0, "NiceUser": false, "Out": "process.out", "ImageSize_RAW": 690056, "BytesSent": 3113566.0d, "CumulativeSuspensionTime": 0, "TransferIn": false, "NumCkpts": 0, "Err": "process.err", "RecentBlockWrites": 0, "JobFinishedHookDone": 1446133322, "ProcId": 0, "ImageSize": 750000, "JobUniverse": 5, "EncryptExecuteDirectory": false, "TransferInputSizeMB": 12, "RecentBlockReads": 0, "SpooledOutputFiles": "ChtcWrapperfabp4-0002.out,AuditLog.fabp4-0002,poses.mol2,CURLTIME_4057178,harvest.log,time_elapsed.log,surf_scores.txt,CURLTIME_38803,count.log,fabp4-0002.out,CURLTIME_253463", "NumJobReconnects": 1, "WantFlocking": true, "BlockWriteKbytes": 0, "WhenToTransferOutput": "ON_EXIT_OR_EVICT", "JobCurrentStartExecutingDate": 1445561278, "ExitBySignal": false, "LastMatchTime": 1445561276, "OnExitHold": false, "OrigMaxHosts": 1, "Request
Memory": 1000, "NumJobStarts": 6, "TerminationPending": true, "TotalSuspensions": 0, "BlockReads": 0, "DAGManJobId": 48940805, "MemoryUsage": 122, "ExitCode": 0, "JobNotification": 0, "BlockReadKbytes": 0, "NumJobMatches": 6, "LocalUserCpu": 0.0d, "LastJobStatus": 2, "BufferBlockSize": 32768, "CommittedTime": 572046, "ExecutableSize_RAW": 6, "LastRejMatchReason": "no match found", "LastSuspensionTime": 0, "UserLog": "/home/ssericksen/dude-14-xdock/ChtcRun/dude14-surf-out-esr1/fabp4-0002/process.log", "DAGManNodesMask": "0,1,2,4,5,7,9,10,11,12,13,16,17,24,27", "CumulativeSlotTime": 769511.0d, "LastJobLeaseRenewal": 1446133322, "MachineAttrSlotWeight0": 1, "NumSystemHolds": 0, "BytesRecvd": 8.7351688E7d, "CondorPlatform": "$CondorPlatform: X86_64-RedHat_6.6 $", "JOBGLIDEIN_ResourceName": "$$([IfThenElse(IsUndefined(TARGET.GLIDEIN_ResourceName), IfThenElse(IsUndefined(TARGET.GLIDEIN_Site), \"wisc.edu\", TARGET.GLIDEIN_Site), TARGET.GLIDEIN_ResourceName)])", "DAGNodeName": "fabp4-0002+f
abp4-0002", "PeriodicRelease": "error", "JobRunCount": 7, "LastRemotePool": "condor.biochem.wisc.edu:9618?sock=collector", "JobLastStartDate": 1445546257, "LastRemoteHost": "slot1@cluster-0008.biochem.wisc.edu", "JobPrio": 0, "LocalSysCpu": 0.0d, "ExecutableSize": 7, "RemoteSysCpu": 0.0d, "TransferInput": "/home/ssericksen/dude-14-xdock/ChtcRun/dude14-surf-in/fabp4-0002/,/home/ssericksen/dude-14-xdock/ChtcRun/dude14-surf-in/shared/", "PeriodicHold": false, "WantRemoteIO": true, "CommittedSuspensionTime": 0, "DAGParentNodeNames": "", "CompletionDate": 1446133322, "StreamErr": false, "is_resumable": true, "RecentBlockReadKbytes": 0, "WantRemoteSyscalls": false, "NumShadowStarts": 7, "MachineAttrCpus0": 1, "DAGManNodesLog": "/home/ssericksen/dude-14-xdock/ChtcRun/dude14-surf-out-esr1/./mydag.dag.nodes.log", "Owner": "ssericksen", "Requirements": "undefined", "DiskUsage": 35000, "LastRejMatchTime": 1445375317, "JobLeaseDuration": 2400, "ClusterId": 48968872, "BufferSize": 524288, "IsCHT
CSubmit": true, "RecentStatsLifetimeStarter": 1200, "CommittedSlotTime": 572046.0d, "Args": "--type=Other --cmdtorun=surflex_run_DUDE_v1.8_esr1.sh --unique=fabp4-0002 --", "Environment": "", "LastPublicClaimId": "<128.104.119.175:9618>#1444067179#3317#...", "Iwd": "/home/ssericksen/dude-14-xdock/ChtcRun/dude14-surf-out-esr1/fabp4-0002", "QDate": 1445354636, "CurrentHosts": 0, "User": "ssericksen@chtc.wisc.edu", "StreamOut": false }
-{ "GlobalJobId": "submit-3.chtc.wisc.edu#49573720.0#1445938922", "StatsLifetimeStarter": 190245, "JobStartDate": 1445943852, "SubmitEventNotes": "DAG Node: 180+180", "JobStatus": 4, "LeaveJobInQueue": false, "StartdPrincipal": "execute-side@matchsession/128.105.245.72", "WantRHEL6": true, "OnExitRemove": true, "JobCurrentStartDate": 1445943852, "CoreSize": 0, "MATCH_EXP_JOBGLIDEIN_ResourceName": "wisc.edu", "Rank": 0.0d, "ExitStatus": 0, "ResidentSetSize": 125000, "RemoteWallClockTime": 190247.0d, "WantCheckpoint": false, "In": "/dev/null", "MaxHosts": 1, "RootDir": "/", "NumRestarts": 0, "RecentBlockWriteKbytes": 0, "DiskUsage_RAW": 1216669, "EnteredCurrentStatus": 1446134099, "ResidentSetSize_RAW": 123680, "RequestDisk": 1000000, "MyType": "Job", "PeriodicRemove": false, "Cmd": "/home/xguo23/finally/Simulation_condor/chtcjobwrapper", "CondorVersion": "$CondorVersion: 8.5.0 Sep 16 2015 BuildID: 341710 $", "ShouldTransferFiles": "YES", "TargetType": "Machine", "MinHosts": 1, "NumCkp
ts_RAW": 0, "RequestCpus": 1, "RemoteUserCpu": 185236.0d, "BlockWrites": 0, "NiceUser": false, "Out": "process.out", "ImageSize_RAW": 811948, "BytesSent": 30766.0d, "CumulativeSuspensionTime": 0, "TransferIn": false, "NumCkpts": 0, "Err": "process.err", "RecentBlockWrites": 0, "JobFinishedHookDone": 1446134099, "ProcId": 0, "ImageSize": 1000000, "JobUniverse": 5, "EncryptExecuteDirectory": false, "TransferInputSizeMB": 0, "RecentBlockReads": 0, "SpooledOutputFiles": "CURLTIME_3753852,ChtcWrapper180.out,AuditLog.180,simu_3_180.txt,harvest.log,180.out", "BlockWriteKbytes": 0, "WhenToTransferOutput": "ON_EXIT", "JobCurrentStartExecutingDate": 1445943853, "ExitBySignal": false, "LastMatchTime": 1445943852, "OnExitHold": false, "OrigMaxHosts": 1, "RequestMemory": 1000, "NumJobStarts": 1, "TerminationPending": true, "TotalSuspensions": 0, "BlockReads": 0, "DAGManJobId": 49572657, "MemoryUsage": 122, "ExitCode": 0, "JobNotification": 0, "BlockReadKbytes": 0, "NumJobMatches": 1, "LocalUserC
pu": 0.0d, "LastJobStatus": 2, "BufferBlockSize": 32768, "CommittedTime": 190247, "ExecutableSize_RAW": 6, "LastSuspensionTime": 0, "Matlab": "R2011b", "UserLog": "/home/xguo23/finally/Simulation_condor/model_3/180/process.log", "DAGManNodesMask": "0,1,2,4,5,7,9,10,11,12,13,16,17,24,27", "CumulativeSlotTime": 190247.0d, "LastJobLeaseRenewal": 1446134099, "MachineAttrSlotWeight0": 1, "NumSystemHolds": 0, "BytesRecvd": 284367.0d, "CondorPlatform": "$CondorPlatform: X86_64-RedHat_6.6 $", "JOBGLIDEIN_ResourceName": "$$([IfThenElse(IsUndefined(TARGET.GLIDEIN_ResourceName), IfThenElse(IsUndefined(TARGET.GLIDEIN_Site), \"wisc.edu\", TARGET.GLIDEIN_Site), TARGET.GLIDEIN_ResourceName)])", "DAGNodeName": "180+180", "PeriodicRelease": "error", "JobRunCount": 1, "LastRemoteHost": "slot1@e272.chtc.wisc.edu", "JobPrio": 0, "LocalSysCpu": 0.0d, "ExecutableSize": 7, "RemoteSysCpu": 1835.0d, "TransferInput": "/home/xguo23/finally/Simulation_condor/data/180/,/home/xguo23/finally/Simulation_condor/dat
a/shared/", "PeriodicHold": false, "WantRemoteIO": true, "CommittedSuspensionTime": 0, "DAGParentNodeNames": "", "CompletionDate": 1446134099, "StreamErr": false, "RecentBlockReadKbytes": 0, "WantRemoteSyscalls": false, "NumShadowStarts": 1, "MachineAttrCpus0": 1, "DAGManNodesLog": "/home/xguo23/finally/Simulation_condor/model_3/./mydag.dag.nodes.log", "Owner": "xguo23", "Requirements": "undefined", "DiskUsage": 1250000, "JobLeaseDuration": 2400, "ClusterId": 49573720, "BufferSize": 524288, "IsCHTCSubmit": true, "RecentStatsLifetimeStarter": 1200, "CommittedSlotTime": 190247.0d, "Args": "--type=Matlab --version=R2011b --cmdtorun=simu_condor --unique=180 -- 3", "Environment": "", "LastPublicClaimId": "<128.105.245.72:29075>#1444753997#6000#...", "Iwd": "/home/xguo23/finally/Simulation_condor/model_3/180", "QDate": 1445938922, "CurrentHosts": 0, "User": "xguo23@chtc.wisc.edu", "StreamOut": false }
-{ "GlobalJobId": "submit-3.chtc.wisc.edu#49581952.0#1446105329", "StatsLifetimeStarter": 27674, "JobStartDate": 1446106061, "SubmitEventNotes": "DAG Node: 40+40", "JobStatus": 4, "LeaveJobInQueue": false, "StartdPrincipal": "execute-side@matchsession/128.105.245.86", "WantRHEL6": true, "OnExitRemove": true, "JobCurrentStartDate": 1446106061, "CoreSize": 0, "MATCH_EXP_JOBGLIDEIN_ResourceName": "wisc.edu", "Rank": 0.0d, "ExitStatus": 0, "ResidentSetSize": 150000, "RemoteWallClockTime": 27676.0d, "WantCheckpoint": false, "In": "/dev/null", "MaxHosts": 1, "RootDir": "/", "NumRestarts": 0, "RecentBlockWriteKbytes": 0, "DiskUsage_RAW": 1216669, "EnteredCurrentStatus": 1446133737, "ResidentSetSize_RAW": 127252, "RequestDisk": 1000000, "MyType": "Job", "PeriodicRemove": false, "Cmd": "/home/xguo23/finally_2/Simulation_condor/chtcjobwrapper", "CondorVersion": "$CondorVersion: 8.5.0 Sep 16 2015 BuildID: 341710 $", "ShouldTransferFiles": "YES", "TargetType": "Machine", "MinHosts": 1, "NumCkpts
_RAW": 0, "RequestCpus": 1, "RemoteUserCpu": 27510.0d, "BlockWrites": 0, "NiceUser": false, "Out": "process.out", "ImageSize_RAW": 811948, "BytesSent": 30584.0d, "CumulativeSuspensionTime": 0, "TransferIn": false, "NumCkpts": 0, "Err": "process.err", "RecentBlockWrites": 0, "JobFinishedHookDone": 1446133737, "ProcId": 0, "ImageSize": 1000000, "JobUniverse": 5, "EncryptExecuteDirectory": false, "TransferInputSizeMB": 0, "RecentBlockReads": 0, "SpooledOutputFiles": "CURLTIME_369560,ChtcWrapper40.out,AuditLog.40,simu_3_40.txt,harvest.log,40.out", "BlockWriteKbytes": 0, "WhenToTransferOutput": "ON_EXIT", "JobCurrentStartExecutingDate": 1446106063, "ExitBySignal": false, "LastMatchTime": 1446106061, "OnExitHold": false, "OrigMaxHosts": 1, "RequestMemory": 1000, "NumJobStarts": 1, "TerminationPending": true, "TotalSuspensions": 0, "BlockReads": 0, "DAGManJobId": 49581933, "MemoryUsage": 146, "ExitCode": 0, "JobNotification": 0, "BlockReadKbytes": 0, "NumJobMatches": 1, "LocalUserCpu": 0.0
d, "LastJobStatus": 2, "BufferBlockSize": 32768, "CommittedTime": 27676, "ExecutableSize_RAW": 6, "LastSuspensionTime": 0, "Matlab": "R2011b", "UserLog": "/home/xguo23/finally_2/Simulation_condor/model_3/40/process.log", "DAGManNodesMask": "0,1,2,4,5,7,9,10,11,12,13,16,17,24,27", "CumulativeSlotTime": 27676.0d, "LastJobLeaseRenewal": 1446133737, "MachineAttrSlotWeight0": 1, "NumSystemHolds": 0, "BytesRecvd": 285053.0d, "CondorPlatform": "$CondorPlatform: X86_64-RedHat_6.6 $", "JOBGLIDEIN_ResourceName": "$$([IfThenElse(IsUndefined(TARGET.GLIDEIN_ResourceName), IfThenElse(IsUndefined(TARGET.GLIDEIN_Site), \"wisc.edu\", TARGET.GLIDEIN_Site), TARGET.GLIDEIN_ResourceName)])", "DAGNodeName": "40+40", "PeriodicRelease": "error", "JobRunCount": 1, "LastRemoteHost": "slot1@e286.chtc.wisc.edu", "JobPrio": 0, "LocalSysCpu": 0.0d, "ExecutableSize": 7, "RemoteSysCpu": 105.0d, "TransferInput": "/home/xguo23/finally_2/Simulation_condor/data/40/,/home/xguo23/finally_2/Simulation_condor/data/shared/
", "PeriodicHold": false, "WantRemoteIO": true, "CommittedSuspensionTime": 0, "DAGParentNodeNames": "", "CompletionDate": 1446133737, "StreamErr": false, "RecentBlockReadKbytes": 0, "WantRemoteSyscalls": false, "NumShadowStarts": 1, "MachineAttrCpus0": 1, "DAGManNodesLog": "/home/xguo23/finally_2/Simulation_condor/model_3/./mydag.dag.nodes.log", "Owner": "xguo23", "Requirements": "undefined", "DiskUsage": 1250000, "JobLeaseDuration": 2400, "ClusterId": 49581952, "BufferSize": 524288, "IsCHTCSubmit": true, "RecentStatsLifetimeStarter": 1200, "CommittedSlotTime": 27676.0d, "Args": "--type=Matlab --version=R2011b --cmdtorun=simu_condor --unique=40 -- 3", "Environment": "", "LastPublicClaimId": "<128.105.245.86:32129>#1444759888#6329#...", "Iwd": "/home/xguo23/finally_2/Simulation_condor/model_3/40", "QDate": 1446105329, "CurrentHosts": 0, "User": "xguo23@chtc.wisc.edu", "StreamOut": false }
-{ "GlobalJobId": "submit-3.chtc.wisc.edu#49581985.0#1446105368", "StatsLifetimeStarter": 26354, "JobStartDate": 1446106289, "SubmitEventNotes": "DAG Node: 36+36", "JobStatus": 4, "LeaveJobInQueue": false, "StartdPrincipal": "execute-side@matchsession/128.105.244.249", "WantRHEL6": true, "OnExitRemove": true, "JobCurrentStartDate": 1446106289, "CoreSize": 0, "MATCH_EXP_JOBGLIDEIN_ResourceName": "wisc.edu", "Rank": 0.0d, "ExitStatus": 0, "ResidentSetSize": 150000, "RemoteWallClockTime": 26357.0d, "WantCheckpoint": false, "In": "/dev/null", "MaxHosts": 1, "RootDir": "/", "NumRestarts": 0, "RecentBlockWriteKbytes": 0, "DiskUsage_RAW": 1216669, "EnteredCurrentStatus": 1446132646, "ResidentSetSize_RAW": 127452, "RequestDisk": 1000000, "MyType": "Job", "PeriodicRemove": false, "Cmd": "/home/xguo23/finally_2/Simulation_condor/chtcjobwrapper", "CondorVersion": "$CondorVersion: 8.5.0 Sep 16 2015 BuildID: 341710 $", "ShouldTransferFiles": "YES", "TargetType": "Machine", "MinHosts": 1, "NumCkpt
s_RAW": 0, "RequestCpus": 1, "RemoteUserCpu": 26239.0d, "BlockWrites": 0, "NiceUser": false, "Out": "process.out", "ImageSize_RAW": 811948, "BytesSent": 31898.0d, "CumulativeSuspensionTime": 0, "TransferIn": false, "NumCkpts": 0, "Err": "process.err", "RecentBlockWrites": 0, "JobFinishedHookDone": 1446132646, "ProcId": 0, "ImageSize": 1000000, "JobUniverse": 5, "EncryptExecuteDirectory": false, "TransferInputSizeMB": 0, "RecentBlockReads": 0, "SpooledOutputFiles": "CURLTIME_1151700,ChtcWrapper36.out,AuditLog.36,simu_3_36.txt,harvest.log,36.out", "BlockWriteKbytes": 0, "WhenToTransferOutput": "ON_EXIT", "JobCurrentStartExecutingDate": 1446106291, "ExitBySignal": false, "LastMatchTime": 1446106289, "OnExitHold": false, "OrigMaxHosts": 1, "RequestMemory": 1000, "NumJobStarts": 1, "TerminationPending": true, "TotalSuspensions": 0, "BlockReads": 0, "DAGManJobId": 49581933, "MemoryUsage": 146, "ExitCode": 0, "JobNotification": 0, "BlockReadKbytes": 0, "NumJobMatches": 1, "LocalUserCpu": 0
.0d, "LastJobStatus": 2, "BufferBlockSize": 32768, "CommittedTime": 26357, "ExecutableSize_RAW": 6, "LastSuspensionTime": 0, "Matlab": "R2011b", "UserLog": "/home/xguo23/finally_2/Simulation_condor/model_3/36/process.log", "DAGManNodesMask": "0,1,2,4,5,7,9,10,11,12,13,16,17,24,27", "CumulativeSlotTime": 26357.0d, "LastJobLeaseRenewal": 1446132646, "MachineAttrSlotWeight0": 1, "NumSystemHolds": 0, "BytesRecvd": 285053.0d, "CondorPlatform": "$CondorPlatform: X86_64-RedHat_6.6 $", "JOBGLIDEIN_ResourceName": "$$([IfThenElse(IsUndefined(TARGET.GLIDEIN_ResourceName), IfThenElse(IsUndefined(TARGET.GLIDEIN_Site), \"wisc.edu\", TARGET.GLIDEIN_Site), TARGET.GLIDEIN_ResourceName)])", "DAGNodeName": "36+36", "PeriodicRelease": "error", "JobRunCount": 1, "LastRemoteHost": "slot1@e457.chtc.wisc.edu", "JobPrio": 0, "LocalSysCpu": 0.0d, "ExecutableSize": 7, "RemoteSysCpu": 96.0d, "TransferInput": "/home/xguo23/finally_2/Simulation_condor/data/36/,/home/xguo23/finally_2/Simulation_condor/data/shared
/", "PeriodicHold": false, "WantRemoteIO": true, "CommittedSuspensionTime": 0, "DAGParentNodeNames": "", "CompletionDate": 1446132646, "StreamErr": false, "RecentBlockReadKbytes": 0, "WantRemoteSyscalls": false, "NumShadowStarts": 1, "MachineAttrCpus0": 1, "DAGManNodesLog": "/home/xguo23/finally_2/Simulation_condor/model_3/./mydag.dag.nodes.log", "Owner": "xguo23", "Requirements": "undefined", "DiskUsage": 1250000, "JobLeaseDuration": 2400, "ClusterId": 49581985, "BufferSize": 524288, "IsCHTCSubmit": true, "RecentStatsLifetimeStarter": 1200, "CommittedSlotTime": 26357.0d, "Args": "--type=Matlab --version=R2011b --cmdtorun=simu_condor --unique=36 -- 3", "Environment": "", "LastPublicClaimId": "<128.105.244.249:28476>#1444685646#10655#...", "Iwd": "/home/xguo23/finally_2/Simulation_condor/model_3/36", "QDate": 1446105368, "CurrentHosts": 0, "User": "xguo23@chtc.wisc.edu", "StreamOut": false }
-{ "GlobalJobId": "submit-3.chtc.wisc.edu#49581989.0#1446105374", "StatsLifetimeStarter": 27490, "JobStartDate": 1446106290, "SubmitEventNotes": "DAG Node: 82+82", "JobStatus": 4, "LeaveJobInQueue": false, "AutoClusterId": 24, "StartdPrincipal": "execute-side@matchsession/128.105.245.233", "WantRHEL6": true, "OnExitRemove": true, "JobCurrentStartDate": 1446106290, "CoreSize": 0, "MATCH_EXP_JOBGLIDEIN_ResourceName": "wisc.edu", "ExitStatus": 0, "Rank": 0.0d, "ResidentSetSize": 150000, "RemoteWallClockTime": 27491.0d, "WantCheckpoint": false, "In": "/dev/null", "MaxHosts": 1, "RootDir": "/", "NumRestarts": 0, "RecentBlockWriteKbytes": 0, "DiskUsage_RAW": 1216669, "EnteredCurrentStatus": 1446133781, "ResidentSetSize_RAW": 126932, "RequestDisk": 1000000, "MyType": "Job", "PeriodicRemove": false, "Cmd": "/home/xguo23/finally_2/Simulation_condor/chtcjobwrapper", "CondorVersion": "$CondorVersion: 8.5.0 Sep 16 2015 BuildID: 341710 $", "ShouldTransferFiles": "YES", "TargetType": "Machine", "M
inHosts": 1, "NumCkpts_RAW": 0, "RequestCpus": 1, "RemoteUserCpu": 27288.0d, "BlockWrites": 0, "NiceUser": false, "Out": "process.out", "ImageSize_RAW": 811948, "BytesSent": 30553.0d, "CumulativeSuspensionTime": 0, "TransferIn": false, "AutoClusterAttrs": "JobUniverse,LastCheckpointPlatform,NumCkpts,ClientMachine,_condor_RequestCpus,_condor_RequestDisk,_condor_RequestGPUs,_condor_RequestMemory,RequestCpus,RequestDisk,RequestGPUs,RequestMemory,BIOCHEM,MachineLastMatchTime,ConcurrencyLimits,NiceUser,Rank,Requirements,ImageSize,MemoryRequirements,User,RemoteGroup,SubmitterGroup,SubmitterUserPrio,Group,WIDsTheme,InteractiveJob,Is_Resumable,WantFlocking,WantGlidein,Scheduler,Owner,JobStart,MemoryUsage,IsExpressQueueJob,DiskUsage,HEP_VO,IsDesktop,OSG_VO,x509userproxysubject,PassedTest,IsLocalCMSJob,IsLocalCMSSlot,IsSAMSlot,IsSAMJob,MaxDiskTempC,IsDedicated,estimated_run_hours,IsCHTCSubmit,RequiresCVMFS,DynamicSlot,PartitionableSlot,Slot1_ExpectedMachineGracefulDrainingCompletion,Slot1_Job
Starts,Slot1_SelfMonitorAge,Slot2_ExpectedMachineGracefulDrainingCompletion,Slot2_JobStarts,Slot2_SelfMonitorAge,Slot3_ExpectedMachineGracefulDrainingCompletion,Slot3_JobStarts,Slot3_SelfMonitorAge,Slot4_ExpectedMachineGracefulDrainingCompletion,Slot4_JobStarts,Slot4_SelfMonitorAge,Slot5_ExpectedMachineGracefulDrainingCompletion,Slot5_JobStarts,Slot5_SelfMonitorAge,Slot6_ExpectedMachineGracefulDrainingCompletion,Slot6_JobStarts,Slot6_SelfMonitorAge,Slot7_ExpectedMachineGracefulDrainingCompletion,Slot7_JobStarts,Slot7_SelfMonitorAge,Slot8_ExpectedMachineGracefulDrainingCompletion,Slot8_JobStarts,Slot8_SelfMonitorAge,Slot1_TotalTimeClaimedBusy,Slot1_TotalTimeUnclaimedIdle,Slot2_TotalTimeClaimedBusy,Slot2_TotalTimeUnclaimedIdle,Slot3_TotalTimeClaimedBusy,Slot3_TotalTimeUnclaimedIdle,Slot4_TotalTimeClaimedBusy,Slot4_TotalTimeUnclaimedIdle,Slot5_TotalTimeClaimedBusy,Slot5_TotalTimeUnclaimedIdle,Slot6_TotalTimeClaimedBusy,Slot6_TotalTimeUnclaimedIdle,Slot7_TotalTimeClaimedBusy,Slot7_Total
TimeUnclaimedIdle,Slot8_TotalTimeClaimedBusy,Slot8_TotalTimeUnclaimedIdle,Slot10_ExpectedMachineGracefulDrainingCompletion,Slot10_JobStarts,Slot10_SelfMonitorAge,Slot11_ExpectedMachineGracefulDrainingCompletion,Slot11_JobStarts,Slot11_SelfMonitorAge,Slot12_ExpectedMachineGracefulDrainingCompletion,Slot12_JobStarts,Slot12_SelfMonitorAge,Slot9_ExpectedMachineGracefulDrainingCompletion,Slot9_JobStarts,Slot9_SelfMonitorAge,Slot12_TotalTimeClaimedBusy,Slot10_TotalTimeClaimedBusy,Slot10_TotalTimeUnclaimedIdle,Slot11_TotalTimeClaimedBusy,Slot11_TotalTimeUnclaimedIdle,Slot12_TotalTimeUnclaimedIdle,Slot9_TotalTimeClaimedBusy,Slot9_TotalTimeUnclaimedIdle,Slot13_ExpectedMachineGracefulDrainingCompletion,Slot13_JobStarts,Slot13_SelfMonitorAge,Slot14_ExpectedMachineGracefulDrainingCompletion,Slot14_JobStarts,Slot14_SelfMonitorAge,Slot15_ExpectedMachineGracefulDrainingCompletion,Slot15_JobStarts,Slot15_SelfMonitorAge,Slot16_ExpectedMachineGracefulDrainingCompletion,Slot16_JobStarts,Slot16_SelfMon
itorAge,IsResumable,WHEN_TO_TRANSFER_OUTPUT,_condor_Requestadmin_mutex_1,_condor_Requestadmin_mutex_2,_condor_Requestadmin_mutex_3,_condor_Requestmachine_token,Requestadmin_mutex_1,Requestadmin_mutex_2,Requestadmin_mutex_3,Requestmachine_token,nyehle,IsBuildJob,IsMatlabBuildJob,TotalJobRunTime,NodeOnline,Slot13_TotalTimeClaimedBusy,Slot13_TotalTimeUnclaimedIdle,Slot14_TotalTimeClaimedBusy,Slot14_TotalTimeUnclaimedIdle,Slot15_TotalTimeClaimedBusy,Slot15_TotalTimeUnclaimedIdle,Slot16_TotalTimeClaimedBusy,Slot16_TotalTimeUnclaimedIdle,TmpIsFull,trResumable,RequiresCMSFrontier,Slot17_ExpectedMachineGracefulDrainingCompletion,Slot17_JobStarts,Slot17_SelfMonitorAge,Slot17_TotalTimeClaimedBusy,Slot17_TotalTimeUnclaimedIdle,Slot18_ExpectedMachineGracefulDrainingCompletion,Slot18_JobStarts,Slot18_SelfMonitorAge,Slot18_TotalTimeClaimedBusy,Slot18_TotalTimeUnclaimedIdle,Slot19_ExpectedMachineGracefulDrainingCompletion,Slot19_JobStarts,Slot19_SelfMonitorAge,Slot19_TotalTimeClaimedBusy,Slot19_To
talTimeUnclaimedIdle,Slot20_ExpectedMachineGracefulDrainingCompletion,Slot20_JobStarts,Slot20_SelfMonitorAge,Slot20_TotalTimeClaimedBusy,Slot20_TotalTimeUnclaimedIdle,Slot21_ExpectedMachineGracefulDrainingCompletion,Slot21_JobStarts,Slot21_SelfMonitorAge,Slot21_TotalTimeClaimedBusy,Slot21_TotalTimeUnclaimedIdle,Slot22_ExpectedMachineGracefulDrainingCompletion,Slot22_JobStarts,Slot22_SelfMonitorAge,Slot22_TotalTimeClaimedBusy,Slot22_TotalTimeUnclaimedIdle,Slot23_ExpectedMachineGracefulDrainingCompletion,Slot23_JobStarts,Slot23_SelfMonitorAge,Slot23_TotalTimeClaimedBusy,Slot23_TotalTimeUnclaimedIdle,Slot24_ExpectedMachineGracefulDrainingCompletion,Slot24_JobStarts,Slot24_SelfMonitorAge,Slot24_TotalTimeClaimedBusy,Slot24_TotalTimeUnclaimedIdle,Slot25_ExpectedMachineGracefulDrainingCompletion,Slot25_JobStarts,Slot25_SelfMonitorAge,Slot25_TotalTimeClaimedBusy,Slot25_TotalTimeUnclaimedIdle,Slot26_ExpectedMachineGracefulDrainingCompletion,Slot26_JobStarts,Slot26_SelfMonitorAge,Slot26_Total
TimeClaimedBusy,Slot26_TotalTimeUnclaimedIdle,Slot27_ExpectedMachineGracefulDrainingCompletion,Slot27_JobStarts,Slot27_SelfMonitorAge,Slot27_TotalTimeClaimedBusy,Slot27_TotalTimeUnclaimedIdle,Slot28_ExpectedMachineGracefulDrainingCompletion,Slot28_JobStarts,Slot28_SelfMonitorAge,Slot28_TotalTimeClaimedBusy,Slot28_TotalTimeUnclaimedIdle,Slot29_ExpectedMachineGracefulDrainingCompletion,Slot29_JobStarts,Slot29_SelfMonitorAge,Slot29_TotalTimeClaimedBusy,Slot29_TotalTimeUnclaimedIdle,Slot30_ExpectedMachineGracefulDrainingCompletion,Slot30_JobStarts,Slot30_SelfMonitorAge,Slot30_TotalTimeClaimedBusy,Slot30_TotalTimeUnclaimedIdle,Slot31_ExpectedMachineGracefulDrainingCompletion,Slot31_JobStarts,Slot31_SelfMonitorAge,Slot31_TotalTimeClaimedBusy,Slot31_TotalTimeUnclaimedIdle,Slot32_ExpectedMachineGracefulDrainingCompletion,Slot32_JobStarts,Slot32_SelfMonitorAge,Slot32_TotalTimeClaimedBusy,Slot32_TotalTimeUnclaimedIdle,ResidentSetSize", "NumCkpts": 0, "Err": "process.err", "RecentBlockWrites":
0, "JobFinishedHookDone": 1446133782, "ProcId": 0, "ImageSize": 1000000, "JobUniverse": 5, "EncryptExecuteDirectory": false, "TransferInputSizeMB": 0, "RecentBlockReads": 0, "SpooledOutputFiles": "CURLTIME_4096502,ChtcWrapper82.out,AuditLog.82,simu_3_82.txt,harvest.log,82.out", "BlockWriteKbytes": 0, "WhenToTransferOutput": "ON_EXIT", "JobCurrentStartExecutingDate": 1446106291, "ExitBySignal": false, "LastMatchTime": 1446106290, "OnExitHold": false, "OrigMaxHosts": 1, "RequestMemory": 1000, "NumJobStarts": 1, "TerminationPending": true, "TotalSuspensions": 0, "BlockReads": 0, "DAGManJobId": 49581933, "MemoryUsage": 146, "ExitCode": 0, "JobNotification": 0, "BlockReadKbytes": 0, "NumJobMatches": 1, "LocalUserCpu": 0.0d, "LastJobStatus": 2, "BufferBlockSize": 32768, "CommittedTime": 27491, "ExecutableSize_RAW": 6, "LastSuspensionTime": 0, "Matlab": "R2011b", "UserLog": "/home/xguo23/finally_2/Simulation_condor/model_3/82/process.log", "DAGManNodesMask": "0,1,2,4,5,7,9,10,11,12,13,16,
17,24,27", "CumulativeSlotTime": 27491.0d, "LastJobLeaseRenewal": 1446133781, "MachineAttrSlotWeight0": 1, "NumSystemHolds": 0, "BytesRecvd": 285053.0d, "CondorPlatform": "$CondorPlatform: X86_64-RedHat_6.6 $", "JOBGLIDEIN_ResourceName": "$$([IfThenElse(IsUndefined(TARGET.GLIDEIN_ResourceName), IfThenElse(IsUndefined(TARGET.GLIDEIN_Site), \"wisc.edu\", TARGET.GLIDEIN_Site), TARGET.GLIDEIN_ResourceName)])", "DAGNodeName": "82+82", "PeriodicRelease": "error", "JobRunCount": 1, "LastRemoteHost": "slot1@e433.chtc.wisc.edu", "JobPrio": 0, "LocalSysCpu": 0.0d, "ExecutableSize": 7, "RemoteSysCpu": 173.0d, "TransferInput": "/home/xguo23/finally_2/Simulation_condor/data/82/,/home/xguo23/finally_2/Simulation_condor/data/shared/", "PeriodicHold": false, "WantRemoteIO": true, "CommittedSuspensionTime": 0, "DAGParentNodeNames": "", "CompletionDate": 1446133781, "StreamErr": false, "RecentBlockReadKbytes": 0, "WantRemoteSyscalls": false, "NumShadowStarts": 1, "MachineAttrCpus0": 1, "DAGManNodesLo
g": "/home/xguo23/finally_2/Simulation_condor/model_3/./mydag.dag.nodes.log", "Owner": "xguo23", "Requirements": "undefined", "DiskUsage": 1250000, "JobLeaseDuration": 2400, "ClusterId": 49581989, "BufferSize": 524288, "IsCHTCSubmit": true, "RecentStatsLifetimeStarter": 1200, "CommittedSlotTime": 27491.0d, "Args": "--type=Matlab --version=R2011b --cmdtorun=simu_condor --unique=82 -- 3", "Environment": "", "LastPublicClaimId": "<128.105.245.233:28601>#1443991451#13496#...", "Iwd": "/home/xguo23/finally_2/Simulation_condor/model_3/82", "QDate": 1446105374, "CurrentHosts": 0, "User": "xguo23@chtc.wisc.edu", "StreamOut": false }
-{ "GlobalJobId": "submit-3.chtc.wisc.edu#49582049.0#1446105441", "StatsLifetimeStarter": 26296, "JobStartDate": 1446106482, "SubmitEventNotes": "DAG Node: 112+112", "JobStatus": 4, "LeaveJobInQueue": false, "StartdPrincipal": "execute-side@matchsession/128.105.245.245", "WantRHEL6": true, "OnExitRemove": true, "JobCurrentStartDate": 1446106482, "CoreSize": 0, "MATCH_EXP_JOBGLIDEIN_ResourceName": "wisc.edu", "Rank": 0.0d, "ExitStatus": 0, "ResidentSetSize": 150000, "RemoteWallClockTime": 26298.0d, "WantCheckpoint": false, "In": "/dev/null", "MaxHosts": 1, "RootDir": "/", "NumRestarts": 0, "RecentBlockWriteKbytes": 0, "DiskUsage_RAW": 1216669, "EnteredCurrentStatus": 1446132780, "ResidentSetSize_RAW": 126892, "RequestDisk": 1000000, "MyType": "Job", "PeriodicRemove": false, "Cmd": "/home/xguo23/finally_2/Simulation_condor/chtcjobwrapper", "CondorVersion": "$CondorVersion: 8.5.0 Sep 16 2015 BuildID: 341710 $", "ShouldTransferFiles": "YES", "TargetType": "Machine", "MinHosts": 1, "NumCk
pts_RAW": 0, "RequestCpus": 1, "RemoteUserCpu": 26097.0d, "BlockWrites": 0, "NiceUser": false, "Out": "process.out", "ImageSize_RAW": 811948, "BytesSent": 31904.0d, "CumulativeSuspensionTime": 0, "TransferIn": false, "NumCkpts": 0, "Err": "process.err", "RecentBlockWrites": 0, "JobFinishedHookDone": 1446132780, "ProcId": 0, "ImageSize": 1000000, "JobUniverse": 5, "EncryptExecuteDirectory": false, "TransferInputSizeMB": 0, "RecentBlockReads": 0, "SpooledOutputFiles": "CURLTIME_2601607,ChtcWrapper112.out,AuditLog.112,simu_3_112.txt,harvest.log,112.out", "BlockWriteKbytes": 0, "WhenToTransferOutput": "ON_EXIT", "JobCurrentStartExecutingDate": 1446106484, "ExitBySignal": false, "LastMatchTime": 1446106482, "OnExitHold": false, "OrigMaxHosts": 1, "RequestMemory": 1000, "NumJobStarts": 1, "TerminationPending": true, "TotalSuspensions": 0, "BlockReads": 0, "DAGManJobId": 49581933, "MemoryUsage": 146, "ExitCode": 0, "JobNotification": 0, "BlockReadKbytes": 0, "NumJobMatches": 1, "LocalUserC
pu": 0.0d, "LastJobStatus": 2, "BufferBlockSize": 32768, "CommittedTime": 26298, "ExecutableSize_RAW": 6, "LastSuspensionTime": 0, "Matlab": "R2011b", "UserLog": "/home/xguo23/finally_2/Simulation_condor/model_3/112/process.log", "DAGManNodesMask": "0,1,2,4,5,7,9,10,11,12,13,16,17,24,27", "CumulativeSlotTime": 26298.0d, "LastJobLeaseRenewal": 1446132780, "MachineAttrSlotWeight0": 1, "NumSystemHolds": 0, "BytesRecvd": 285054.0d, "CondorPlatform": "$CondorPlatform: X86_64-RedHat_6.6 $", "JOBGLIDEIN_ResourceName": "$$([IfThenElse(IsUndefined(TARGET.GLIDEIN_ResourceName), IfThenElse(IsUndefined(TARGET.GLIDEIN_Site), \"wisc.edu\", TARGET.GLIDEIN_Site), TARGET.GLIDEIN_ResourceName)])", "DAGNodeName": "112+112", "PeriodicRelease": "error", "JobRunCount": 1, "LastRemoteHost": "slot1@e445.chtc.WISC.EDU", "JobPrio": 0, "LocalSysCpu": 0.0d, "ExecutableSize": 7, "RemoteSysCpu": 164.0d, "TransferInput": "/home/xguo23/finally_2/Simulation_condor/data/112/,/home/xguo23/finally_2/Simulation_condor/
data/shared/", "PeriodicHold": false, "WantRemoteIO": true, "CommittedSuspensionTime": 0, "DAGParentNodeNames": "", "CompletionDate": 1446132780, "StreamErr": false, "RecentBlockReadKbytes": 0, "WantRemoteSyscalls": false, "NumShadowStarts": 1, "MachineAttrCpus0": 1, "DAGManNodesLog": "/home/xguo23/finally_2/Simulation_condor/model_3/./mydag.dag.nodes.log", "Owner": "xguo23", "Requirements": "undefined", "DiskUsage": 1250000, "JobLeaseDuration": 2400, "ClusterId": 49582049, "BufferSize": 524288, "IsCHTCSubmit": true, "RecentStatsLifetimeStarter": 1200, "CommittedSlotTime": 26298.0d, "Args": "--type=Matlab --version=R2011b --cmdtorun=simu_condor --unique=112 -- 3", "Environment": "", "LastPublicClaimId": "<128.105.245.245:48407>#1443991450#14631#...", "Iwd": "/home/xguo23/finally_2/Simulation_condor/model_3/112", "QDate": 1446105441, "CurrentHosts": 0, "User": "xguo23@chtc.wisc.edu", "StreamOut": false }
-{ "GlobalJobId": "submit-3.chtc.wisc.edu#49582050.0#1446105441", "StatsLifetimeStarter": 27141, "JobStartDate": 1446106482, "SubmitEventNotes": "DAG Node: 301+301", "JobStatus": 4, "LeaveJobInQueue": false, "StartdPrincipal": "execute-side@matchsession/128.105.245.172", "WantRHEL6": true, "OnExitRemove": true, "JobCurrentStartDate": 1446106482, "CoreSize": 0, "MATCH_EXP_JOBGLIDEIN_ResourceName": "wisc.edu", "Rank": 0.0d, "ExitStatus": 0, "ResidentSetSize": 150000, "RemoteWallClockTime": 27143.0d, "WantCheckpoint": false, "In": "/dev/null", "MaxHosts": 1, "RootDir": "/", "NumRestarts": 0, "RecentBlockWriteKbytes": 0, "DiskUsage_RAW": 1216669, "EnteredCurrentStatus": 1446133625, "ResidentSetSize_RAW": 126464, "RequestDisk": 1000000, "MyType": "Job", "PeriodicRemove": false, "Cmd": "/home/xguo23/finally_2/Simulation_condor/chtcjobwrapper", "CondorVersion": "$CondorVersion: 8.5.0 Sep 16 2015 BuildID: 341710 $", "ShouldTransferFiles": "YES", "TargetType": "Machine", "MinHosts": 1, "NumCk
pts_RAW": 0, "RequestCpus": 1, "RemoteUserCpu": 26895.0d, "BlockWrites": 0, "NiceUser": false, "Out": "process.out", "ImageSize_RAW": 811948, "BytesSent": 31905.0d, "CumulativeSuspensionTime": 0, "TransferIn": false, "NumCkpts": 0, "Err": "process.err", "RecentBlockWrites": 0, "JobFinishedHookDone": 1446133625, "ProcId": 0, "ImageSize": 1000000, "JobUniverse": 5, "EncryptExecuteDirectory": false, "TransferInputSizeMB": 0, "RecentBlockReads": 0, "SpooledOutputFiles": "CURLTIME_2158419,ChtcWrapper301.out,AuditLog.301,simu_3_301.txt,harvest.log,301.out", "BlockWriteKbytes": 0, "WhenToTransferOutput": "ON_EXIT", "JobCurrentStartExecutingDate": 1446106484, "ExitBySignal": false, "LastMatchTime": 1446106482, "OnExitHold": false, "OrigMaxHosts": 1, "RequestMemory": 1000, "NumJobStarts": 1, "TerminationPending": true, "TotalSuspensions": 0, "BlockReads": 0, "DAGManJobId": 49581933, "MemoryUsage": 146, "ExitCode": 0, "JobNotification": 0, "BlockReadKbytes": 0, "NumJobMatches": 1, "LocalUserC
pu": 0.0d, "LastJobStatus": 2, "BufferBlockSize": 32768, "CommittedTime": 27143, "ExecutableSize_RAW": 6, "LastSuspensionTime": 0, "Matlab": "R2011b", "UserLog": "/home/xguo23/finally_2/Simulation_condor/model_3/301/process.log", "DAGManNodesMask": "0,1,2,4,5,7,9,10,11,12,13,16,17,24,27", "CumulativeSlotTime": 27143.0d, "LastJobLeaseRenewal": 1446133625, "MachineAttrSlotWeight0": 1, "NumSystemHolds": 0, "BytesRecvd": 285054.0d, "CondorPlatform": "$CondorPlatform: X86_64-RedHat_6.6 $", "JOBGLIDEIN_ResourceName": "$$([IfThenElse(IsUndefined(TARGET.GLIDEIN_ResourceName), IfThenElse(IsUndefined(TARGET.GLIDEIN_Site), \"wisc.edu\", TARGET.GLIDEIN_Site), TARGET.GLIDEIN_ResourceName)])", "DAGNodeName": "301+301", "PeriodicRelease": "error", "JobRunCount": 1, "LastRemoteHost": "slot1@e372.chtc.wisc.edu", "JobPrio": 0, "LocalSysCpu": 0.0d, "ExecutableSize": 7, "RemoteSysCpu": 201.0d, "TransferInput": "/home/xguo23/finally_2/Simulation_condor/data/301/,/home/xguo23/finally_2/Simulation_condor/
data/shared/", "PeriodicHold": false, "WantRemoteIO": true, "CommittedSuspensionTime": 0, "DAGParentNodeNames": "", "CompletionDate": 1446133625, "StreamErr": false, "RecentBlockReadKbytes": 0, "WantRemoteSyscalls": false, "NumShadowStarts": 1, "MachineAttrCpus0": 1, "DAGManNodesLog": "/home/xguo23/finally_2/Simulation_condor/model_3/./mydag.dag.nodes.log", "Owner": "xguo23", "Requirements": "undefined", "DiskUsage": 1250000, "JobLeaseDuration": 2400, "ClusterId": 49582050, "BufferSize": 524288, "IsCHTCSubmit": true, "RecentStatsLifetimeStarter": 1200, "CommittedSlotTime": 27143.0d, "Args": "--type=Matlab --version=R2011b --cmdtorun=simu_condor --unique=301 -- 3", "Environment": "", "LastPublicClaimId": "<128.105.245.172:19856>#1444760019#9307#...", "Iwd": "/home/xguo23/finally_2/Simulation_condor/model_3/301", "QDate": 1446105441, "CurrentHosts": 0, "User": "xguo23@chtc.wisc.edu", "StreamOut": false }
-{ "GlobalJobId": "submit-3.chtc.wisc.edu#49582065.0#1446105458", "StatsLifetimeStarter": 25606, "JobStartDate": 1446107042, "SubmitEventNotes": "DAG Node: 401+401", "JobStatus": 4, "LeaveJobInQueue": false, "AutoClusterId": 24, "StartdPrincipal": "execute-side@matchsession/128.105.245.206", "WantRHEL6": true, "OnExitRemove": true, "JobCurrentStartDate": 1446107042, "CoreSize": 0, "MATCH_EXP_JOBGLIDEIN_ResourceName": "wisc.edu", "ExitStatus": 0, "Rank": 0.0d, "ResidentSetSize": 150000, "RemoteWallClockTime": 25607.0d, "WantCheckpoint": false, "In": "/dev/null", "MaxHosts": 1, "RootDir": "/", "NumRestarts": 0, "RecentBlockWriteKbytes": 0, "DiskUsage_RAW": 1216669, "EnteredCurrentStatus": 1446132649, "ResidentSetSize_RAW": 126608, "RequestDisk": 1000000, "MyType": "Job", "PeriodicRemove": false, "Cmd": "/home/xguo23/finally_2/Simulation_condor/chtcjobwrapper", "CondorVersion": "$CondorVersion: 8.5.0 Sep 16 2015 BuildID: 341710 $", "ShouldTransferFiles": "YES", "TargetType": "Machine",
"MinHosts": 1, "NumCkpts_RAW": 0, "RequestCpus": 1, "RemoteUserCpu": 25478.0d, "BlockWrites": 0, "NiceUser": false, "Out": "process.out", "ImageSize_RAW": 811948, "BytesSent": 30661.0d, "CumulativeSuspensionTime": 0, "TransferIn": false, "AutoClusterAttrs": "JobUniverse,LastCheckpointPlatform,NumCkpts,ClientMachine,_condor_RequestCpus,_condor_RequestDisk,_condor_RequestGPUs,_condor_RequestMemory,RequestCpus,RequestDisk,RequestGPUs,RequestMemory,BIOCHEM,MachineLastMatchTime,ConcurrencyLimits,NiceUser,Rank,Requirements,ImageSize,MemoryRequirements,User,RemoteGroup,SubmitterGroup,SubmitterUserPrio,Group,WIDsTheme,InteractiveJob,Is_Resumable,WantFlocking,WantGlidein,Scheduler,Owner,JobStart,MemoryUsage,IsExpressQueueJob,DiskUsage,HEP_VO,IsDesktop,OSG_VO,x509userproxysubject,PassedTest,IsLocalCMSJob,IsLocalCMSSlot,IsSAMSlot,IsSAMJob,MaxDiskTempC,IsDedicated,estimated_run_hours,IsCHTCSubmit,RequiresCVMFS,DynamicSlot,PartitionableSlot,Slot1_ExpectedMachineGracefulDrainingCompletion,Slot1_J
obStarts,Slot1_SelfMonitorAge,Slot2_ExpectedMachineGracefulDrainingCompletion,Slot2_JobStarts,Slot2_SelfMonitorAge,Slot3_ExpectedMachineGracefulDrainingCompletion,Slot3_JobStarts,Slot3_SelfMonitorAge,Slot4_ExpectedMachineGracefulDrainingCompletion,Slot4_JobStarts,Slot4_SelfMonitorAge,Slot5_ExpectedMachineGracefulDrainingCompletion,Slot5_JobStarts,Slot5_SelfMonitorAge,Slot6_ExpectedMachineGracefulDrainingCompletion,Slot6_JobStarts,Slot6_SelfMonitorAge,Slot7_ExpectedMachineGracefulDrainingCompletion,Slot7_JobStarts,Slot7_SelfMonitorAge,Slot8_ExpectedMachineGracefulDrainingCompletion,Slot8_JobStarts,Slot8_SelfMonitorAge,Slot1_TotalTimeClaimedBusy,Slot1_TotalTimeUnclaimedIdle,Slot2_TotalTimeClaimedBusy,Slot2_TotalTimeUnclaimedIdle,Slot3_TotalTimeClaimedBusy,Slot3_TotalTimeUnclaimedIdle,Slot4_TotalTimeClaimedBusy,Slot4_TotalTimeUnclaimedIdle,Slot5_TotalTimeClaimedBusy,Slot5_TotalTimeUnclaimedIdle,Slot6_TotalTimeClaimedBusy,Slot6_TotalTimeUnclaimedIdle,Slot7_TotalTimeClaimedBusy,Slot7_Tot
alTimeUnclaimedIdle,Slot8_TotalTimeClaimedBusy,Slot8_TotalTimeUnclaimedIdle,Slot10_ExpectedMachineGracefulDrainingCompletion,Slot10_JobStarts,Slot10_SelfMonitorAge,Slot11_ExpectedMachineGracefulDrainingCompletion,Slot11_JobStarts,Slot11_SelfMonitorAge,Slot12_ExpectedMachineGracefulDrainingCompletion,Slot12_JobStarts,Slot12_SelfMonitorAge,Slot9_ExpectedMachineGracefulDrainingCompletion,Slot9_JobStarts,Slot9_SelfMonitorAge,Slot12_TotalTimeClaimedBusy,Slot10_TotalTimeClaimedBusy,Slot10_TotalTimeUnclaimedIdle,Slot11_TotalTimeClaimedBusy,Slot11_TotalTimeUnclaimedIdle,Slot12_TotalTimeUnclaimedIdle,Slot9_TotalTimeClaimedBusy,Slot9_TotalTimeUnclaimedIdle,Slot13_ExpectedMachineGracefulDrainingCompletion,Slot13_JobStarts,Slot13_SelfMonitorAge,Slot14_ExpectedMachineGracefulDrainingCompletion,Slot14_JobStarts,Slot14_SelfMonitorAge,Slot15_ExpectedMachineGracefulDrainingCompletion,Slot15_JobStarts,Slot15_SelfMonitorAge,Slot16_ExpectedMachineGracefulDrainingCompletion,Slot16_JobStarts,Slot16_SelfM
onitorAge,IsResumable,WHEN_TO_TRANSFER_OUTPUT,_condor_Requestadmin_mutex_1,_condor_Requestadmin_mutex_2,_condor_Requestadmin_mutex_3,_condor_Requestmachine_token,Requestadmin_mutex_1,Requestadmin_mutex_2,Requestadmin_mutex_3,Requestmachine_token,nyehle,IsBuildJob,IsMatlabBuildJob,TotalJobRunTime,NodeOnline,Slot13_TotalTimeClaimedBusy,Slot13_TotalTimeUnclaimedIdle,Slot14_TotalTimeClaimedBusy,Slot14_TotalTimeUnclaimedIdle,Slot15_TotalTimeClaimedBusy,Slot15_TotalTimeUnclaimedIdle,Slot16_TotalTimeClaimedBusy,Slot16_TotalTimeUnclaimedIdle,TmpIsFull,trResumable,RequiresCMSFrontier,Slot17_ExpectedMachineGracefulDrainingCompletion,Slot17_JobStarts,Slot17_SelfMonitorAge,Slot17_TotalTimeClaimedBusy,Slot17_TotalTimeUnclaimedIdle,Slot18_ExpectedMachineGracefulDrainingCompletion,Slot18_JobStarts,Slot18_SelfMonitorAge,Slot18_TotalTimeClaimedBusy,Slot18_TotalTimeUnclaimedIdle,Slot19_ExpectedMachineGracefulDrainingCompletion,Slot19_JobStarts,Slot19_SelfMonitorAge,Slot19_TotalTimeClaimedBusy,Slot19_
TotalTimeUnclaimedIdle,Slot20_ExpectedMachineGracefulDrainingCompletion,Slot20_JobStarts,Slot20_SelfMonitorAge,Slot20_TotalTimeClaimedBusy,Slot20_TotalTimeUnclaimedIdle,Slot21_ExpectedMachineGracefulDrainingCompletion,Slot21_JobStarts,Slot21_SelfMonitorAge,Slot21_TotalTimeClaimedBusy,Slot21_TotalTimeUnclaimedIdle,Slot22_ExpectedMachineGracefulDrainingCompletion,Slot22_JobStarts,Slot22_SelfMonitorAge,Slot22_TotalTimeClaimedBusy,Slot22_TotalTimeUnclaimedIdle,Slot23_ExpectedMachineGracefulDrainingCompletion,Slot23_JobStarts,Slot23_SelfMonitorAge,Slot23_TotalTimeClaimedBusy,Slot23_TotalTimeUnclaimedIdle,Slot24_ExpectedMachineGracefulDrainingCompletion,Slot24_JobStarts,Slot24_SelfMonitorAge,Slot24_TotalTimeClaimedBusy,Slot24_TotalTimeUnclaimedIdle,Slot25_ExpectedMachineGracefulDrainingCompletion,Slot25_JobStarts,Slot25_SelfMonitorAge,Slot25_TotalTimeClaimedBusy,Slot25_TotalTimeUnclaimedIdle,Slot26_ExpectedMachineGracefulDrainingCompletion,Slot26_JobStarts,Slot26_SelfMonitorAge,Slot26_Tot
alTimeClaimedBusy,Slot26_TotalTimeUnclaimedIdle,Slot27_ExpectedMachineGracefulDrainingCompletion,Slot27_JobStarts,Slot27_SelfMonitorAge,Slot27_TotalTimeClaimedBusy,Slot27_TotalTimeUnclaimedIdle,Slot28_ExpectedMachineGracefulDrainingCompletion,Slot28_JobStarts,Slot28_SelfMonitorAge,Slot28_TotalTimeClaimedBusy,Slot28_TotalTimeUnclaimedIdle,Slot29_ExpectedMachineGracefulDrainingCompletion,Slot29_JobStarts,Slot29_SelfMonitorAge,Slot29_TotalTimeClaimedBusy,Slot29_TotalTimeUnclaimedIdle,Slot30_ExpectedMachineGracefulDrainingCompletion,Slot30_JobStarts,Slot30_SelfMonitorAge,Slot30_TotalTimeClaimedBusy,Slot30_TotalTimeUnclaimedIdle,Slot31_ExpectedMachineGracefulDrainingCompletion,Slot31_JobStarts,Slot31_SelfMonitorAge,Slot31_TotalTimeClaimedBusy,Slot31_TotalTimeUnclaimedIdle,Slot32_ExpectedMachineGracefulDrainingCompletion,Slot32_JobStarts,Slot32_SelfMonitorAge,Slot32_TotalTimeClaimedBusy,Slot32_TotalTimeUnclaimedIdle,ResidentSetSize", "NumCkpts": 0, "Err": "process.err", "RecentBlockWrites
": 0, "JobFinishedHookDone": 1446132649, "ProcId": 0, "ImageSize": 1000000, "JobUniverse": 5, "EncryptExecuteDirectory": false, "TransferInputSizeMB": 0, "RecentBlockReads": 0, "SpooledOutputFiles": "CURLTIME_1160521,ChtcWrapper401.out,AuditLog.401,simu_3_401.txt,harvest.log,401.out", "BlockWriteKbytes": 0, "WhenToTransferOutput": "ON_EXIT", "JobCurrentStartExecutingDate": 1446107043, "ExitBySignal": false, "LastMatchTime": 1446107042, "OnExitHold": false, "OrigMaxHosts": 1, "RequestMemory": 1000, "NumJobStarts": 1, "TerminationPending": true, "TotalSuspensions": 0, "BlockReads": 0, "DAGManJobId": 49581933, "MemoryUsage": 146, "ExitCode": 0, "JobNotification": 0, "BlockReadKbytes": 0, "NumJobMatches": 1, "LocalUserCpu": 0.0d, "LastJobStatus": 2, "BufferBlockSize": 32768, "CommittedTime": 25607, "ExecutableSize_RAW": 6, "LastSuspensionTime": 0, "Matlab": "R2011b", "UserLog": "/home/xguo23/finally_2/Simulation_condor/model_3/401/process.log", "DAGManNodesMask": "0,1,2,4,5,7,9,10,11,12
,13,16,17,24,27", "CumulativeSlotTime": 25607.0d, "LastJobLeaseRenewal": 1446132649, "MachineAttrSlotWeight0": 1, "NumSystemHolds": 0, "BytesRecvd": 285054.0d, "CondorPlatform": "$CondorPlatform: X86_64-RedHat_6.6 $", "JOBGLIDEIN_ResourceName": "$$([IfThenElse(IsUndefined(TARGET.GLIDEIN_ResourceName), IfThenElse(IsUndefined(TARGET.GLIDEIN_Site), \"wisc.edu\", TARGET.GLIDEIN_Site), TARGET.GLIDEIN_ResourceName)])", "DAGNodeName": "401+401", "PeriodicRelease": "error", "JobRunCount": 1, "LastRemoteHost": "slot1@e406.chtc.wisc.edu", "JobPrio": 0, "LocalSysCpu": 0.0d, "ExecutableSize": 7, "RemoteSysCpu": 89.0d, "TransferInput": "/home/xguo23/finally_2/Simulation_condor/data/401/,/home/xguo23/finally_2/Simulation_condor/data/shared/", "PeriodicHold": false, "WantRemoteIO": true, "CommittedSuspensionTime": 0, "DAGParentNodeNames": "", "CompletionDate": 1446132649, "StreamErr": false, "RecentBlockReadKbytes": 0, "WantRemoteSyscalls": false, "NumShadowStarts": 1, "MachineAttrCpus0": 1, "DAGM
anNodesLog": "/home/xguo23/finally_2/Simulation_condor/model_3/./mydag.dag.nodes.log", "Owner": "xguo23", "Requirements": "undefined", "DiskUsage": 1250000, "JobLeaseDuration": 2400, "ClusterId": 49582065, "BufferSize": 524288, "IsCHTCSubmit": true, "RecentStatsLifetimeStarter": 1200, "CommittedSlotTime": 25607.0d, "Args": "--type=Matlab --version=R2011b --cmdtorun=simu_condor --unique=401 -- 3", "Environment": "", "LastPublicClaimId": "<128.105.245.206:27946>#1443991437#15826#...", "Iwd": "/home/xguo23/finally_2/Simulation_condor/model_3/401", "QDate": 1446105458, "CurrentHosts": 0, "User": "xguo23@chtc.wisc.edu", "StreamOut": false }
-{ "GlobalJobId": "submit-3.chtc.wisc.edu#49582094.0#1446105491", "StatsLifetimeStarter": 25168, "JobStartDate": 1446107489, "SubmitEventNotes": "DAG Node: 106+106", "JobStatus": 4, "LeaveJobInQueue": false, "StartdPrincipal": "execute-side@matchsession/128.104.55.83", "WantRHEL6": true, "OnExitRemove": true, "JobCurrentStartDate": 1446107489, "CoreSize": 0, "MATCH_EXP_JOBGLIDEIN_ResourceName": "wisc.edu", "Rank": 0.0d, "ExitStatus": 0, "ResidentSetSize": 75000, "RemoteWallClockTime": 25169.0d, "WantCheckpoint": false, "In": "/dev/null", "MaxHosts": 1, "RootDir": "/", "NumRestarts": 0, "RecentBlockWriteKbytes": 4, "DiskUsage_RAW": 1216669, "EnteredCurrentStatus": 1446132658, "ResidentSetSize_RAW": 72016, "RequestDisk": 1000000, "MyType": "Job", "PeriodicRemove": false, "Cmd": "/home/xguo23/finally_2/Simulation_condor/chtcjobwrapper", "CondorVersion": "$CondorVersion: 8.5.0 Sep 16 2015 BuildID: 341710 $", "ShouldTransferFiles": "YES", "TargetType": "Machine", "MinHosts": 1, "NumCkpts_
RAW": 0, "RequestCpus": 1, "RemoteUserCpu": 24949.0d, "BlockWrites": 1, "NiceUser": false, "Out": "process.out", "ImageSize_RAW": 119520, "BytesSent": 30486.0d, "CumulativeSuspensionTime": 0, "TransferIn": false, "NumCkpts": 0, "Err": "process.err", "RecentBlockWrites": 1, "JobFinishedHookDone": 1446132658, "ProcId": 0, "ImageSize": 125000, "JobUniverse": 5, "EncryptExecuteDirectory": false, "TransferInputSizeMB": 0, "RecentBlockReads": 86, "SpooledOutputFiles": "CURLTIME_122139,ChtcWrapper106.out,AuditLog.106,simu_3_106.txt,harvest.log,106.out", "BlockWriteKbytes": 4, "WhenToTransferOutput": "ON_EXIT", "JobCurrentStartExecutingDate": 1446107490, "ExitBySignal": false, "LastMatchTime": 1446107489, "OnExitHold": false, "OrigMaxHosts": 1, "RequestMemory": 1000, "NumJobStarts": 1, "TerminationPending": true, "TotalSuspensions": 0, "BlockReads": 665, "DAGManJobId": 49581933, "MemoryUsage": 73, "ExitCode": 0, "JobNotification": 0, "BlockReadKbytes": 26620, "NumJobMatches": 1, "LocalUserC
pu": 0.0d, "LastJobStatus": 2, "BufferBlockSize": 32768, "CommittedTime": 25169, "ExecutableSize_RAW": 6, "LastSuspensionTime": 0, "Matlab": "R2011b", "UserLog": "/home/xguo23/finally_2/Simulation_condor/model_3/106/process.log", "DAGManNodesMask": "0,1,2,4,5,7,9,10,11,12,13,16,17,24,27", "CumulativeSlotTime": 25169.0d, "LastJobLeaseRenewal": 1446132658, "MachineAttrSlotWeight0": 1, "NumSystemHolds": 0, "BytesRecvd": 285054.0d, "CondorPlatform": "$CondorPlatform: X86_64-RedHat_6.6 $", "JOBGLIDEIN_ResourceName": "$$([IfThenElse(IsUndefined(TARGET.GLIDEIN_ResourceName), IfThenElse(IsUndefined(TARGET.GLIDEIN_Site), \"wisc.edu\", TARGET.GLIDEIN_Site), TARGET.GLIDEIN_ResourceName)])", "DAGNodeName": "106+106", "PeriodicRelease": "error", "JobRunCount": 1, "LastRemoteHost": "slot1@c064.chtc.wisc.edu", "JobPrio": 0, "LocalSysCpu": 0.0d, "ExecutableSize": 7, "RemoteSysCpu": 204.0d, "TransferInput": "/home/xguo23/finally_2/Simulation_condor/data/106/,/home/xguo23/finally_2/Simulation_condor/
data/shared/", "PeriodicHold": false, "WantRemoteIO": true, "CommittedSuspensionTime": 0, "DAGParentNodeNames": "", "CompletionDate": 1446132658, "StreamErr": false, "RecentBlockReadKbytes": 960, "WantRemoteSyscalls": false, "NumShadowStarts": 1, "MachineAttrCpus0": 1, "DAGManNodesLog": "/home/xguo23/finally_2/Simulation_condor/model_3/./mydag.dag.nodes.log", "Owner": "xguo23", "Requirements": "undefined", "DiskUsage": 1250000, "JobLeaseDuration": 2400, "ClusterId": 49582094, "BufferSize": 524288, "IsCHTCSubmit": true, "RecentStatsLifetimeStarter": 1200, "CommittedSlotTime": 25169.0d, "Args": "--type=Matlab --version=R2011b --cmdtorun=simu_condor --unique=106 -- 3", "Environment": "", "LastPublicClaimId": "<128.104.55.83:25899>#1445308581#1240#...", "Iwd": "/home/xguo23/finally_2/Simulation_condor/model_3/106", "QDate": 1446105491, "CurrentHosts": 0, "User": "xguo23@chtc.wisc.edu", "StreamOut": false }
-{ "GlobalJobId": "submit-3.chtc.wisc.edu#49582098.0#1446105492", "StatsLifetimeStarter": 26020, "JobStartDate": 1446107489, "SubmitEventNotes": "DAG Node: 304+304", "JobStatus": 4, "LeaveJobInQueue": false, "StartdPrincipal": "execute-side@matchsession/128.105.245.223", "WantRHEL6": true, "OnExitRemove": true, "JobCurrentStartDate": 1446107489, "CoreSize": 0, "MATCH_EXP_JOBGLIDEIN_ResourceName": "wisc.edu", "Rank": 0.0d, "ExitStatus": 0, "ResidentSetSize": 150000, "RemoteWallClockTime": 26022.0d, "WantCheckpoint": false, "In": "/dev/null", "MaxHosts": 1, "RootDir": "/", "NumRestarts": 0, "RecentBlockWriteKbytes": 0, "DiskUsage_RAW": 1216669, "EnteredCurrentStatus": 1446133511, "ResidentSetSize_RAW": 128776, "RequestDisk": 1000000, "MyType": "Job", "PeriodicRemove": false, "Cmd": "/home/xguo23/finally_2/Simulation_condor/chtcjobwrapper", "CondorVersion": "$CondorVersion: 8.5.0 Sep 16 2015 BuildID: 341710 $", "ShouldTransferFiles": "YES", "TargetType": "Machine", "MinHosts": 1, "NumCk
pts_RAW": 0, "RequestCpus": 1, "RemoteUserCpu": 25844.0d, "BlockWrites": 0, "NiceUser": false, "Out": "process.out", "ImageSize_RAW": 811948, "BytesSent": 31801.0d, "CumulativeSuspensionTime": 0, "TransferIn": false, "NumCkpts": 0, "Err": "process.err", "RecentBlockWrites": 0, "JobFinishedHookDone": 1446133511, "ProcId": 0, "ImageSize": 1000000, "JobUniverse": 5, "EncryptExecuteDirectory": false, "TransferInputSizeMB": 0, "RecentBlockReads": 0, "SpooledOutputFiles": "CURLTIME_3651606,ChtcWrapper304.out,AuditLog.304,simu_3_304.txt,harvest.log,304.out", "BlockWriteKbytes": 0, "WhenToTransferOutput": "ON_EXIT", "JobCurrentStartExecutingDate": 1446107491, "ExitBySignal": false, "LastMatchTime": 1446107489, "OnExitHold": false, "OrigMaxHosts": 1, "RequestMemory": 1000, "NumJobStarts": 1, "TerminationPending": true, "TotalSuspensions": 0, "BlockReads": 0, "DAGManJobId": 49581933, "MemoryUsage": 146, "ExitCode": 0, "JobNotification": 0, "BlockReadKbytes": 0, "NumJobMatches": 1, "LocalUserC
pu": 0.0d, "LastJobStatus": 2, "BufferBlockSize": 32768, "CommittedTime": 26022, "ExecutableSize_RAW": 6, "LastSuspensionTime": 0, "Matlab": "R2011b", "UserLog": "/home/xguo23/finally_2/Simulation_condor/model_3/304/process.log", "DAGManNodesMask": "0,1,2,4,5,7,9,10,11,12,13,16,17,24,27", "CumulativeSlotTime": 26022.0d, "LastJobLeaseRenewal": 1446133511, "MachineAttrSlotWeight0": 1, "NumSystemHolds": 0, "BytesRecvd": 285054.0d, "CondorPlatform": "$CondorPlatform: X86_64-RedHat_6.6 $", "JOBGLIDEIN_ResourceName": "$$([IfThenElse(IsUndefined(TARGET.GLIDEIN_ResourceName), IfThenElse(IsUndefined(TARGET.GLIDEIN_Site), \"wisc.edu\", TARGET.GLIDEIN_Site), TARGET.GLIDEIN_ResourceName)])", "DAGNodeName": "304+304", "PeriodicRelease": "error", "JobRunCount": 1, "LastRemoteHost": "slot1@e423.chtc.wisc.edu", "JobPrio": 0, "LocalSysCpu": 0.0d, "ExecutableSize": 7, "RemoteSysCpu": 143.0d, "TransferInput": "/home/xguo23/finally_2/Simulation_condor/data/304/,/home/xguo23/finally_2/Simulation_condor/
data/shared/", "PeriodicHold": false, "WantRemoteIO": true, "CommittedSuspensionTime": 0, "DAGParentNodeNames": "", "CompletionDate": 1446133511, "StreamErr": false, "RecentBlockReadKbytes": 0, "WantRemoteSyscalls": false, "NumShadowStarts": 1, "MachineAttrCpus0": 1, "DAGManNodesLog": "/home/xguo23/finally_2/Simulation_condor/model_3/./mydag.dag.nodes.log", "Owner": "xguo23", "Requirements": "undefined", "DiskUsage": 1250000, "JobLeaseDuration": 2400, "ClusterId": 49582098, "BufferSize": 524288, "IsCHTCSubmit": true, "RecentStatsLifetimeStarter": 1200, "CommittedSlotTime": 26022.0d, "Args": "--type=Matlab --version=R2011b --cmdtorun=simu_condor --unique=304 -- 3", "Environment": "", "LastPublicClaimId": "<128.105.245.223:13467>#1444760039#6376#...", "Iwd": "/home/xguo23/finally_2/Simulation_condor/model_3/304", "QDate": 1446105492, "CurrentHosts": 0, "User": "xguo23@chtc.wisc.edu", "StreamOut": false }
-{ "GlobalJobId": "submit-3.chtc.wisc.edu#49582113.0#1446105509", "StatsLifetimeStarter": 26044, "JobStartDate": 1446107490, "SubmitEventNotes": "DAG Node: 206+206", "JobStatus": 4, "LeaveJobInQueue": false, "StartdPrincipal": "execute-side@matchsession/128.105.245.120", "WantRHEL6": true, "OnExitRemove": true, "JobCurrentStartDate": 1446107490, "CoreSize": 0, "MATCH_EXP_JOBGLIDEIN_ResourceName": "wisc.edu", "Rank": 0.0d, "ExitStatus": 0, "ResidentSetSize": 150000, "RemoteWallClockTime": 26045.0d, "WantCheckpoint": false, "In": "/dev/null", "MaxHosts": 1, "RootDir": "/", "NumRestarts": 0, "RecentBlockWriteKbytes": 0, "DiskUsage_RAW": 1216669, "EnteredCurrentStatus": 1446133535, "ResidentSetSize_RAW": 126460, "RequestDisk": 1000000, "MyType": "Job", "PeriodicRemove": false, "Cmd": "/home/xguo23/finally_2/Simulation_condor/chtcjobwrapper", "CondorVersion": "$CondorVersion: 8.5.0 Sep 16 2015 BuildID: 341710 $", "ShouldTransferFiles": "YES", "TargetType": "Machine", "MinHosts": 1, "NumCk
pts_RAW": 0, "RequestCpus": 1, "RemoteUserCpu": 25939.0d, "BlockWrites": 0, "NiceUser": false, "Out": "process.out", "ImageSize_RAW": 811948, "BytesSent": 30596.0d, "CumulativeSuspensionTime": 0, "TransferIn": false, "NumCkpts": 0, "Err": "process.err", "RecentBlockWrites": 0, "JobFinishedHookDone": 1446133535, "ProcId": 0, "ImageSize": 1000000, "JobUniverse": 5, "EncryptExecuteDirectory": false, "TransferInputSizeMB": 0, "RecentBlockReads": 0, "SpooledOutputFiles": "CURLTIME_522843,ChtcWrapper206.out,AuditLog.206,simu_3_206.txt,harvest.log,206.out", "BlockWriteKbytes": 0, "WhenToTransferOutput": "ON_EXIT", "JobCurrentStartExecutingDate": 1446107491, "ExitBySignal": false, "LastMatchTime": 1446107490, "OnExitHold": false, "OrigMaxHosts": 1, "RequestMemory": 1000, "NumJobStarts": 1, "TerminationPending": true, "TotalSuspensions": 0, "BlockReads": 0, "DAGManJobId": 49581933, "MemoryUsage": 146, "ExitCode": 0, "JobNotification": 0, "BlockReadKbytes": 0, "NumJobMatches": 1, "LocalUserCp
u": 0.0d, "LastJobStatus": 2, "BufferBlockSize": 32768, "CommittedTime": 26045, "ExecutableSize_RAW": 6, "LastSuspensionTime": 0, "Matlab": "R2011b", "UserLog": "/home/xguo23/finally_2/Simulation_condor/model_3/206/process.log", "DAGManNodesMask": "0,1,2,4,5,7,9,10,11,12,13,16,17,24,27", "CumulativeSlotTime": 26045.0d, "LastJobLeaseRenewal": 1446133535, "MachineAttrSlotWeight0": 1, "NumSystemHolds": 0, "BytesRecvd": 285054.0d, "CondorPlatform": "$CondorPlatform: X86_64-RedHat_6.6 $", "JOBGLIDEIN_ResourceName": "$$([IfThenElse(IsUndefined(TARGET.GLIDEIN_ResourceName), IfThenElse(IsUndefined(TARGET.GLIDEIN_Site), \"wisc.edu\", TARGET.GLIDEIN_Site), TARGET.GLIDEIN_ResourceName)])", "DAGNodeName": "206+206", "PeriodicRelease": "error", "JobRunCount": 1, "LastRemoteHost": "slot1@e320.chtc.wisc.edu", "JobPrio": 0, "LocalSysCpu": 0.0d, "ExecutableSize": 7, "RemoteSysCpu": 87.0d, "TransferInput": "/home/xguo23/finally_2/Simulation_condor/data/206/,/home/xguo23/finally_2/Simulation_condor/da
ta/shared/", "PeriodicHold": false, "WantRemoteIO": true, "CommittedSuspensionTime": 0, "DAGParentNodeNames": "", "CompletionDate": 1446133535, "StreamErr": false, "RecentBlockReadKbytes": 0, "WantRemoteSyscalls": false, "NumShadowStarts": 1, "MachineAttrCpus0": 1, "DAGManNodesLog": "/home/xguo23/finally_2/Simulation_condor/model_3/./mydag.dag.nodes.log", "Owner": "xguo23", "Requirements": "undefined", "DiskUsage": 1250000, "JobLeaseDuration": 2400, "ClusterId": 49582113, "BufferSize": 524288, "IsCHTCSubmit": true, "RecentStatsLifetimeStarter": 1200, "CommittedSlotTime": 26045.0d, "Args": "--type=Matlab --version=R2011b --cmdtorun=simu_condor --unique=206 -- 3", "Environment": "", "LastPublicClaimId": "<128.105.245.120:45185>#1443991409#14238#...", "Iwd": "/home/xguo23/finally_2/Simulation_condor/model_3/206", "QDate": 1446105509, "CurrentHosts": 0, "User": "xguo23@chtc.wisc.edu", "StreamOut": false }
-{ "GlobalJobId": "submit-3.chtc.wisc.edu#49582119.0#1446105519", "StatsLifetimeStarter": 24928, "JobStartDate": 1446107490, "SubmitEventNotes": "DAG Node: 152+152", "JobStatus": 4, "LeaveJobInQueue": false, "StartdPrincipal": "execute-side@matchsession/128.105.245.242", "WantRHEL6": true, "OnExitRemove": true, "JobCurrentStartDate": 1446107490, "CoreSize": 0, "MATCH_EXP_JOBGLIDEIN_ResourceName": "wisc.edu", "Rank": 0.0d, "ExitStatus": 0, "ResidentSetSize": 150000, "RemoteWallClockTime": 24930.0d, "WantCheckpoint": false, "In": "/dev/null", "MaxHosts": 1, "RootDir": "/", "NumRestarts": 0, "RecentBlockWriteKbytes": 0, "DiskUsage_RAW": 1216669, "EnteredCurrentStatus": 1446132420, "ResidentSetSize_RAW": 128972, "RequestDisk": 1000000, "MyType": "Job", "PeriodicRemove": false, "Cmd": "/home/xguo23/finally_2/Simulation_condor/chtcjobwrapper", "CondorVersion": "$CondorVersion: 8.5.0 Sep 16 2015 BuildID: 341710 $", "ShouldTransferFiles": "YES", "TargetType": "Machine", "MinHosts": 1, "NumCk
pts_RAW": 0, "RequestCpus": 1, "RemoteUserCpu": 24742.0d, "BlockWrites": 0, "NiceUser": false, "Out": "process.out", "ImageSize_RAW": 811948, "BytesSent": 30431.0d, "CumulativeSuspensionTime": 0, "TransferIn": false, "NumCkpts": 0, "Err": "process.err", "RecentBlockWrites": 0, "JobFinishedHookDone": 1446132420, "ProcId": 0, "ImageSize": 1000000, "JobUniverse": 5, "EncryptExecuteDirectory": false, "TransferInputSizeMB": 0, "RecentBlockReads": 0, "SpooledOutputFiles": "CURLTIME_674,ChtcWrapper152.out,AuditLog.152,simu_3_152.txt,harvest.log,152.out", "BlockWriteKbytes": 0, "WhenToTransferOutput": "ON_EXIT", "JobCurrentStartExecutingDate": 1446107491, "ExitBySignal": false, "LastMatchTime": 1446107490, "OnExitHold": false, "OrigMaxHosts": 1, "RequestMemory": 1000, "NumJobStarts": 1, "TerminationPending": true, "TotalSuspensions": 0, "BlockReads": 0, "DAGManJobId": 49581933, "MemoryUsage": 146, "ExitCode": 0, "JobNotification": 0, "BlockReadKbytes": 0, "NumJobMatches": 1, "LocalUserCpu":
0.0d, "LastJobStatus": 2, "BufferBlockSize": 32768, "CommittedTime": 24930, "ExecutableSize_RAW": 6, "LastSuspensionTime": 0, "Matlab": "R2011b", "UserLog": "/home/xguo23/finally_2/Simulation_condor/model_3/152/process.log", "DAGManNodesMask": "0,1,2,4,5,7,9,10,11,12,13,16,17,24,27", "CumulativeSlotTime": 24930.0d, "LastJobLeaseRenewal": 1446132420, "MachineAttrSlotWeight0": 1, "NumSystemHolds": 0, "BytesRecvd": 285054.0d, "CondorPlatform": "$CondorPlatform: X86_64-RedHat_6.6 $", "JOBGLIDEIN_ResourceName": "$$([IfThenElse(IsUndefined(TARGET.GLIDEIN_ResourceName), IfThenElse(IsUndefined(TARGET.GLIDEIN_Site), \"wisc.edu\", TARGET.GLIDEIN_Site), TARGET.GLIDEIN_ResourceName)])", "DAGNodeName": "152+152", "PeriodicRelease": "error", "JobRunCount": 1, "LastRemoteHost": "slot1@e442.chtc.WISC.EDU", "JobPrio": 0, "LocalSysCpu": 0.0d, "ExecutableSize": 7, "RemoteSysCpu": 156.0d, "TransferInput": "/home/xguo23/finally_2/Simulation_condor/data/152/,/home/xguo23/finally_2/Simulation_condor/data
/shared/", "PeriodicHold": false, "WantRemoteIO": true, "CommittedSuspensionTime": 0, "DAGParentNodeNames": "", "CompletionDate": 1446132420, "StreamErr": false, "RecentBlockReadKbytes": 0, "WantRemoteSyscalls": false, "NumShadowStarts": 1, "MachineAttrCpus0": 1, "DAGManNodesLog": "/home/xguo23/finally_2/Simulation_condor/model_3/./mydag.dag.nodes.log", "Owner": "xguo23", "Requirements": "undefined", "DiskUsage": 1250000, "JobLeaseDuration": 2400, "ClusterId": 49582119, "BufferSize": 524288, "IsCHTCSubmit": true, "RecentStatsLifetimeStarter": 1200, "CommittedSlotTime": 24930.0d, "Args": "--type=Matlab --version=R2011b --cmdtorun=simu_condor --unique=152 -- 3", "Environment": "", "LastPublicClaimId": "<128.105.245.242:38884>#1443991450#10374#...", "Iwd": "/home/xguo23/finally_2/Simulation_condor/model_3/152", "QDate": 1446105519, "CurrentHosts": 0, "User": "xguo23@chtc.wisc.edu", "StreamOut": false }
-{ "GlobalJobId": "submit-3.chtc.wisc.edu#49582148.0#1446105547", "StatsLifetimeStarter": 26230, "JobStartDate": 1446107686, "SubmitEventNotes": "DAG Node: 162+162", "JobStatus": 4, "LeaveJobInQueue": false, "AutoClusterId": 24, "StartdPrincipal": "execute-side@matchsession/128.105.245.170", "WantRHEL6": true, "OnExitRemove": true, "JobCurrentStartDate": 1446107686, "CoreSize": 0, "MATCH_EXP_JOBGLIDEIN_ResourceName": "wisc.edu", "ExitStatus": 0, "Rank": 0.0d, "ResidentSetSize": 150000, "RemoteWallClockTime": 26233.0d, "WantCheckpoint": false, "In": "/dev/null", "MaxHosts": 1, "RootDir": "/", "NumRestarts": 0, "RecentBlockWriteKbytes": 0, "DiskUsage_RAW": 1216669, "EnteredCurrentStatus": 1446133919, "ResidentSetSize_RAW": 126384, "RequestDisk": 1000000, "MyType": "Job", "PeriodicRemove": false, "Cmd": "/home/xguo23/finally_2/Simulation_condor/chtcjobwrapper", "CondorVersion": "$CondorVersion: 8.5.0 Sep 16 2015 BuildID: 341710 $", "ShouldTransferFiles": "YES", "TargetType": "Machine",
"MinHosts": 1, "NumCkpts_RAW": 0, "RequestCpus": 1, "RemoteUserCpu": 26088.0d, "BlockWrites": 0, "NiceUser": false, "Out": "process.out", "ImageSize_RAW": 811948, "BytesSent": 30612.0d, "CumulativeSuspensionTime": 0, "TransferIn": false, "AutoClusterAttrs": "JobUniverse,LastCheckpointPlatform,NumCkpts,ClientMachine,_condor_RequestCpus,_condor_RequestDisk,_condor_RequestGPUs,_condor_RequestMemory,RequestCpus,RequestDisk,RequestGPUs,RequestMemory,BIOCHEM,MachineLastMatchTime,ConcurrencyLimits,NiceUser,Rank,Requirements,ImageSize,MemoryRequirements,User,RemoteGroup,SubmitterGroup,SubmitterUserPrio,Group,WIDsTheme,InteractiveJob,Is_Resumable,WantFlocking,WantGlidein,Scheduler,Owner,JobStart,MemoryUsage,IsExpressQueueJob,DiskUsage,HEP_VO,IsDesktop,OSG_VO,x509userproxysubject,PassedTest,IsLocalCMSJob,IsLocalCMSSlot,IsSAMSlot,IsSAMJob,MaxDiskTempC,IsDedicated,estimated_run_hours,IsCHTCSubmit,RequiresCVMFS,DynamicSlot,PartitionableSlot,Slot1_ExpectedMachineGracefulDrainingCompletion,Slot1_J
obStarts,Slot1_SelfMonitorAge,Slot2_ExpectedMachineGracefulDrainingCompletion,Slot2_JobStarts,Slot2_SelfMonitorAge,Slot3_ExpectedMachineGracefulDrainingCompletion,Slot3_JobStarts,Slot3_SelfMonitorAge,Slot4_ExpectedMachineGracefulDrainingCompletion,Slot4_JobStarts,Slot4_SelfMonitorAge,Slot5_ExpectedMachineGracefulDrainingCompletion,Slot5_JobStarts,Slot5_SelfMonitorAge,Slot6_ExpectedMachineGracefulDrainingCompletion,Slot6_JobStarts,Slot6_SelfMonitorAge,Slot7_ExpectedMachineGracefulDrainingCompletion,Slot7_JobStarts,Slot7_SelfMonitorAge,Slot8_ExpectedMachineGracefulDrainingCompletion,Slot8_JobStarts,Slot8_SelfMonitorAge,Slot1_TotalTimeClaimedBusy,Slot1_TotalTimeUnclaimedIdle,Slot2_TotalTimeClaimedBusy,Slot2_TotalTimeUnclaimedIdle,Slot3_TotalTimeClaimedBusy,Slot3_TotalTimeUnclaimedIdle,Slot4_TotalTimeClaimedBusy,Slot4_TotalTimeUnclaimedIdle,Slot5_TotalTimeClaimedBusy,Slot5_TotalTimeUnclaimedIdle,Slot6_TotalTimeClaimedBusy,Slot6_TotalTimeUnclaimedIdle,Slot7_TotalTimeClaimedBusy,Slot7_Tot
alTimeUnclaimedIdle,Slot8_TotalTimeClaimedBusy,Slot8_TotalTimeUnclaimedIdle,Slot10_ExpectedMachineGracefulDrainingCompletion,Slot10_JobStarts,Slot10_SelfMonitorAge,Slot11_ExpectedMachineGracefulDrainingCompletion,Slot11_JobStarts,Slot11_SelfMonitorAge,Slot12_ExpectedMachineGracefulDrainingCompletion,Slot12_JobStarts,Slot12_SelfMonitorAge,Slot9_ExpectedMachineGracefulDrainingCompletion,Slot9_JobStarts,Slot9_SelfMonitorAge,Slot12_TotalTimeClaimedBusy,Slot10_TotalTimeClaimedBusy,Slot10_TotalTimeUnclaimedIdle,Slot11_TotalTimeClaimedBusy,Slot11_TotalTimeUnclaimedIdle,Slot12_TotalTimeUnclaimedIdle,Slot9_TotalTimeClaimedBusy,Slot9_TotalTimeUnclaimedIdle,Slot13_ExpectedMachineGracefulDrainingCompletion,Slot13_JobStarts,Slot13_SelfMonitorAge,Slot14_ExpectedMachineGracefulDrainingCompletion,Slot14_JobStarts,Slot14_SelfMonitorAge,Slot15_ExpectedMachineGracefulDrainingCompletion,Slot15_JobStarts,Slot15_SelfMonitorAge,Slot16_ExpectedMachineGracefulDrainingCompletion,Slot16_JobStarts,Slot16_SelfM
onitorAge,IsResumable,WHEN_TO_TRANSFER_OUTPUT,_condor_Requestadmin_mutex_1,_condor_Requestadmin_mutex_2,_condor_Requestadmin_mutex_3,_condor_Requestmachine_token,Requestadmin_mutex_1,Requestadmin_mutex_2,Requestadmin_mutex_3,Requestmachine_token,nyehle,IsBuildJob,IsMatlabBuildJob,TotalJobRunTime,NodeOnline,Slot13_TotalTimeClaimedBusy,Slot13_TotalTimeUnclaimedIdle,Slot14_TotalTimeClaimedBusy,Slot14_TotalTimeUnclaimedIdle,Slot15_TotalTimeClaimedBusy,Slot15_TotalTimeUnclaimedIdle,Slot16_TotalTimeClaimedBusy,Slot16_TotalTimeUnclaimedIdle,TmpIsFull,trResumable,RequiresCMSFrontier,Slot17_ExpectedMachineGracefulDrainingCompletion,Slot17_JobStarts,Slot17_SelfMonitorAge,Slot17_TotalTimeClaimedBusy,Slot17_TotalTimeUnclaimedIdle,Slot18_ExpectedMachineGracefulDrainingCompletion,Slot18_JobStarts,Slot18_SelfMonitorAge,Slot18_TotalTimeClaimedBusy,Slot18_TotalTimeUnclaimedIdle,Slot19_ExpectedMachineGracefulDrainingCompletion,Slot19_JobStarts,Slot19_SelfMonitorAge,Slot19_TotalTimeClaimedBusy,Slot19_
TotalTimeUnclaimedIdle,Slot20_ExpectedMachineGracefulDrainingCompletion,Slot20_JobStarts,Slot20_SelfMonitorAge,Slot20_TotalTimeClaimedBusy,Slot20_TotalTimeUnclaimedIdle,Slot21_ExpectedMachineGracefulDrainingCompletion,Slot21_JobStarts,Slot21_SelfMonitorAge,Slot21_TotalTimeClaimedBusy,Slot21_TotalTimeUnclaimedIdle,Slot22_ExpectedMachineGracefulDrainingCompletion,Slot22_JobStarts,Slot22_SelfMonitorAge,Slot22_TotalTimeClaimedBusy,Slot22_TotalTimeUnclaimedIdle,Slot23_ExpectedMachineGracefulDrainingCompletion,Slot23_JobStarts,Slot23_SelfMonitorAge,Slot23_TotalTimeClaimedBusy,Slot23_TotalTimeUnclaimedIdle,Slot24_ExpectedMachineGracefulDrainingCompletion,Slot24_JobStarts,Slot24_SelfMonitorAge,Slot24_TotalTimeClaimedBusy,Slot24_TotalTimeUnclaimedIdle,Slot25_ExpectedMachineGracefulDrainingCompletion,Slot25_JobStarts,Slot25_SelfMonitorAge,Slot25_TotalTimeClaimedBusy,Slot25_TotalTimeUnclaimedIdle,Slot26_ExpectedMachineGracefulDrainingCompletion,Slot26_JobStarts,Slot26_SelfMonitorAge,Slot26_Tot
alTimeClaimedBusy,Slot26_TotalTimeUnclaimedIdle,Slot27_ExpectedMachineGracefulDrainingCompletion,Slot27_JobStarts,Slot27_SelfMonitorAge,Slot27_TotalTimeClaimedBusy,Slot27_TotalTimeUnclaimedIdle,Slot28_ExpectedMachineGracefulDrainingCompletion,Slot28_JobStarts,Slot28_SelfMonitorAge,Slot28_TotalTimeClaimedBusy,Slot28_TotalTimeUnclaimedIdle,Slot29_ExpectedMachineGracefulDrainingCompletion,Slot29_JobStarts,Slot29_SelfMonitorAge,Slot29_TotalTimeClaimedBusy,Slot29_TotalTimeUnclaimedIdle,Slot30_ExpectedMachineGracefulDrainingCompletion,Slot30_JobStarts,Slot30_SelfMonitorAge,Slot30_TotalTimeClaimedBusy,Slot30_TotalTimeUnclaimedIdle,Slot31_ExpectedMachineGracefulDrainingCompletion,Slot31_JobStarts,Slot31_SelfMonitorAge,Slot31_TotalTimeClaimedBusy,Slot31_TotalTimeUnclaimedIdle,Slot32_ExpectedMachineGracefulDrainingCompletion,Slot32_JobStarts,Slot32_SelfMonitorAge,Slot32_TotalTimeClaimedBusy,Slot32_TotalTimeUnclaimedIdle,ResidentSetSize", "NumCkpts": 0, "Err": "process.err", "RecentBlockWrites
": 0, "JobFinishedHookDone": 1446133919, "ProcId": 0, "ImageSize": 1000000, "JobUniverse": 5, "EncryptExecuteDirectory": false, "TransferInputSizeMB": 0, "RecentBlockReads": 0, "SpooledOutputFiles": "CURLTIME_1114551,ChtcWrapper162.out,AuditLog.162,simu_3_162.txt,harvest.log,162.out", "BlockWriteKbytes": 0, "WhenToTransferOutput": "ON_EXIT", "JobCurrentStartExecutingDate": 1446107688, "ExitBySignal": false, "LastMatchTime": 1446107686, "OnExitHold": false, "OrigMaxHosts": 1, "RequestMemory": 1000, "NumJobStarts": 1, "TerminationPending": true, "TotalSuspensions": 0, "BlockReads": 0, "DAGManJobId": 49581933, "MemoryUsage": 146, "ExitCode": 0, "JobNotification": 0, "BlockReadKbytes": 0, "NumJobMatches": 1, "LocalUserCpu": 0.0d, "LastJobStatus": 2, "BufferBlockSize": 32768, "CommittedTime": 26233, "ExecutableSize_RAW": 6, "LastSuspensionTime": 0, "Matlab": "R2011b", "UserLog": "/home/xguo23/finally_2/Simulation_condor/model_3/162/process.log", "DAGManNodesMask": "0,1,2,4,5,7,9,10,11,12
,13,16,17,24,27", "CumulativeSlotTime": 26233.0d, "LastJobLeaseRenewal": 1446133919, "MachineAttrSlotWeight0": 1, "NumSystemHolds": 0, "BytesRecvd": 285054.0d, "CondorPlatform": "$CondorPlatform: X86_64-RedHat_6.6 $", "JOBGLIDEIN_ResourceName": "$$([IfThenElse(IsUndefined(TARGET.GLIDEIN_ResourceName), IfThenElse(IsUndefined(TARGET.GLIDEIN_Site), \"wisc.edu\", TARGET.GLIDEIN_Site), TARGET.GLIDEIN_ResourceName)])", "DAGNodeName": "162+162", "PeriodicRelease": "error", "JobRunCount": 1, "LastRemoteHost": "slot1@e370.chtc.WISC.EDU", "JobPrio": 0, "LocalSysCpu": 0.0d, "ExecutableSize": 7, "RemoteSysCpu": 96.0d, "TransferInput": "/home/xguo23/finally_2/Simulation_condor/data/162/,/home/xguo23/finally_2/Simulation_condor/data/shared/", "PeriodicHold": false, "WantRemoteIO": true, "CommittedSuspensionTime": 0, "DAGParentNodeNames": "", "CompletionDate": 1446133919, "StreamErr": false, "RecentBlockReadKbytes": 0, "WantRemoteSyscalls": false, "NumShadowStarts": 1, "MachineAttrCpus0": 1, "DAGM
anNodesLog": "/home/xguo23/finally_2/Simulation_condor/model_3/./mydag.dag.nodes.log", "Owner": "xguo23", "Requirements": "undefined", "DiskUsage": 1250000, "JobLeaseDuration": 2400, "ClusterId": 49582148, "BufferSize": 524288, "IsCHTCSubmit": true, "RecentStatsLifetimeStarter": 1200, "CommittedSlotTime": 26233.0d, "Args": "--type=Matlab --version=R2011b --cmdtorun=simu_condor --unique=162 -- 3", "Environment": "", "LastPublicClaimId": "<128.105.245.170:9482>#1443991414#13008#...", "Iwd": "/home/xguo23/finally_2/Simulation_condor/model_3/162", "QDate": 1446105547, "CurrentHosts": 0, "User": "xguo23@chtc.wisc.edu", "StreamOut": false }
-{ "GlobalJobId": "submit-3.chtc.wisc.edu#49582154.0#1446105553", "StatsLifetimeStarter": 25874, "JobStartDate": 1446107686, "SubmitEventNotes": "DAG Node: 333+333", "JobStatus": 4, "LeaveJobInQueue": false, "StartdPrincipal": "execute-side@matchsession/128.105.245.120", "WantRHEL6": true, "OnExitRemove": true, "JobCurrentStartDate": 1446107686, "CoreSize": 0, "MATCH_EXP_JOBGLIDEIN_ResourceName": "wisc.edu", "Rank": 0.0d, "ExitStatus": 0, "ResidentSetSize": 150000, "RemoteWallClockTime": 25876.0d, "WantCheckpoint": false, "In": "/dev/null", "MaxHosts": 1, "RootDir": "/", "NumRestarts": 0, "RecentBlockWriteKbytes": 0, "DiskUsage_RAW": 1216669, "EnteredCurrentStatus": 1446133562, "ResidentSetSize_RAW": 125740, "RequestDisk": 1000000, "MyType": "Job", "PeriodicRemove": false, "Cmd": "/home/xguo23/finally_2/Simulation_condor/chtcjobwrapper", "CondorVersion": "$CondorVersion: 8.5.0 Sep 16 2015 BuildID: 341710 $", "ShouldTransferFiles": "YES", "TargetType": "Machine", "MinHosts": 1, "NumCk
pts_RAW": 0, "RequestCpus": 1, "RemoteUserCpu": 25692.0d, "BlockWrites": 0, "NiceUser": false, "Out": "process.out", "ImageSize_RAW": 811948, "BytesSent": 30542.0d, "CumulativeSuspensionTime": 0, "TransferIn": false, "NumCkpts": 0, "Err": "process.err", "RecentBlockWrites": 0, "JobFinishedHookDone": 1446133562, "ProcId": 0, "ImageSize": 1000000, "JobUniverse": 5, "EncryptExecuteDirectory": false, "TransferInputSizeMB": 0, "RecentBlockReads": 0, "SpooledOutputFiles": "CURLTIME_523030,ChtcWrapper333.out,AuditLog.333,simu_3_333.txt,harvest.log,333.out", "BlockWriteKbytes": 0, "WhenToTransferOutput": "ON_EXIT", "JobCurrentStartExecutingDate": 1446107688, "ExitBySignal": false, "LastMatchTime": 1446107686, "OnExitHold": false, "OrigMaxHosts": 1, "RequestMemory": 1000, "NumJobStarts": 1, "TerminationPending": true, "TotalSuspensions": 0, "BlockReads": 0, "DAGManJobId": 49581933, "MemoryUsage": 146, "ExitCode": 0, "JobNotification": 0, "BlockReadKbytes": 0, "NumJobMatches": 1, "LocalUserCp
u": 0.0d, "LastJobStatus": 2, "BufferBlockSize": 32768, "CommittedTime": 25876, "ExecutableSize_RAW": 6, "LastSuspensionTime": 0, "Matlab": "R2011b", "UserLog": "/home/xguo23/finally_2/Simulation_condor/model_3/333/process.log", "DAGManNodesMask": "0,1,2,4,5,7,9,10,11,12,13,16,17,24,27", "CumulativeSlotTime": 25876.0d, "LastJobLeaseRenewal": 1446133562, "MachineAttrSlotWeight0": 1, "NumSystemHolds": 0, "BytesRecvd": 285054.0d, "CondorPlatform": "$CondorPlatform: X86_64-RedHat_6.6 $", "JOBGLIDEIN_ResourceName": "$$([IfThenElse(IsUndefined(TARGET.GLIDEIN_ResourceName), IfThenElse(IsUndefined(TARGET.GLIDEIN_Site), \"wisc.edu\", TARGET.GLIDEIN_Site), TARGET.GLIDEIN_ResourceName)])", "DAGNodeName": "333+333", "PeriodicRelease": "error", "JobRunCount": 1, "LastRemoteHost": "slot1@e320.chtc.wisc.edu", "JobPrio": 0, "LocalSysCpu": 0.0d, "ExecutableSize": 7, "RemoteSysCpu": 157.0d, "TransferInput": "/home/xguo23/finally_2/Simulation_condor/data/333/,/home/xguo23/finally_2/Simulation_condor/d
ata/shared/", "PeriodicHold": false, "WantRemoteIO": true, "CommittedSuspensionTime": 0, "DAGParentNodeNames": "", "CompletionDate": 1446133562, "StreamErr": false, "RecentBlockReadKbytes": 0, "WantRemoteSyscalls": false, "NumShadowStarts": 1, "MachineAttrCpus0": 1, "DAGManNodesLog": "/home/xguo23/finally_2/Simulation_condor/model_3/./mydag.dag.nodes.log", "Owner": "xguo23", "Requirements": "undefined", "DiskUsage": 1250000, "JobLeaseDuration": 2400, "ClusterId": 49582154, "BufferSize": 524288, "IsCHTCSubmit": true, "RecentStatsLifetimeStarter": 1200, "CommittedSlotTime": 25876.0d, "Args": "--type=Matlab --version=R2011b --cmdtorun=simu_condor --unique=333 -- 3", "Environment": "", "LastPublicClaimId": "<128.105.245.120:45185>#1443991409#14242#...", "Iwd": "/home/xguo23/finally_2/Simulation_condor/model_3/333", "QDate": 1446105553, "CurrentHosts": 0, "User": "xguo23@chtc.wisc.edu", "StreamOut": false }
-{ "GlobalJobId": "submit-3.chtc.wisc.edu#49582177.0#1446105581", "StatsLifetimeStarter": 25025, "JobStartDate": 1446108665, "SubmitEventNotes": "DAG Node: 145+145", "JobStatus": 4, "LeaveJobInQueue": false, "StartdPrincipal": "execute-side@matchsession/128.104.55.57", "WantRHEL6": true, "OnExitRemove": true, "JobCurrentStartDate": 1446108665, "CoreSize": 0, "MATCH_EXP_JOBGLIDEIN_ResourceName": "wisc.edu", "Rank": 0.0d, "ExitStatus": 0, "ResidentSetSize": 75000, "RemoteWallClockTime": 25026.0d, "WantCheckpoint": false, "In": "/dev/null", "MaxHosts": 1, "RootDir": "/", "NumRestarts": 0, "RecentBlockWriteKbytes": 4, "DiskUsage_RAW": 1216669, "EnteredCurrentStatus": 1446133691, "ResidentSetSize_RAW": 73308, "RequestDisk": 1000000, "MyType": "Job", "PeriodicRemove": false, "Cmd": "/home/xguo23/finally_2/Simulation_condor/chtcjobwrapper", "CondorVersion": "$CondorVersion: 8.5.0 Sep 16 2015 BuildID: 341710 $", "ShouldTransferFiles": "YES", "TargetType": "Machine", "MinHosts": 1, "NumCkpts_
RAW": 0, "RequestCpus": 1, "RemoteUserCpu": 24770.0d, "BlockWrites": 1, "NiceUser": false, "Out": "process.out", "ImageSize_RAW": 120972, "BytesSent": 28290.0d, "CumulativeSuspensionTime": 0, "TransferIn": false, "NumCkpts": 0, "Err": "process.err", "RecentBlockWrites": 1, "JobFinishedHookDone": 1446133691, "ProcId": 0, "ImageSize": 125000, "JobUniverse": 5, "EncryptExecuteDirectory": false, "TransferInputSizeMB": 0, "RecentBlockReads": 146, "SpooledOutputFiles": "CURLTIME_4179033,ChtcWrapper145.out,AuditLog.145,simu_3_145.txt,harvest.log,145.out", "BlockWriteKbytes": 4, "WhenToTransferOutput": "ON_EXIT", "JobCurrentStartExecutingDate": 1446108666, "ExitBySignal": false, "LastMatchTime": 1446108665, "OnExitHold": false, "OrigMaxHosts": 1, "RequestMemory": 1000, "NumJobStarts": 1, "TerminationPending": true, "TotalSuspensions": 0, "BlockReads": 796, "DAGManJobId": 49581933, "MemoryUsage": 73, "ExitCode": 0, "JobNotification": 0, "BlockReadKbytes": 28476, "NumJobMatches": 1, "LocalUse
rCpu": 0.0d, "LastJobStatus": 2, "BufferBlockSize": 32768, "CommittedTime": 25026, "ExecutableSize_RAW": 6, "LastSuspensionTime": 0, "Matlab": "R2011b", "UserLog": "/home/xguo23/finally_2/Simulation_condor/model_3/145/process.log", "DAGManNodesMask": "0,1,2,4,5,7,9,10,11,12,13,16,17,24,27", "CumulativeSlotTime": 25026.0d, "LastJobLeaseRenewal": 1446133691, "MachineAttrSlotWeight0": 1, "NumSystemHolds": 0, "BytesRecvd": 285054.0d, "CondorPlatform": "$CondorPlatform: X86_64-RedHat_6.6 $", "JOBGLIDEIN_ResourceName": "$$([IfThenElse(IsUndefined(TARGET.GLIDEIN_ResourceName), IfThenElse(IsUndefined(TARGET.GLIDEIN_Site), \"wisc.edu\", TARGET.GLIDEIN_Site), TARGET.GLIDEIN_ResourceName)])", "DAGNodeName": "145+145", "PeriodicRelease": "error", "JobRunCount": 1, "LastRemoteHost": "slot1@c038.chtc.wisc.edu", "JobPrio": 0, "LocalSysCpu": 0.0d, "ExecutableSize": 7, "RemoteSysCpu": 217.0d, "TransferInput": "/home/xguo23/finally_2/Simulation_condor/data/145/,/home/xguo23/finally_2/Simulation_condo
r/data/shared/", "PeriodicHold": false, "WantRemoteIO": true, "CommittedSuspensionTime": 0, "DAGParentNodeNames": "", "CompletionDate": 1446133691, "StreamErr": false, "RecentBlockReadKbytes": 1932, "WantRemoteSyscalls": false, "NumShadowStarts": 1, "MachineAttrCpus0": 1, "DAGManNodesLog": "/home/xguo23/finally_2/Simulation_condor/model_3/./mydag.dag.nodes.log", "Owner": "xguo23", "Requirements": "undefined", "DiskUsage": 1250000, "JobLeaseDuration": 2400, "ClusterId": 49582177, "BufferSize": 524288, "IsCHTCSubmit": true, "RecentStatsLifetimeStarter": 1200, "CommittedSlotTime": 25026.0d, "Args": "--type=Matlab --version=R2011b --cmdtorun=simu_condor --unique=145 -- 3", "Environment": "", "LastPublicClaimId": "<128.104.55.57:49793>#1445322694#1541#...", "Iwd": "/home/xguo23/finally_2/Simulation_condor/model_3/145", "QDate": 1446105581, "CurrentHosts": 0, "User": "xguo23@chtc.wisc.edu", "StreamOut": false }
-{ "GlobalJobId": "submit-3.chtc.wisc.edu#49582178.0#1446105581", "StatsLifetimeStarter": 24871, "JobStartDate": 1446108666, "SubmitEventNotes": "DAG Node: 154+154", "JobStatus": 4, "LeaveJobInQueue": false, "StartdPrincipal": "execute-side@matchsession/128.105.245.158", "WantRHEL6": true, "OnExitRemove": true, "JobCurrentStartDate": 1446108666, "CoreSize": 0, "MATCH_EXP_JOBGLIDEIN_ResourceName": "wisc.edu", "Rank": 0.0d, "ExitStatus": 0, "ResidentSetSize": 150000, "RemoteWallClockTime": 24874.0d, "WantCheckpoint": false, "In": "/dev/null", "MaxHosts": 1, "RootDir": "/", "NumRestarts": 0, "RecentBlockWriteKbytes": 0, "DiskUsage_RAW": 1216669, "EnteredCurrentStatus": 1446133540, "ResidentSetSize_RAW": 125792, "RequestDisk": 1000000, "MyType": "Job", "PeriodicRemove": false, "Cmd": "/home/xguo23/finally_2/Simulation_condor/chtcjobwrapper", "CondorVersion": "$CondorVersion: 8.5.0 Sep 16 2015 BuildID: 341710 $", "ShouldTransferFiles": "YES", "TargetType": "Machine", "MinHosts": 1, "NumCk
pts_RAW": 0, "RequestCpus": 1, "RemoteUserCpu": 24626.0d, "BlockWrites": 0, "NiceUser": false, "Out": "process.out", "ImageSize_RAW": 811948, "BytesSent": 30559.0d, "CumulativeSuspensionTime": 0, "TransferIn": false, "NumCkpts": 0, "Err": "process.err", "RecentBlockWrites": 0, "JobFinishedHookDone": 1446133540, "ProcId": 0, "ImageSize": 1000000, "JobUniverse": 5, "EncryptExecuteDirectory": false, "TransferInputSizeMB": 0, "RecentBlockReads": 0, "SpooledOutputFiles": "CURLTIME_1382128,ChtcWrapper154.out,AuditLog.154,simu_3_154.txt,harvest.log,154.out", "BlockWriteKbytes": 0, "WhenToTransferOutput": "ON_EXIT", "JobCurrentStartExecutingDate": 1446108668, "ExitBySignal": false, "LastMatchTime": 1446108666, "OnExitHold": false, "OrigMaxHosts": 1, "RequestMemory": 1000, "NumJobStarts": 1, "TerminationPending": true, "TotalSuspensions": 0, "BlockReads": 0, "DAGManJobId": 49581933, "MemoryUsage": 146, "ExitCode": 0, "JobNotification": 0, "BlockReadKbytes": 0, "NumJobMatches": 1, "LocalUserC
pu": 0.0d, "LastJobStatus": 2, "BufferBlockSize": 32768, "CommittedTime": 24874, "ExecutableSize_RAW": 6, "LastSuspensionTime": 0, "Matlab": "R2011b", "UserLog": "/home/xguo23/finally_2/Simulation_condor/model_3/154/process.log", "DAGManNodesMask": "0,1,2,4,5,7,9,10,11,12,13,16,17,24,27", "CumulativeSlotTime": 24874.0d, "LastJobLeaseRenewal": 1446133540, "MachineAttrSlotWeight0": 1, "NumSystemHolds": 0, "BytesRecvd": 285054.0d, "CondorPlatform": "$CondorPlatform: X86_64-RedHat_6.6 $", "JOBGLIDEIN_ResourceName": "$$([IfThenElse(IsUndefined(TARGET.GLIDEIN_ResourceName), IfThenElse(IsUndefined(TARGET.GLIDEIN_Site), \"wisc.edu\", TARGET.GLIDEIN_Site), TARGET.GLIDEIN_ResourceName)])", "DAGNodeName": "154+154", "PeriodicRelease": "error", "JobRunCount": 1, "LastRemoteHost": "slot1@e358.chtc.wisc.edu", "JobPrio": 0, "LocalSysCpu": 0.0d, "ExecutableSize": 7, "RemoteSysCpu": 183.0d, "TransferInput": "/home/xguo23/finally_2/Simulation_condor/data/154/,/home/xguo23/finally_2/Simulation_condor/
data/shared/", "PeriodicHold": false, "WantRemoteIO": true, "CommittedSuspensionTime": 0, "DAGParentNodeNames": "", "CompletionDate": 1446133540, "StreamErr": false, "RecentBlockReadKbytes": 0, "WantRemoteSyscalls": false, "NumShadowStarts": 1, "MachineAttrCpus0": 1, "DAGManNodesLog": "/home/xguo23/finally_2/Simulation_condor/model_3/./mydag.dag.nodes.log", "Owner": "xguo23", "Requirements": "undefined", "DiskUsage": 1250000, "JobLeaseDuration": 2400, "ClusterId": 49582178, "BufferSize": 524288, "IsCHTCSubmit": true, "RecentStatsLifetimeStarter": 1200, "CommittedSlotTime": 24874.0d, "Args": "--type=Matlab --version=R2011b --cmdtorun=simu_condor --unique=154 -- 3", "Environment": "", "LastPublicClaimId": "<128.105.245.158:24962>#1444759998#9379#...", "Iwd": "/home/xguo23/finally_2/Simulation_condor/model_3/154", "QDate": 1446105581, "CurrentHosts": 0, "User": "xguo23@chtc.wisc.edu", "StreamOut": false }
-{ "GlobalJobId": "submit-3.chtc.wisc.edu#49582181.0#1446105586", "StatsLifetimeStarter": 25146, "JobStartDate": 1446108665, "SubmitEventNotes": "DAG Node: 181+181", "JobStatus": 4, "LeaveJobInQueue": false, "StartdPrincipal": "execute-side@matchsession/128.105.245.102", "WantRHEL6": true, "OnExitRemove": true, "JobCurrentStartDate": 1446108665, "CoreSize": 0, "MATCH_EXP_JOBGLIDEIN_ResourceName": "wisc.edu", "Rank": 0.0d, "ExitStatus": 0, "ResidentSetSize": 150000, "RemoteWallClockTime": 25148.0d, "WantCheckpoint": false, "In": "/dev/null", "MaxHosts": 1, "RootDir": "/", "NumRestarts": 0, "RecentBlockWriteKbytes": 0, "DiskUsage_RAW": 1216669, "EnteredCurrentStatus": 1446133813, "ResidentSetSize_RAW": 125368, "RequestDisk": 1000000, "MyType": "Job", "PeriodicRemove": false, "Cmd": "/home/xguo23/finally_2/Simulation_condor/chtcjobwrapper", "CondorVersion": "$CondorVersion: 8.5.0 Sep 16 2015 BuildID: 341710 $", "ShouldTransferFiles": "YES", "TargetType": "Machine", "MinHosts": 1, "NumCk
pts_RAW": 0, "RequestCpus": 1, "RemoteUserCpu": 24957.0d, "BlockWrites": 0, "NiceUser": false, "Out": "process.out", "ImageSize_RAW": 811948, "BytesSent": 30557.0d, "CumulativeSuspensionTime": 0, "TransferIn": false, "NumCkpts": 0, "Err": "process.err", "RecentBlockWrites": 0, "JobFinishedHookDone": 1446133813, "ProcId": 0, "ImageSize": 1000000, "JobUniverse": 5, "EncryptExecuteDirectory": false, "TransferInputSizeMB": 0, "RecentBlockReads": 0, "SpooledOutputFiles": "CURLTIME_3925831,ChtcWrapper181.out,AuditLog.181,simu_3_181.txt,harvest.log,181.out", "BlockWriteKbytes": 0, "WhenToTransferOutput": "ON_EXIT", "JobCurrentStartExecutingDate": 1446108667, "ExitBySignal": false, "LastMatchTime": 1446108665, "OnExitHold": false, "OrigMaxHosts": 1, "RequestMemory": 1000, "NumJobStarts": 1, "TerminationPending": true, "TotalSuspensions": 0, "BlockReads": 0, "DAGManJobId": 49581933, "MemoryUsage": 146, "ExitCode": 0, "JobNotification": 0, "BlockReadKbytes": 0, "NumJobMatches": 1, "LocalUserC
pu": 0.0d, "LastJobStatus": 2, "BufferBlockSize": 32768, "CommittedTime": 25148, "ExecutableSize_RAW": 6, "LastSuspensionTime": 0, "Matlab": "R2011b", "UserLog": "/home/xguo23/finally_2/Simulation_condor/model_3/181/process.log", "DAGManNodesMask": "0,1,2,4,5,7,9,10,11,12,13,16,17,24,27", "CumulativeSlotTime": 25148.0d, "LastJobLeaseRenewal": 1446133813, "MachineAttrSlotWeight0": 1, "NumSystemHolds": 0, "BytesRecvd": 285054.0d, "CondorPlatform": "$CondorPlatform: X86_64-RedHat_6.6 $", "JOBGLIDEIN_ResourceName": "$$([IfThenElse(IsUndefined(TARGET.GLIDEIN_ResourceName), IfThenElse(IsUndefined(TARGET.GLIDEIN_Site), \"wisc.edu\", TARGET.GLIDEIN_Site), TARGET.GLIDEIN_ResourceName)])", "DAGNodeName": "181+181", "PeriodicRelease": "error", "JobRunCount": 1, "LastRemoteHost": "slot1@e302.chtc.wisc.edu", "JobPrio": 0, "LocalSysCpu": 0.0d, "ExecutableSize": 7, "RemoteSysCpu": 148.0d, "TransferInput": "/home/xguo23/finally_2/Simulation_condor/data/181/,/home/xguo23/finally_2/Simulation_condor/
data/shared/", "PeriodicHold": false, "WantRemoteIO": true, "CommittedSuspensionTime": 0, "DAGParentNodeNames": "", "CompletionDate": 1446133813, "StreamErr": false, "RecentBlockReadKbytes": 0, "WantRemoteSyscalls": false, "NumShadowStarts": 1, "MachineAttrCpus0": 1, "DAGManNodesLog": "/home/xguo23/finally_2/Simulation_condor/model_3/./mydag.dag.nodes.log", "Owner": "xguo23", "Requirements": "undefined", "DiskUsage": 1250000, "JobLeaseDuration": 2400, "ClusterId": 49582181, "BufferSize": 524288, "IsCHTCSubmit": true, "RecentStatsLifetimeStarter": 1200, "CommittedSlotTime": 25148.0d, "Args": "--type=Matlab --version=R2011b --cmdtorun=simu_condor --unique=181 -- 3", "Environment": "", "LastPublicClaimId": "<128.105.245.102:26944>#1443991374#13401#...", "Iwd": "/home/xguo23/finally_2/Simulation_condor/model_3/181", "QDate": 1446105586, "CurrentHosts": 0, "User": "xguo23@chtc.wisc.edu", "StreamOut": false }
-{ "GlobalJobId": "submit-3.chtc.wisc.edu#49582187.0#1446105592", "StatsLifetimeStarter": 25238, "JobStartDate": 1446108666, "SubmitEventNotes": "DAG Node: 343+343", "JobStatus": 4, "LeaveJobInQueue": false, "AutoClusterId": 24, "StartdPrincipal": "execute-side@matchsession/128.105.245.141", "WantRHEL6": true, "OnExitRemove": true, "JobCurrentStartDate": 1446108666, "CoreSize": 0, "MATCH_EXP_JOBGLIDEIN_ResourceName": "wisc.edu", "ExitStatus": 0, "Rank": 0.0d, "ResidentSetSize": 150000, "RemoteWallClockTime": 25241.0d, "WantCheckpoint": false, "In": "/dev/null", "MaxHosts": 1, "RootDir": "/", "NumRestarts": 0, "RecentBlockWriteKbytes": 0, "DiskUsage_RAW": 1216669, "EnteredCurrentStatus": 1446133907, "ResidentSetSize_RAW": 127540, "RequestDisk": 1000000, "MyType": "Job", "PeriodicRemove": false, "Cmd": "/home/xguo23/finally_2/Simulation_condor/chtcjobwrapper", "CondorVersion": "$CondorVersion: 8.5.0 Sep 16 2015 BuildID: 341710 $", "ShouldTransferFiles": "YES", "TargetType": "Machine",
"MinHosts": 1, "NumCkpts_RAW": 0, "RequestCpus": 1, "RemoteUserCpu": 25080.0d, "BlockWrites": 0, "NiceUser": false, "Out": "process.out", "ImageSize_RAW": 811948, "BytesSent": 31798.0d, "CumulativeSuspensionTime": 0, "TransferIn": false, "AutoClusterAttrs": "JobUniverse,LastCheckpointPlatform,NumCkpts,ClientMachine,_condor_RequestCpus,_condor_RequestDisk,_condor_RequestGPUs,_condor_RequestMemory,RequestCpus,RequestDisk,RequestGPUs,RequestMemory,BIOCHEM,MachineLastMatchTime,ConcurrencyLimits,NiceUser,Rank,Requirements,ImageSize,MemoryRequirements,User,RemoteGroup,SubmitterGroup,SubmitterUserPrio,Group,WIDsTheme,InteractiveJob,Is_Resumable,WantFlocking,WantGlidein,Scheduler,Owner,JobStart,MemoryUsage,IsExpressQueueJob,DiskUsage,HEP_VO,IsDesktop,OSG_VO,x509userproxysubject,PassedTest,IsLocalCMSJob,IsLocalCMSSlot,IsSAMSlot,IsSAMJob,MaxDiskTempC,IsDedicated,estimated_run_hours,IsCHTCSubmit,RequiresCVMFS,DynamicSlot,PartitionableSlot,Slot1_ExpectedMachineGracefulDrainingCompletion,Slot1_J
obStarts,Slot1_SelfMonitorAge,Slot2_ExpectedMachineGracefulDrainingCompletion,Slot2_JobStarts,Slot2_SelfMonitorAge,Slot3_ExpectedMachineGracefulDrainingCompletion,Slot3_JobStarts,Slot3_SelfMonitorAge,Slot4_ExpectedMachineGracefulDrainingCompletion,Slot4_JobStarts,Slot4_SelfMonitorAge,Slot5_ExpectedMachineGracefulDrainingCompletion,Slot5_JobStarts,Slot5_SelfMonitorAge,Slot6_ExpectedMachineGracefulDrainingCompletion,Slot6_JobStarts,Slot6_SelfMonitorAge,Slot7_ExpectedMachineGracefulDrainingCompletion,Slot7_JobStarts,Slot7_SelfMonitorAge,Slot8_ExpectedMachineGracefulDrainingCompletion,Slot8_JobStarts,Slot8_SelfMonitorAge,Slot1_TotalTimeClaimedBusy,Slot1_TotalTimeUnclaimedIdle,Slot2_TotalTimeClaimedBusy,Slot2_TotalTimeUnclaimedIdle,Slot3_TotalTimeClaimedBusy,Slot3_TotalTimeUnclaimedIdle,Slot4_TotalTimeClaimedBusy,Slot4_TotalTimeUnclaimedIdle,Slot5_TotalTimeClaimedBusy,Slot5_TotalTimeUnclaimedIdle,Slot6_TotalTimeClaimedBusy,Slot6_TotalTimeUnclaimedIdle,Slot7_TotalTimeClaimedBusy,Slot7_Tot
alTimeUnclaimedIdle,Slot8_TotalTimeClaimedBusy,Slot8_TotalTimeUnclaimedIdle,Slot10_ExpectedMachineGracefulDrainingCompletion,Slot10_JobStarts,Slot10_SelfMonitorAge,Slot11_ExpectedMachineGracefulDrainingCompletion,Slot11_JobStarts,Slot11_SelfMonitorAge,Slot12_ExpectedMachineGracefulDrainingCompletion,Slot12_JobStarts,Slot12_SelfMonitorAge,Slot9_ExpectedMachineGracefulDrainingCompletion,Slot9_JobStarts,Slot9_SelfMonitorAge,Slot12_TotalTimeClaimedBusy,Slot10_TotalTimeClaimedBusy,Slot10_TotalTimeUnclaimedIdle,Slot11_TotalTimeClaimedBusy,Slot11_TotalTimeUnclaimedIdle,Slot12_TotalTimeUnclaimedIdle,Slot9_TotalTimeClaimedBusy,Slot9_TotalTimeUnclaimedIdle,Slot13_ExpectedMachineGracefulDrainingCompletion,Slot13_JobStarts,Slot13_SelfMonitorAge,Slot14_ExpectedMachineGracefulDrainingCompletion,Slot14_JobStarts,Slot14_SelfMonitorAge,Slot15_ExpectedMachineGracefulDrainingCompletion,Slot15_JobStarts,Slot15_SelfMonitorAge,Slot16_ExpectedMachineGracefulDrainingCompletion,Slot16_JobStarts,Slot16_SelfM
onitorAge,IsResumable,WHEN_TO_TRANSFER_OUTPUT,_condor_Requestadmin_mutex_1,_condor_Requestadmin_mutex_2,_condor_Requestadmin_mutex_3,_condor_Requestmachine_token,Requestadmin_mutex_1,Requestadmin_mutex_2,Requestadmin_mutex_3,Requestmachine_token,nyehle,IsBuildJob,IsMatlabBuildJob,TotalJobRunTime,NodeOnline,Slot13_TotalTimeClaimedBusy,Slot13_TotalTimeUnclaimedIdle,Slot14_TotalTimeClaimedBusy,Slot14_TotalTimeUnclaimedIdle,Slot15_TotalTimeClaimedBusy,Slot15_TotalTimeUnclaimedIdle,Slot16_TotalTimeClaimedBusy,Slot16_TotalTimeUnclaimedIdle,TmpIsFull,trResumable,RequiresCMSFrontier,Slot17_ExpectedMachineGracefulDrainingCompletion,Slot17_JobStarts,Slot17_SelfMonitorAge,Slot17_
<TRUNCATED>
[12/13] incubator-asterixdb git commit: Improve Error Handling in
Local Directory Feeds
Posted by am...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-app/src/test/resources/runtimets/results/external-library/classad-parser/classad-parser.1.adm
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/results/external-library/classad-parser/classad-parser.1.adm b/asterix-app/src/test/resources/runtimets/results/external-library/classad-parser/classad-parser.1.adm
index 858285e..cabaa67 100644
--- a/asterix-app/src/test/resources/runtimets/results/external-library/classad-parser/classad-parser.1.adm
+++ b/asterix-app/src/test/resources/runtimets/results/external-library/classad-parser/classad-parser.1.adm
@@ -1,100 +1,100 @@
-{ "GlobalJobId": "submit-3.chtc.wisc.edu#48968872.0#1445354636", "StatsLifetimeStarter": 572059, "JobStartDate": 1445362267, "SubmitEventNotes": "DAG Node: fabp4-0002+fabp4-0002", "JobStatus": 4, "LeaveJobInQueue": false, "WantGlidein": true, "StartdPrincipal": "execute-side@matchsession/128.104.119.175", "WantRHEL6": true, "OnExitRemove": true, "JobCurrentStartDate": 1445561276, "CoreSize": 0, "MATCH_EXP_JOBGLIDEIN_ResourceName": "wisc.edu", "Rank": 0.0d, "ExitStatus": 0, "ResidentSetSize": 125000, "ScheddBday": 1445383086, "RemoteWallClockTime": 769511.0d, "WantCheckpoint": false, "In": "/dev/null", "LastVacateTime": 1445546251, "MaxHosts": 1, "RootDir": "/", "NumRestarts": 0, "RecentBlockWriteKbytes": 0, "DiskUsage_RAW": 32543, "EnteredCurrentStatus": 1446133322, "ResidentSetSize_RAW": 100432, "RequestDisk": 1000000, "MyType": "Job", "PeriodicRemove": false, "Cmd": "/home/ssericksen/dude-14-xdock/ChtcRun/chtcjobwrapper", "CondorVersion": "$CondorVersion: 8.5.0 Sep 16 2015 BuildID
: 341710 $", "ShouldTransferFiles": "YES", "TargetType": "Machine", "MinHosts": 1, "NumCkpts_RAW": 0, "RequestCpus": 1, "RemoteUserCpu": 571737.0d, "BlockWrites": 0, "NiceUser": false, "Out": "process.out", "ImageSize_RAW": 690056, "BytesSent": 3113566.0d, "CumulativeSuspensionTime": 0, "TransferIn": false, "NumCkpts": 0, "Err": "process.err", "RecentBlockWrites": 0, "JobFinishedHookDone": 1446133322, "ProcId": 0, "ImageSize": 750000, "JobUniverse": 5, "EncryptExecuteDirectory": false, "TransferInputSizeMB": 12, "RecentBlockReads": 0, "SpooledOutputFiles": "ChtcWrapperfabp4-0002.out,AuditLog.fabp4-0002,poses.mol2,CURLTIME_4057178,harvest.log,time_elapsed.log,surf_scores.txt,CURLTIME_38803,count.log,fabp4-0002.out,CURLTIME_253463", "NumJobReconnects": 1, "WantFlocking": true, "BlockWriteKbytes": 0, "WhenToTransferOutput": "ON_EXIT_OR_EVICT", "JobCurrentStartExecutingDate": 1445561278, "ExitBySignal": false, "LastMatchTime": 1445561276, "OnExitHold": false, "OrigMaxHosts": 1, "Request
Memory": 1000, "NumJobStarts": 6, "TerminationPending": true, "TotalSuspensions": 0, "BlockReads": 0, "DAGManJobId": 48940805, "MemoryUsage": 122, "ExitCode": 0, "JobNotification": 0, "BlockReadKbytes": 0, "NumJobMatches": 6, "LocalUserCpu": 0.0d, "LastJobStatus": 2, "BufferBlockSize": 32768, "CommittedTime": 572046, "ExecutableSize_RAW": 6, "LastRejMatchReason": "no match found", "LastSuspensionTime": 0, "UserLog": "/home/ssericksen/dude-14-xdock/ChtcRun/dude14-surf-out-esr1/fabp4-0002/process.log", "DAGManNodesMask": "0,1,2,4,5,7,9,10,11,12,13,16,17,24,27", "CumulativeSlotTime": 769511.0d, "LastJobLeaseRenewal": 1446133322, "MachineAttrSlotWeight0": 1, "NumSystemHolds": 0, "BytesRecvd": 8.7351688E7d, "CondorPlatform": "$CondorPlatform: X86_64-RedHat_6.6 $", "JOBGLIDEIN_ResourceName": "$$([IfThenElse(IsUndefined(TARGET.GLIDEIN_ResourceName), IfThenElse(IsUndefined(TARGET.GLIDEIN_Site), \"wisc.edu\", TARGET.GLIDEIN_Site), TARGET.GLIDEIN_ResourceName)])", "DAGNodeName": "fabp4-0002+f
abp4-0002", "PeriodicRelease": "error", "JobRunCount": 7, "LastRemotePool": "condor.biochem.wisc.edu:9618?sock=collector", "JobLastStartDate": 1445546257, "LastRemoteHost": "slot1@cluster-0008.biochem.wisc.edu", "JobPrio": 0, "LocalSysCpu": 0.0d, "ExecutableSize": 7, "RemoteSysCpu": 0.0d, "TransferInput": "/home/ssericksen/dude-14-xdock/ChtcRun/dude14-surf-in/fabp4-0002/,/home/ssericksen/dude-14-xdock/ChtcRun/dude14-surf-in/shared/", "PeriodicHold": false, "WantRemoteIO": true, "CommittedSuspensionTime": 0, "DAGParentNodeNames": "", "CompletionDate": 1446133322, "StreamErr": false, "is_resumable": true, "RecentBlockReadKbytes": 0, "WantRemoteSyscalls": false, "NumShadowStarts": 7, "MachineAttrCpus0": 1, "DAGManNodesLog": "/home/ssericksen/dude-14-xdock/ChtcRun/dude14-surf-out-esr1/./mydag.dag.nodes.log", "Owner": "ssericksen", "Requirements": "undefined", "DiskUsage": 35000, "LastRejMatchTime": 1445375317, "JobLeaseDuration": 2400, "ClusterId": 48968872, "BufferSize": 524288, "IsCHT
CSubmit": true, "RecentStatsLifetimeStarter": 1200, "CommittedSlotTime": 572046.0d, "Args": "--type=Other --cmdtorun=surflex_run_DUDE_v1.8_esr1.sh --unique=fabp4-0002 --", "Environment": "", "LastPublicClaimId": "<128.104.119.175:9618>#1444067179#3317#...", "Iwd": "/home/ssericksen/dude-14-xdock/ChtcRun/dude14-surf-out-esr1/fabp4-0002", "QDate": 1445354636, "CurrentHosts": 0, "User": "ssericksen@chtc.wisc.edu", "StreamOut": false }
-{ "GlobalJobId": "submit-3.chtc.wisc.edu#49573720.0#1445938922", "StatsLifetimeStarter": 190245, "JobStartDate": 1445943852, "SubmitEventNotes": "DAG Node: 180+180", "JobStatus": 4, "LeaveJobInQueue": false, "StartdPrincipal": "execute-side@matchsession/128.105.245.72", "WantRHEL6": true, "OnExitRemove": true, "JobCurrentStartDate": 1445943852, "CoreSize": 0, "MATCH_EXP_JOBGLIDEIN_ResourceName": "wisc.edu", "Rank": 0.0d, "ExitStatus": 0, "ResidentSetSize": 125000, "RemoteWallClockTime": 190247.0d, "WantCheckpoint": false, "In": "/dev/null", "MaxHosts": 1, "RootDir": "/", "NumRestarts": 0, "RecentBlockWriteKbytes": 0, "DiskUsage_RAW": 1216669, "EnteredCurrentStatus": 1446134099, "ResidentSetSize_RAW": 123680, "RequestDisk": 1000000, "MyType": "Job", "PeriodicRemove": false, "Cmd": "/home/xguo23/finally/Simulation_condor/chtcjobwrapper", "CondorVersion": "$CondorVersion: 8.5.0 Sep 16 2015 BuildID: 341710 $", "ShouldTransferFiles": "YES", "TargetType": "Machine", "MinHosts": 1, "NumCkp
ts_RAW": 0, "RequestCpus": 1, "RemoteUserCpu": 185236.0d, "BlockWrites": 0, "NiceUser": false, "Out": "process.out", "ImageSize_RAW": 811948, "BytesSent": 30766.0d, "CumulativeSuspensionTime": 0, "TransferIn": false, "NumCkpts": 0, "Err": "process.err", "RecentBlockWrites": 0, "JobFinishedHookDone": 1446134099, "ProcId": 0, "ImageSize": 1000000, "JobUniverse": 5, "EncryptExecuteDirectory": false, "TransferInputSizeMB": 0, "RecentBlockReads": 0, "SpooledOutputFiles": "CURLTIME_3753852,ChtcWrapper180.out,AuditLog.180,simu_3_180.txt,harvest.log,180.out", "BlockWriteKbytes": 0, "WhenToTransferOutput": "ON_EXIT", "JobCurrentStartExecutingDate": 1445943853, "ExitBySignal": false, "LastMatchTime": 1445943852, "OnExitHold": false, "OrigMaxHosts": 1, "RequestMemory": 1000, "NumJobStarts": 1, "TerminationPending": true, "TotalSuspensions": 0, "BlockReads": 0, "DAGManJobId": 49572657, "MemoryUsage": 122, "ExitCode": 0, "JobNotification": 0, "BlockReadKbytes": 0, "NumJobMatches": 1, "LocalUserC
pu": 0.0d, "LastJobStatus": 2, "BufferBlockSize": 32768, "CommittedTime": 190247, "ExecutableSize_RAW": 6, "LastSuspensionTime": 0, "Matlab": "R2011b", "UserLog": "/home/xguo23/finally/Simulation_condor/model_3/180/process.log", "DAGManNodesMask": "0,1,2,4,5,7,9,10,11,12,13,16,17,24,27", "CumulativeSlotTime": 190247.0d, "LastJobLeaseRenewal": 1446134099, "MachineAttrSlotWeight0": 1, "NumSystemHolds": 0, "BytesRecvd": 284367.0d, "CondorPlatform": "$CondorPlatform: X86_64-RedHat_6.6 $", "JOBGLIDEIN_ResourceName": "$$([IfThenElse(IsUndefined(TARGET.GLIDEIN_ResourceName), IfThenElse(IsUndefined(TARGET.GLIDEIN_Site), \"wisc.edu\", TARGET.GLIDEIN_Site), TARGET.GLIDEIN_ResourceName)])", "DAGNodeName": "180+180", "PeriodicRelease": "error", "JobRunCount": 1, "LastRemoteHost": "slot1@e272.chtc.wisc.edu", "JobPrio": 0, "LocalSysCpu": 0.0d, "ExecutableSize": 7, "RemoteSysCpu": 1835.0d, "TransferInput": "/home/xguo23/finally/Simulation_condor/data/180/,/home/xguo23/finally/Simulation_condor/dat
a/shared/", "PeriodicHold": false, "WantRemoteIO": true, "CommittedSuspensionTime": 0, "DAGParentNodeNames": "", "CompletionDate": 1446134099, "StreamErr": false, "RecentBlockReadKbytes": 0, "WantRemoteSyscalls": false, "NumShadowStarts": 1, "MachineAttrCpus0": 1, "DAGManNodesLog": "/home/xguo23/finally/Simulation_condor/model_3/./mydag.dag.nodes.log", "Owner": "xguo23", "Requirements": "undefined", "DiskUsage": 1250000, "JobLeaseDuration": 2400, "ClusterId": 49573720, "BufferSize": 524288, "IsCHTCSubmit": true, "RecentStatsLifetimeStarter": 1200, "CommittedSlotTime": 190247.0d, "Args": "--type=Matlab --version=R2011b --cmdtorun=simu_condor --unique=180 -- 3", "Environment": "", "LastPublicClaimId": "<128.105.245.72:29075>#1444753997#6000#...", "Iwd": "/home/xguo23/finally/Simulation_condor/model_3/180", "QDate": 1445938922, "CurrentHosts": 0, "User": "xguo23@chtc.wisc.edu", "StreamOut": false }
-{ "GlobalJobId": "submit-3.chtc.wisc.edu#49581952.0#1446105329", "StatsLifetimeStarter": 27674, "JobStartDate": 1446106061, "SubmitEventNotes": "DAG Node: 40+40", "JobStatus": 4, "LeaveJobInQueue": false, "StartdPrincipal": "execute-side@matchsession/128.105.245.86", "WantRHEL6": true, "OnExitRemove": true, "JobCurrentStartDate": 1446106061, "CoreSize": 0, "MATCH_EXP_JOBGLIDEIN_ResourceName": "wisc.edu", "Rank": 0.0d, "ExitStatus": 0, "ResidentSetSize": 150000, "RemoteWallClockTime": 27676.0d, "WantCheckpoint": false, "In": "/dev/null", "MaxHosts": 1, "RootDir": "/", "NumRestarts": 0, "RecentBlockWriteKbytes": 0, "DiskUsage_RAW": 1216669, "EnteredCurrentStatus": 1446133737, "ResidentSetSize_RAW": 127252, "RequestDisk": 1000000, "MyType": "Job", "PeriodicRemove": false, "Cmd": "/home/xguo23/finally_2/Simulation_condor/chtcjobwrapper", "CondorVersion": "$CondorVersion: 8.5.0 Sep 16 2015 BuildID: 341710 $", "ShouldTransferFiles": "YES", "TargetType": "Machine", "MinHosts": 1, "NumCkpts
_RAW": 0, "RequestCpus": 1, "RemoteUserCpu": 27510.0d, "BlockWrites": 0, "NiceUser": false, "Out": "process.out", "ImageSize_RAW": 811948, "BytesSent": 30584.0d, "CumulativeSuspensionTime": 0, "TransferIn": false, "NumCkpts": 0, "Err": "process.err", "RecentBlockWrites": 0, "JobFinishedHookDone": 1446133737, "ProcId": 0, "ImageSize": 1000000, "JobUniverse": 5, "EncryptExecuteDirectory": false, "TransferInputSizeMB": 0, "RecentBlockReads": 0, "SpooledOutputFiles": "CURLTIME_369560,ChtcWrapper40.out,AuditLog.40,simu_3_40.txt,harvest.log,40.out", "BlockWriteKbytes": 0, "WhenToTransferOutput": "ON_EXIT", "JobCurrentStartExecutingDate": 1446106063, "ExitBySignal": false, "LastMatchTime": 1446106061, "OnExitHold": false, "OrigMaxHosts": 1, "RequestMemory": 1000, "NumJobStarts": 1, "TerminationPending": true, "TotalSuspensions": 0, "BlockReads": 0, "DAGManJobId": 49581933, "MemoryUsage": 146, "ExitCode": 0, "JobNotification": 0, "BlockReadKbytes": 0, "NumJobMatches": 1, "LocalUserCpu": 0.0
d, "LastJobStatus": 2, "BufferBlockSize": 32768, "CommittedTime": 27676, "ExecutableSize_RAW": 6, "LastSuspensionTime": 0, "Matlab": "R2011b", "UserLog": "/home/xguo23/finally_2/Simulation_condor/model_3/40/process.log", "DAGManNodesMask": "0,1,2,4,5,7,9,10,11,12,13,16,17,24,27", "CumulativeSlotTime": 27676.0d, "LastJobLeaseRenewal": 1446133737, "MachineAttrSlotWeight0": 1, "NumSystemHolds": 0, "BytesRecvd": 285053.0d, "CondorPlatform": "$CondorPlatform: X86_64-RedHat_6.6 $", "JOBGLIDEIN_ResourceName": "$$([IfThenElse(IsUndefined(TARGET.GLIDEIN_ResourceName), IfThenElse(IsUndefined(TARGET.GLIDEIN_Site), \"wisc.edu\", TARGET.GLIDEIN_Site), TARGET.GLIDEIN_ResourceName)])", "DAGNodeName": "40+40", "PeriodicRelease": "error", "JobRunCount": 1, "LastRemoteHost": "slot1@e286.chtc.wisc.edu", "JobPrio": 0, "LocalSysCpu": 0.0d, "ExecutableSize": 7, "RemoteSysCpu": 105.0d, "TransferInput": "/home/xguo23/finally_2/Simulation_condor/data/40/,/home/xguo23/finally_2/Simulation_condor/data/shared/
", "PeriodicHold": false, "WantRemoteIO": true, "CommittedSuspensionTime": 0, "DAGParentNodeNames": "", "CompletionDate": 1446133737, "StreamErr": false, "RecentBlockReadKbytes": 0, "WantRemoteSyscalls": false, "NumShadowStarts": 1, "MachineAttrCpus0": 1, "DAGManNodesLog": "/home/xguo23/finally_2/Simulation_condor/model_3/./mydag.dag.nodes.log", "Owner": "xguo23", "Requirements": "undefined", "DiskUsage": 1250000, "JobLeaseDuration": 2400, "ClusterId": 49581952, "BufferSize": 524288, "IsCHTCSubmit": true, "RecentStatsLifetimeStarter": 1200, "CommittedSlotTime": 27676.0d, "Args": "--type=Matlab --version=R2011b --cmdtorun=simu_condor --unique=40 -- 3", "Environment": "", "LastPublicClaimId": "<128.105.245.86:32129>#1444759888#6329#...", "Iwd": "/home/xguo23/finally_2/Simulation_condor/model_3/40", "QDate": 1446105329, "CurrentHosts": 0, "User": "xguo23@chtc.wisc.edu", "StreamOut": false }
-{ "GlobalJobId": "submit-3.chtc.wisc.edu#49581985.0#1446105368", "StatsLifetimeStarter": 26354, "JobStartDate": 1446106289, "SubmitEventNotes": "DAG Node: 36+36", "JobStatus": 4, "LeaveJobInQueue": false, "StartdPrincipal": "execute-side@matchsession/128.105.244.249", "WantRHEL6": true, "OnExitRemove": true, "JobCurrentStartDate": 1446106289, "CoreSize": 0, "MATCH_EXP_JOBGLIDEIN_ResourceName": "wisc.edu", "Rank": 0.0d, "ExitStatus": 0, "ResidentSetSize": 150000, "RemoteWallClockTime": 26357.0d, "WantCheckpoint": false, "In": "/dev/null", "MaxHosts": 1, "RootDir": "/", "NumRestarts": 0, "RecentBlockWriteKbytes": 0, "DiskUsage_RAW": 1216669, "EnteredCurrentStatus": 1446132646, "ResidentSetSize_RAW": 127452, "RequestDisk": 1000000, "MyType": "Job", "PeriodicRemove": false, "Cmd": "/home/xguo23/finally_2/Simulation_condor/chtcjobwrapper", "CondorVersion": "$CondorVersion: 8.5.0 Sep 16 2015 BuildID: 341710 $", "ShouldTransferFiles": "YES", "TargetType": "Machine", "MinHosts": 1, "NumCkpt
s_RAW": 0, "RequestCpus": 1, "RemoteUserCpu": 26239.0d, "BlockWrites": 0, "NiceUser": false, "Out": "process.out", "ImageSize_RAW": 811948, "BytesSent": 31898.0d, "CumulativeSuspensionTime": 0, "TransferIn": false, "NumCkpts": 0, "Err": "process.err", "RecentBlockWrites": 0, "JobFinishedHookDone": 1446132646, "ProcId": 0, "ImageSize": 1000000, "JobUniverse": 5, "EncryptExecuteDirectory": false, "TransferInputSizeMB": 0, "RecentBlockReads": 0, "SpooledOutputFiles": "CURLTIME_1151700,ChtcWrapper36.out,AuditLog.36,simu_3_36.txt,harvest.log,36.out", "BlockWriteKbytes": 0, "WhenToTransferOutput": "ON_EXIT", "JobCurrentStartExecutingDate": 1446106291, "ExitBySignal": false, "LastMatchTime": 1446106289, "OnExitHold": false, "OrigMaxHosts": 1, "RequestMemory": 1000, "NumJobStarts": 1, "TerminationPending": true, "TotalSuspensions": 0, "BlockReads": 0, "DAGManJobId": 49581933, "MemoryUsage": 146, "ExitCode": 0, "JobNotification": 0, "BlockReadKbytes": 0, "NumJobMatches": 1, "LocalUserCpu": 0
.0d, "LastJobStatus": 2, "BufferBlockSize": 32768, "CommittedTime": 26357, "ExecutableSize_RAW": 6, "LastSuspensionTime": 0, "Matlab": "R2011b", "UserLog": "/home/xguo23/finally_2/Simulation_condor/model_3/36/process.log", "DAGManNodesMask": "0,1,2,4,5,7,9,10,11,12,13,16,17,24,27", "CumulativeSlotTime": 26357.0d, "LastJobLeaseRenewal": 1446132646, "MachineAttrSlotWeight0": 1, "NumSystemHolds": 0, "BytesRecvd": 285053.0d, "CondorPlatform": "$CondorPlatform: X86_64-RedHat_6.6 $", "JOBGLIDEIN_ResourceName": "$$([IfThenElse(IsUndefined(TARGET.GLIDEIN_ResourceName), IfThenElse(IsUndefined(TARGET.GLIDEIN_Site), \"wisc.edu\", TARGET.GLIDEIN_Site), TARGET.GLIDEIN_ResourceName)])", "DAGNodeName": "36+36", "PeriodicRelease": "error", "JobRunCount": 1, "LastRemoteHost": "slot1@e457.chtc.wisc.edu", "JobPrio": 0, "LocalSysCpu": 0.0d, "ExecutableSize": 7, "RemoteSysCpu": 96.0d, "TransferInput": "/home/xguo23/finally_2/Simulation_condor/data/36/,/home/xguo23/finally_2/Simulation_condor/data/shared
/", "PeriodicHold": false, "WantRemoteIO": true, "CommittedSuspensionTime": 0, "DAGParentNodeNames": "", "CompletionDate": 1446132646, "StreamErr": false, "RecentBlockReadKbytes": 0, "WantRemoteSyscalls": false, "NumShadowStarts": 1, "MachineAttrCpus0": 1, "DAGManNodesLog": "/home/xguo23/finally_2/Simulation_condor/model_3/./mydag.dag.nodes.log", "Owner": "xguo23", "Requirements": "undefined", "DiskUsage": 1250000, "JobLeaseDuration": 2400, "ClusterId": 49581985, "BufferSize": 524288, "IsCHTCSubmit": true, "RecentStatsLifetimeStarter": 1200, "CommittedSlotTime": 26357.0d, "Args": "--type=Matlab --version=R2011b --cmdtorun=simu_condor --unique=36 -- 3", "Environment": "", "LastPublicClaimId": "<128.105.244.249:28476>#1444685646#10655#...", "Iwd": "/home/xguo23/finally_2/Simulation_condor/model_3/36", "QDate": 1446105368, "CurrentHosts": 0, "User": "xguo23@chtc.wisc.edu", "StreamOut": false }
-{ "GlobalJobId": "submit-3.chtc.wisc.edu#49581989.0#1446105374", "StatsLifetimeStarter": 27490, "JobStartDate": 1446106290, "SubmitEventNotes": "DAG Node: 82+82", "JobStatus": 4, "LeaveJobInQueue": false, "AutoClusterId": 24, "StartdPrincipal": "execute-side@matchsession/128.105.245.233", "WantRHEL6": true, "OnExitRemove": true, "JobCurrentStartDate": 1446106290, "CoreSize": 0, "MATCH_EXP_JOBGLIDEIN_ResourceName": "wisc.edu", "ExitStatus": 0, "Rank": 0.0d, "ResidentSetSize": 150000, "RemoteWallClockTime": 27491.0d, "WantCheckpoint": false, "In": "/dev/null", "MaxHosts": 1, "RootDir": "/", "NumRestarts": 0, "RecentBlockWriteKbytes": 0, "DiskUsage_RAW": 1216669, "EnteredCurrentStatus": 1446133781, "ResidentSetSize_RAW": 126932, "RequestDisk": 1000000, "MyType": "Job", "PeriodicRemove": false, "Cmd": "/home/xguo23/finally_2/Simulation_condor/chtcjobwrapper", "CondorVersion": "$CondorVersion: 8.5.0 Sep 16 2015 BuildID: 341710 $", "ShouldTransferFiles": "YES", "TargetType": "Machine", "M
inHosts": 1, "NumCkpts_RAW": 0, "RequestCpus": 1, "RemoteUserCpu": 27288.0d, "BlockWrites": 0, "NiceUser": false, "Out": "process.out", "ImageSize_RAW": 811948, "BytesSent": 30553.0d, "CumulativeSuspensionTime": 0, "TransferIn": false, "AutoClusterAttrs": "JobUniverse,LastCheckpointPlatform,NumCkpts,ClientMachine,_condor_RequestCpus,_condor_RequestDisk,_condor_RequestGPUs,_condor_RequestMemory,RequestCpus,RequestDisk,RequestGPUs,RequestMemory,BIOCHEM,MachineLastMatchTime,ConcurrencyLimits,NiceUser,Rank,Requirements,ImageSize,MemoryRequirements,User,RemoteGroup,SubmitterGroup,SubmitterUserPrio,Group,WIDsTheme,InteractiveJob,Is_Resumable,WantFlocking,WantGlidein,Scheduler,Owner,JobStart,MemoryUsage,IsExpressQueueJob,DiskUsage,HEP_VO,IsDesktop,OSG_VO,x509userproxysubject,PassedTest,IsLocalCMSJob,IsLocalCMSSlot,IsSAMSlot,IsSAMJob,MaxDiskTempC,IsDedicated,estimated_run_hours,IsCHTCSubmit,RequiresCVMFS,DynamicSlot,PartitionableSlot,Slot1_ExpectedMachineGracefulDrainingCompletion,Slot1_Job
Starts,Slot1_SelfMonitorAge,Slot2_ExpectedMachineGracefulDrainingCompletion,Slot2_JobStarts,Slot2_SelfMonitorAge,Slot3_ExpectedMachineGracefulDrainingCompletion,Slot3_JobStarts,Slot3_SelfMonitorAge,Slot4_ExpectedMachineGracefulDrainingCompletion,Slot4_JobStarts,Slot4_SelfMonitorAge,Slot5_ExpectedMachineGracefulDrainingCompletion,Slot5_JobStarts,Slot5_SelfMonitorAge,Slot6_ExpectedMachineGracefulDrainingCompletion,Slot6_JobStarts,Slot6_SelfMonitorAge,Slot7_ExpectedMachineGracefulDrainingCompletion,Slot7_JobStarts,Slot7_SelfMonitorAge,Slot8_ExpectedMachineGracefulDrainingCompletion,Slot8_JobStarts,Slot8_SelfMonitorAge,Slot1_TotalTimeClaimedBusy,Slot1_TotalTimeUnclaimedIdle,Slot2_TotalTimeClaimedBusy,Slot2_TotalTimeUnclaimedIdle,Slot3_TotalTimeClaimedBusy,Slot3_TotalTimeUnclaimedIdle,Slot4_TotalTimeClaimedBusy,Slot4_TotalTimeUnclaimedIdle,Slot5_TotalTimeClaimedBusy,Slot5_TotalTimeUnclaimedIdle,Slot6_TotalTimeClaimedBusy,Slot6_TotalTimeUnclaimedIdle,Slot7_TotalTimeClaimedBusy,Slot7_Total
TimeUnclaimedIdle,Slot8_TotalTimeClaimedBusy,Slot8_TotalTimeUnclaimedIdle,Slot10_ExpectedMachineGracefulDrainingCompletion,Slot10_JobStarts,Slot10_SelfMonitorAge,Slot11_ExpectedMachineGracefulDrainingCompletion,Slot11_JobStarts,Slot11_SelfMonitorAge,Slot12_ExpectedMachineGracefulDrainingCompletion,Slot12_JobStarts,Slot12_SelfMonitorAge,Slot9_ExpectedMachineGracefulDrainingCompletion,Slot9_JobStarts,Slot9_SelfMonitorAge,Slot12_TotalTimeClaimedBusy,Slot10_TotalTimeClaimedBusy,Slot10_TotalTimeUnclaimedIdle,Slot11_TotalTimeClaimedBusy,Slot11_TotalTimeUnclaimedIdle,Slot12_TotalTimeUnclaimedIdle,Slot9_TotalTimeClaimedBusy,Slot9_TotalTimeUnclaimedIdle,Slot13_ExpectedMachineGracefulDrainingCompletion,Slot13_JobStarts,Slot13_SelfMonitorAge,Slot14_ExpectedMachineGracefulDrainingCompletion,Slot14_JobStarts,Slot14_SelfMonitorAge,Slot15_ExpectedMachineGracefulDrainingCompletion,Slot15_JobStarts,Slot15_SelfMonitorAge,Slot16_ExpectedMachineGracefulDrainingCompletion,Slot16_JobStarts,Slot16_SelfMon
itorAge,IsResumable,WHEN_TO_TRANSFER_OUTPUT,_condor_Requestadmin_mutex_1,_condor_Requestadmin_mutex_2,_condor_Requestadmin_mutex_3,_condor_Requestmachine_token,Requestadmin_mutex_1,Requestadmin_mutex_2,Requestadmin_mutex_3,Requestmachine_token,nyehle,IsBuildJob,IsMatlabBuildJob,TotalJobRunTime,NodeOnline,Slot13_TotalTimeClaimedBusy,Slot13_TotalTimeUnclaimedIdle,Slot14_TotalTimeClaimedBusy,Slot14_TotalTimeUnclaimedIdle,Slot15_TotalTimeClaimedBusy,Slot15_TotalTimeUnclaimedIdle,Slot16_TotalTimeClaimedBusy,Slot16_TotalTimeUnclaimedIdle,TmpIsFull,trResumable,RequiresCMSFrontier,Slot17_ExpectedMachineGracefulDrainingCompletion,Slot17_JobStarts,Slot17_SelfMonitorAge,Slot17_TotalTimeClaimedBusy,Slot17_TotalTimeUnclaimedIdle,Slot18_ExpectedMachineGracefulDrainingCompletion,Slot18_JobStarts,Slot18_SelfMonitorAge,Slot18_TotalTimeClaimedBusy,Slot18_TotalTimeUnclaimedIdle,Slot19_ExpectedMachineGracefulDrainingCompletion,Slot19_JobStarts,Slot19_SelfMonitorAge,Slot19_TotalTimeClaimedBusy,Slot19_To
talTimeUnclaimedIdle,Slot20_ExpectedMachineGracefulDrainingCompletion,Slot20_JobStarts,Slot20_SelfMonitorAge,Slot20_TotalTimeClaimedBusy,Slot20_TotalTimeUnclaimedIdle,Slot21_ExpectedMachineGracefulDrainingCompletion,Slot21_JobStarts,Slot21_SelfMonitorAge,Slot21_TotalTimeClaimedBusy,Slot21_TotalTimeUnclaimedIdle,Slot22_ExpectedMachineGracefulDrainingCompletion,Slot22_JobStarts,Slot22_SelfMonitorAge,Slot22_TotalTimeClaimedBusy,Slot22_TotalTimeUnclaimedIdle,Slot23_ExpectedMachineGracefulDrainingCompletion,Slot23_JobStarts,Slot23_SelfMonitorAge,Slot23_TotalTimeClaimedBusy,Slot23_TotalTimeUnclaimedIdle,Slot24_ExpectedMachineGracefulDrainingCompletion,Slot24_JobStarts,Slot24_SelfMonitorAge,Slot24_TotalTimeClaimedBusy,Slot24_TotalTimeUnclaimedIdle,Slot25_ExpectedMachineGracefulDrainingCompletion,Slot25_JobStarts,Slot25_SelfMonitorAge,Slot25_TotalTimeClaimedBusy,Slot25_TotalTimeUnclaimedIdle,Slot26_ExpectedMachineGracefulDrainingCompletion,Slot26_JobStarts,Slot26_SelfMonitorAge,Slot26_Total
TimeClaimedBusy,Slot26_TotalTimeUnclaimedIdle,Slot27_ExpectedMachineGracefulDrainingCompletion,Slot27_JobStarts,Slot27_SelfMonitorAge,Slot27_TotalTimeClaimedBusy,Slot27_TotalTimeUnclaimedIdle,Slot28_ExpectedMachineGracefulDrainingCompletion,Slot28_JobStarts,Slot28_SelfMonitorAge,Slot28_TotalTimeClaimedBusy,Slot28_TotalTimeUnclaimedIdle,Slot29_ExpectedMachineGracefulDrainingCompletion,Slot29_JobStarts,Slot29_SelfMonitorAge,Slot29_TotalTimeClaimedBusy,Slot29_TotalTimeUnclaimedIdle,Slot30_ExpectedMachineGracefulDrainingCompletion,Slot30_JobStarts,Slot30_SelfMonitorAge,Slot30_TotalTimeClaimedBusy,Slot30_TotalTimeUnclaimedIdle,Slot31_ExpectedMachineGracefulDrainingCompletion,Slot31_JobStarts,Slot31_SelfMonitorAge,Slot31_TotalTimeClaimedBusy,Slot31_TotalTimeUnclaimedIdle,Slot32_ExpectedMachineGracefulDrainingCompletion,Slot32_JobStarts,Slot32_SelfMonitorAge,Slot32_TotalTimeClaimedBusy,Slot32_TotalTimeUnclaimedIdle,ResidentSetSize", "NumCkpts": 0, "Err": "process.err", "RecentBlockWrites":
0, "JobFinishedHookDone": 1446133782, "ProcId": 0, "ImageSize": 1000000, "JobUniverse": 5, "EncryptExecuteDirectory": false, "TransferInputSizeMB": 0, "RecentBlockReads": 0, "SpooledOutputFiles": "CURLTIME_4096502,ChtcWrapper82.out,AuditLog.82,simu_3_82.txt,harvest.log,82.out", "BlockWriteKbytes": 0, "WhenToTransferOutput": "ON_EXIT", "JobCurrentStartExecutingDate": 1446106291, "ExitBySignal": false, "LastMatchTime": 1446106290, "OnExitHold": false, "OrigMaxHosts": 1, "RequestMemory": 1000, "NumJobStarts": 1, "TerminationPending": true, "TotalSuspensions": 0, "BlockReads": 0, "DAGManJobId": 49581933, "MemoryUsage": 146, "ExitCode": 0, "JobNotification": 0, "BlockReadKbytes": 0, "NumJobMatches": 1, "LocalUserCpu": 0.0d, "LastJobStatus": 2, "BufferBlockSize": 32768, "CommittedTime": 27491, "ExecutableSize_RAW": 6, "LastSuspensionTime": 0, "Matlab": "R2011b", "UserLog": "/home/xguo23/finally_2/Simulation_condor/model_3/82/process.log", "DAGManNodesMask": "0,1,2,4,5,7,9,10,11,12,13,16,
17,24,27", "CumulativeSlotTime": 27491.0d, "LastJobLeaseRenewal": 1446133781, "MachineAttrSlotWeight0": 1, "NumSystemHolds": 0, "BytesRecvd": 285053.0d, "CondorPlatform": "$CondorPlatform: X86_64-RedHat_6.6 $", "JOBGLIDEIN_ResourceName": "$$([IfThenElse(IsUndefined(TARGET.GLIDEIN_ResourceName), IfThenElse(IsUndefined(TARGET.GLIDEIN_Site), \"wisc.edu\", TARGET.GLIDEIN_Site), TARGET.GLIDEIN_ResourceName)])", "DAGNodeName": "82+82", "PeriodicRelease": "error", "JobRunCount": 1, "LastRemoteHost": "slot1@e433.chtc.wisc.edu", "JobPrio": 0, "LocalSysCpu": 0.0d, "ExecutableSize": 7, "RemoteSysCpu": 173.0d, "TransferInput": "/home/xguo23/finally_2/Simulation_condor/data/82/,/home/xguo23/finally_2/Simulation_condor/data/shared/", "PeriodicHold": false, "WantRemoteIO": true, "CommittedSuspensionTime": 0, "DAGParentNodeNames": "", "CompletionDate": 1446133781, "StreamErr": false, "RecentBlockReadKbytes": 0, "WantRemoteSyscalls": false, "NumShadowStarts": 1, "MachineAttrCpus0": 1, "DAGManNodesLo
g": "/home/xguo23/finally_2/Simulation_condor/model_3/./mydag.dag.nodes.log", "Owner": "xguo23", "Requirements": "undefined", "DiskUsage": 1250000, "JobLeaseDuration": 2400, "ClusterId": 49581989, "BufferSize": 524288, "IsCHTCSubmit": true, "RecentStatsLifetimeStarter": 1200, "CommittedSlotTime": 27491.0d, "Args": "--type=Matlab --version=R2011b --cmdtorun=simu_condor --unique=82 -- 3", "Environment": "", "LastPublicClaimId": "<128.105.245.233:28601>#1443991451#13496#...", "Iwd": "/home/xguo23/finally_2/Simulation_condor/model_3/82", "QDate": 1446105374, "CurrentHosts": 0, "User": "xguo23@chtc.wisc.edu", "StreamOut": false }
-{ "GlobalJobId": "submit-3.chtc.wisc.edu#49582049.0#1446105441", "StatsLifetimeStarter": 26296, "JobStartDate": 1446106482, "SubmitEventNotes": "DAG Node: 112+112", "JobStatus": 4, "LeaveJobInQueue": false, "StartdPrincipal": "execute-side@matchsession/128.105.245.245", "WantRHEL6": true, "OnExitRemove": true, "JobCurrentStartDate": 1446106482, "CoreSize": 0, "MATCH_EXP_JOBGLIDEIN_ResourceName": "wisc.edu", "Rank": 0.0d, "ExitStatus": 0, "ResidentSetSize": 150000, "RemoteWallClockTime": 26298.0d, "WantCheckpoint": false, "In": "/dev/null", "MaxHosts": 1, "RootDir": "/", "NumRestarts": 0, "RecentBlockWriteKbytes": 0, "DiskUsage_RAW": 1216669, "EnteredCurrentStatus": 1446132780, "ResidentSetSize_RAW": 126892, "RequestDisk": 1000000, "MyType": "Job", "PeriodicRemove": false, "Cmd": "/home/xguo23/finally_2/Simulation_condor/chtcjobwrapper", "CondorVersion": "$CondorVersion: 8.5.0 Sep 16 2015 BuildID: 341710 $", "ShouldTransferFiles": "YES", "TargetType": "Machine", "MinHosts": 1, "NumCk
pts_RAW": 0, "RequestCpus": 1, "RemoteUserCpu": 26097.0d, "BlockWrites": 0, "NiceUser": false, "Out": "process.out", "ImageSize_RAW": 811948, "BytesSent": 31904.0d, "CumulativeSuspensionTime": 0, "TransferIn": false, "NumCkpts": 0, "Err": "process.err", "RecentBlockWrites": 0, "JobFinishedHookDone": 1446132780, "ProcId": 0, "ImageSize": 1000000, "JobUniverse": 5, "EncryptExecuteDirectory": false, "TransferInputSizeMB": 0, "RecentBlockReads": 0, "SpooledOutputFiles": "CURLTIME_2601607,ChtcWrapper112.out,AuditLog.112,simu_3_112.txt,harvest.log,112.out", "BlockWriteKbytes": 0, "WhenToTransferOutput": "ON_EXIT", "JobCurrentStartExecutingDate": 1446106484, "ExitBySignal": false, "LastMatchTime": 1446106482, "OnExitHold": false, "OrigMaxHosts": 1, "RequestMemory": 1000, "NumJobStarts": 1, "TerminationPending": true, "TotalSuspensions": 0, "BlockReads": 0, "DAGManJobId": 49581933, "MemoryUsage": 146, "ExitCode": 0, "JobNotification": 0, "BlockReadKbytes": 0, "NumJobMatches": 1, "LocalUserC
pu": 0.0d, "LastJobStatus": 2, "BufferBlockSize": 32768, "CommittedTime": 26298, "ExecutableSize_RAW": 6, "LastSuspensionTime": 0, "Matlab": "R2011b", "UserLog": "/home/xguo23/finally_2/Simulation_condor/model_3/112/process.log", "DAGManNodesMask": "0,1,2,4,5,7,9,10,11,12,13,16,17,24,27", "CumulativeSlotTime": 26298.0d, "LastJobLeaseRenewal": 1446132780, "MachineAttrSlotWeight0": 1, "NumSystemHolds": 0, "BytesRecvd": 285054.0d, "CondorPlatform": "$CondorPlatform: X86_64-RedHat_6.6 $", "JOBGLIDEIN_ResourceName": "$$([IfThenElse(IsUndefined(TARGET.GLIDEIN_ResourceName), IfThenElse(IsUndefined(TARGET.GLIDEIN_Site), \"wisc.edu\", TARGET.GLIDEIN_Site), TARGET.GLIDEIN_ResourceName)])", "DAGNodeName": "112+112", "PeriodicRelease": "error", "JobRunCount": 1, "LastRemoteHost": "slot1@e445.chtc.WISC.EDU", "JobPrio": 0, "LocalSysCpu": 0.0d, "ExecutableSize": 7, "RemoteSysCpu": 164.0d, "TransferInput": "/home/xguo23/finally_2/Simulation_condor/data/112/,/home/xguo23/finally_2/Simulation_condor/
data/shared/", "PeriodicHold": false, "WantRemoteIO": true, "CommittedSuspensionTime": 0, "DAGParentNodeNames": "", "CompletionDate": 1446132780, "StreamErr": false, "RecentBlockReadKbytes": 0, "WantRemoteSyscalls": false, "NumShadowStarts": 1, "MachineAttrCpus0": 1, "DAGManNodesLog": "/home/xguo23/finally_2/Simulation_condor/model_3/./mydag.dag.nodes.log", "Owner": "xguo23", "Requirements": "undefined", "DiskUsage": 1250000, "JobLeaseDuration": 2400, "ClusterId": 49582049, "BufferSize": 524288, "IsCHTCSubmit": true, "RecentStatsLifetimeStarter": 1200, "CommittedSlotTime": 26298.0d, "Args": "--type=Matlab --version=R2011b --cmdtorun=simu_condor --unique=112 -- 3", "Environment": "", "LastPublicClaimId": "<128.105.245.245:48407>#1443991450#14631#...", "Iwd": "/home/xguo23/finally_2/Simulation_condor/model_3/112", "QDate": 1446105441, "CurrentHosts": 0, "User": "xguo23@chtc.wisc.edu", "StreamOut": false }
-{ "GlobalJobId": "submit-3.chtc.wisc.edu#49582050.0#1446105441", "StatsLifetimeStarter": 27141, "JobStartDate": 1446106482, "SubmitEventNotes": "DAG Node: 301+301", "JobStatus": 4, "LeaveJobInQueue": false, "StartdPrincipal": "execute-side@matchsession/128.105.245.172", "WantRHEL6": true, "OnExitRemove": true, "JobCurrentStartDate": 1446106482, "CoreSize": 0, "MATCH_EXP_JOBGLIDEIN_ResourceName": "wisc.edu", "Rank": 0.0d, "ExitStatus": 0, "ResidentSetSize": 150000, "RemoteWallClockTime": 27143.0d, "WantCheckpoint": false, "In": "/dev/null", "MaxHosts": 1, "RootDir": "/", "NumRestarts": 0, "RecentBlockWriteKbytes": 0, "DiskUsage_RAW": 1216669, "EnteredCurrentStatus": 1446133625, "ResidentSetSize_RAW": 126464, "RequestDisk": 1000000, "MyType": "Job", "PeriodicRemove": false, "Cmd": "/home/xguo23/finally_2/Simulation_condor/chtcjobwrapper", "CondorVersion": "$CondorVersion: 8.5.0 Sep 16 2015 BuildID: 341710 $", "ShouldTransferFiles": "YES", "TargetType": "Machine", "MinHosts": 1, "NumCk
pts_RAW": 0, "RequestCpus": 1, "RemoteUserCpu": 26895.0d, "BlockWrites": 0, "NiceUser": false, "Out": "process.out", "ImageSize_RAW": 811948, "BytesSent": 31905.0d, "CumulativeSuspensionTime": 0, "TransferIn": false, "NumCkpts": 0, "Err": "process.err", "RecentBlockWrites": 0, "JobFinishedHookDone": 1446133625, "ProcId": 0, "ImageSize": 1000000, "JobUniverse": 5, "EncryptExecuteDirectory": false, "TransferInputSizeMB": 0, "RecentBlockReads": 0, "SpooledOutputFiles": "CURLTIME_2158419,ChtcWrapper301.out,AuditLog.301,simu_3_301.txt,harvest.log,301.out", "BlockWriteKbytes": 0, "WhenToTransferOutput": "ON_EXIT", "JobCurrentStartExecutingDate": 1446106484, "ExitBySignal": false, "LastMatchTime": 1446106482, "OnExitHold": false, "OrigMaxHosts": 1, "RequestMemory": 1000, "NumJobStarts": 1, "TerminationPending": true, "TotalSuspensions": 0, "BlockReads": 0, "DAGManJobId": 49581933, "MemoryUsage": 146, "ExitCode": 0, "JobNotification": 0, "BlockReadKbytes": 0, "NumJobMatches": 1, "LocalUserC
pu": 0.0d, "LastJobStatus": 2, "BufferBlockSize": 32768, "CommittedTime": 27143, "ExecutableSize_RAW": 6, "LastSuspensionTime": 0, "Matlab": "R2011b", "UserLog": "/home/xguo23/finally_2/Simulation_condor/model_3/301/process.log", "DAGManNodesMask": "0,1,2,4,5,7,9,10,11,12,13,16,17,24,27", "CumulativeSlotTime": 27143.0d, "LastJobLeaseRenewal": 1446133625, "MachineAttrSlotWeight0": 1, "NumSystemHolds": 0, "BytesRecvd": 285054.0d, "CondorPlatform": "$CondorPlatform: X86_64-RedHat_6.6 $", "JOBGLIDEIN_ResourceName": "$$([IfThenElse(IsUndefined(TARGET.GLIDEIN_ResourceName), IfThenElse(IsUndefined(TARGET.GLIDEIN_Site), \"wisc.edu\", TARGET.GLIDEIN_Site), TARGET.GLIDEIN_ResourceName)])", "DAGNodeName": "301+301", "PeriodicRelease": "error", "JobRunCount": 1, "LastRemoteHost": "slot1@e372.chtc.wisc.edu", "JobPrio": 0, "LocalSysCpu": 0.0d, "ExecutableSize": 7, "RemoteSysCpu": 201.0d, "TransferInput": "/home/xguo23/finally_2/Simulation_condor/data/301/,/home/xguo23/finally_2/Simulation_condor/
data/shared/", "PeriodicHold": false, "WantRemoteIO": true, "CommittedSuspensionTime": 0, "DAGParentNodeNames": "", "CompletionDate": 1446133625, "StreamErr": false, "RecentBlockReadKbytes": 0, "WantRemoteSyscalls": false, "NumShadowStarts": 1, "MachineAttrCpus0": 1, "DAGManNodesLog": "/home/xguo23/finally_2/Simulation_condor/model_3/./mydag.dag.nodes.log", "Owner": "xguo23", "Requirements": "undefined", "DiskUsage": 1250000, "JobLeaseDuration": 2400, "ClusterId": 49582050, "BufferSize": 524288, "IsCHTCSubmit": true, "RecentStatsLifetimeStarter": 1200, "CommittedSlotTime": 27143.0d, "Args": "--type=Matlab --version=R2011b --cmdtorun=simu_condor --unique=301 -- 3", "Environment": "", "LastPublicClaimId": "<128.105.245.172:19856>#1444760019#9307#...", "Iwd": "/home/xguo23/finally_2/Simulation_condor/model_3/301", "QDate": 1446105441, "CurrentHosts": 0, "User": "xguo23@chtc.wisc.edu", "StreamOut": false }
-{ "GlobalJobId": "submit-3.chtc.wisc.edu#49582065.0#1446105458", "StatsLifetimeStarter": 25606, "JobStartDate": 1446107042, "SubmitEventNotes": "DAG Node: 401+401", "JobStatus": 4, "LeaveJobInQueue": false, "AutoClusterId": 24, "StartdPrincipal": "execute-side@matchsession/128.105.245.206", "WantRHEL6": true, "OnExitRemove": true, "JobCurrentStartDate": 1446107042, "CoreSize": 0, "MATCH_EXP_JOBGLIDEIN_ResourceName": "wisc.edu", "ExitStatus": 0, "Rank": 0.0d, "ResidentSetSize": 150000, "RemoteWallClockTime": 25607.0d, "WantCheckpoint": false, "In": "/dev/null", "MaxHosts": 1, "RootDir": "/", "NumRestarts": 0, "RecentBlockWriteKbytes": 0, "DiskUsage_RAW": 1216669, "EnteredCurrentStatus": 1446132649, "ResidentSetSize_RAW": 126608, "RequestDisk": 1000000, "MyType": "Job", "PeriodicRemove": false, "Cmd": "/home/xguo23/finally_2/Simulation_condor/chtcjobwrapper", "CondorVersion": "$CondorVersion: 8.5.0 Sep 16 2015 BuildID: 341710 $", "ShouldTransferFiles": "YES", "TargetType": "Machine",
"MinHosts": 1, "NumCkpts_RAW": 0, "RequestCpus": 1, "RemoteUserCpu": 25478.0d, "BlockWrites": 0, "NiceUser": false, "Out": "process.out", "ImageSize_RAW": 811948, "BytesSent": 30661.0d, "CumulativeSuspensionTime": 0, "TransferIn": false, "AutoClusterAttrs": "JobUniverse,LastCheckpointPlatform,NumCkpts,ClientMachine,_condor_RequestCpus,_condor_RequestDisk,_condor_RequestGPUs,_condor_RequestMemory,RequestCpus,RequestDisk,RequestGPUs,RequestMemory,BIOCHEM,MachineLastMatchTime,ConcurrencyLimits,NiceUser,Rank,Requirements,ImageSize,MemoryRequirements,User,RemoteGroup,SubmitterGroup,SubmitterUserPrio,Group,WIDsTheme,InteractiveJob,Is_Resumable,WantFlocking,WantGlidein,Scheduler,Owner,JobStart,MemoryUsage,IsExpressQueueJob,DiskUsage,HEP_VO,IsDesktop,OSG_VO,x509userproxysubject,PassedTest,IsLocalCMSJob,IsLocalCMSSlot,IsSAMSlot,IsSAMJob,MaxDiskTempC,IsDedicated,estimated_run_hours,IsCHTCSubmit,RequiresCVMFS,DynamicSlot,PartitionableSlot,Slot1_ExpectedMachineGracefulDrainingCompletion,Slot1_J
obStarts,Slot1_SelfMonitorAge,Slot2_ExpectedMachineGracefulDrainingCompletion,Slot2_JobStarts,Slot2_SelfMonitorAge,Slot3_ExpectedMachineGracefulDrainingCompletion,Slot3_JobStarts,Slot3_SelfMonitorAge,Slot4_ExpectedMachineGracefulDrainingCompletion,Slot4_JobStarts,Slot4_SelfMonitorAge,Slot5_ExpectedMachineGracefulDrainingCompletion,Slot5_JobStarts,Slot5_SelfMonitorAge,Slot6_ExpectedMachineGracefulDrainingCompletion,Slot6_JobStarts,Slot6_SelfMonitorAge,Slot7_ExpectedMachineGracefulDrainingCompletion,Slot7_JobStarts,Slot7_SelfMonitorAge,Slot8_ExpectedMachineGracefulDrainingCompletion,Slot8_JobStarts,Slot8_SelfMonitorAge,Slot1_TotalTimeClaimedBusy,Slot1_TotalTimeUnclaimedIdle,Slot2_TotalTimeClaimedBusy,Slot2_TotalTimeUnclaimedIdle,Slot3_TotalTimeClaimedBusy,Slot3_TotalTimeUnclaimedIdle,Slot4_TotalTimeClaimedBusy,Slot4_TotalTimeUnclaimedIdle,Slot5_TotalTimeClaimedBusy,Slot5_TotalTimeUnclaimedIdle,Slot6_TotalTimeClaimedBusy,Slot6_TotalTimeUnclaimedIdle,Slot7_TotalTimeClaimedBusy,Slot7_Tot
alTimeUnclaimedIdle,Slot8_TotalTimeClaimedBusy,Slot8_TotalTimeUnclaimedIdle,Slot10_ExpectedMachineGracefulDrainingCompletion,Slot10_JobStarts,Slot10_SelfMonitorAge,Slot11_ExpectedMachineGracefulDrainingCompletion,Slot11_JobStarts,Slot11_SelfMonitorAge,Slot12_ExpectedMachineGracefulDrainingCompletion,Slot12_JobStarts,Slot12_SelfMonitorAge,Slot9_ExpectedMachineGracefulDrainingCompletion,Slot9_JobStarts,Slot9_SelfMonitorAge,Slot12_TotalTimeClaimedBusy,Slot10_TotalTimeClaimedBusy,Slot10_TotalTimeUnclaimedIdle,Slot11_TotalTimeClaimedBusy,Slot11_TotalTimeUnclaimedIdle,Slot12_TotalTimeUnclaimedIdle,Slot9_TotalTimeClaimedBusy,Slot9_TotalTimeUnclaimedIdle,Slot13_ExpectedMachineGracefulDrainingCompletion,Slot13_JobStarts,Slot13_SelfMonitorAge,Slot14_ExpectedMachineGracefulDrainingCompletion,Slot14_JobStarts,Slot14_SelfMonitorAge,Slot15_ExpectedMachineGracefulDrainingCompletion,Slot15_JobStarts,Slot15_SelfMonitorAge,Slot16_ExpectedMachineGracefulDrainingCompletion,Slot16_JobStarts,Slot16_SelfM
onitorAge,IsResumable,WHEN_TO_TRANSFER_OUTPUT,_condor_Requestadmin_mutex_1,_condor_Requestadmin_mutex_2,_condor_Requestadmin_mutex_3,_condor_Requestmachine_token,Requestadmin_mutex_1,Requestadmin_mutex_2,Requestadmin_mutex_3,Requestmachine_token,nyehle,IsBuildJob,IsMatlabBuildJob,TotalJobRunTime,NodeOnline,Slot13_TotalTimeClaimedBusy,Slot13_TotalTimeUnclaimedIdle,Slot14_TotalTimeClaimedBusy,Slot14_TotalTimeUnclaimedIdle,Slot15_TotalTimeClaimedBusy,Slot15_TotalTimeUnclaimedIdle,Slot16_TotalTimeClaimedBusy,Slot16_TotalTimeUnclaimedIdle,TmpIsFull,trResumable,RequiresCMSFrontier,Slot17_ExpectedMachineGracefulDrainingCompletion,Slot17_JobStarts,Slot17_SelfMonitorAge,Slot17_TotalTimeClaimedBusy,Slot17_TotalTimeUnclaimedIdle,Slot18_ExpectedMachineGracefulDrainingCompletion,Slot18_JobStarts,Slot18_SelfMonitorAge,Slot18_TotalTimeClaimedBusy,Slot18_TotalTimeUnclaimedIdle,Slot19_ExpectedMachineGracefulDrainingCompletion,Slot19_JobStarts,Slot19_SelfMonitorAge,Slot19_TotalTimeClaimedBusy,Slot19_
TotalTimeUnclaimedIdle,Slot20_ExpectedMachineGracefulDrainingCompletion,Slot20_JobStarts,Slot20_SelfMonitorAge,Slot20_TotalTimeClaimedBusy,Slot20_TotalTimeUnclaimedIdle,Slot21_ExpectedMachineGracefulDrainingCompletion,Slot21_JobStarts,Slot21_SelfMonitorAge,Slot21_TotalTimeClaimedBusy,Slot21_TotalTimeUnclaimedIdle,Slot22_ExpectedMachineGracefulDrainingCompletion,Slot22_JobStarts,Slot22_SelfMonitorAge,Slot22_TotalTimeClaimedBusy,Slot22_TotalTimeUnclaimedIdle,Slot23_ExpectedMachineGracefulDrainingCompletion,Slot23_JobStarts,Slot23_SelfMonitorAge,Slot23_TotalTimeClaimedBusy,Slot23_TotalTimeUnclaimedIdle,Slot24_ExpectedMachineGracefulDrainingCompletion,Slot24_JobStarts,Slot24_SelfMonitorAge,Slot24_TotalTimeClaimedBusy,Slot24_TotalTimeUnclaimedIdle,Slot25_ExpectedMachineGracefulDrainingCompletion,Slot25_JobStarts,Slot25_SelfMonitorAge,Slot25_TotalTimeClaimedBusy,Slot25_TotalTimeUnclaimedIdle,Slot26_ExpectedMachineGracefulDrainingCompletion,Slot26_JobStarts,Slot26_SelfMonitorAge,Slot26_Tot
alTimeClaimedBusy,Slot26_TotalTimeUnclaimedIdle,Slot27_ExpectedMachineGracefulDrainingCompletion,Slot27_JobStarts,Slot27_SelfMonitorAge,Slot27_TotalTimeClaimedBusy,Slot27_TotalTimeUnclaimedIdle,Slot28_ExpectedMachineGracefulDrainingCompletion,Slot28_JobStarts,Slot28_SelfMonitorAge,Slot28_TotalTimeClaimedBusy,Slot28_TotalTimeUnclaimedIdle,Slot29_ExpectedMachineGracefulDrainingCompletion,Slot29_JobStarts,Slot29_SelfMonitorAge,Slot29_TotalTimeClaimedBusy,Slot29_TotalTimeUnclaimedIdle,Slot30_ExpectedMachineGracefulDrainingCompletion,Slot30_JobStarts,Slot30_SelfMonitorAge,Slot30_TotalTimeClaimedBusy,Slot30_TotalTimeUnclaimedIdle,Slot31_ExpectedMachineGracefulDrainingCompletion,Slot31_JobStarts,Slot31_SelfMonitorAge,Slot31_TotalTimeClaimedBusy,Slot31_TotalTimeUnclaimedIdle,Slot32_ExpectedMachineGracefulDrainingCompletion,Slot32_JobStarts,Slot32_SelfMonitorAge,Slot32_TotalTimeClaimedBusy,Slot32_TotalTimeUnclaimedIdle,ResidentSetSize", "NumCkpts": 0, "Err": "process.err", "RecentBlockWrites
": 0, "JobFinishedHookDone": 1446132649, "ProcId": 0, "ImageSize": 1000000, "JobUniverse": 5, "EncryptExecuteDirectory": false, "TransferInputSizeMB": 0, "RecentBlockReads": 0, "SpooledOutputFiles": "CURLTIME_1160521,ChtcWrapper401.out,AuditLog.401,simu_3_401.txt,harvest.log,401.out", "BlockWriteKbytes": 0, "WhenToTransferOutput": "ON_EXIT", "JobCurrentStartExecutingDate": 1446107043, "ExitBySignal": false, "LastMatchTime": 1446107042, "OnExitHold": false, "OrigMaxHosts": 1, "RequestMemory": 1000, "NumJobStarts": 1, "TerminationPending": true, "TotalSuspensions": 0, "BlockReads": 0, "DAGManJobId": 49581933, "MemoryUsage": 146, "ExitCode": 0, "JobNotification": 0, "BlockReadKbytes": 0, "NumJobMatches": 1, "LocalUserCpu": 0.0d, "LastJobStatus": 2, "BufferBlockSize": 32768, "CommittedTime": 25607, "ExecutableSize_RAW": 6, "LastSuspensionTime": 0, "Matlab": "R2011b", "UserLog": "/home/xguo23/finally_2/Simulation_condor/model_3/401/process.log", "DAGManNodesMask": "0,1,2,4,5,7,9,10,11,12
,13,16,17,24,27", "CumulativeSlotTime": 25607.0d, "LastJobLeaseRenewal": 1446132649, "MachineAttrSlotWeight0": 1, "NumSystemHolds": 0, "BytesRecvd": 285054.0d, "CondorPlatform": "$CondorPlatform: X86_64-RedHat_6.6 $", "JOBGLIDEIN_ResourceName": "$$([IfThenElse(IsUndefined(TARGET.GLIDEIN_ResourceName), IfThenElse(IsUndefined(TARGET.GLIDEIN_Site), \"wisc.edu\", TARGET.GLIDEIN_Site), TARGET.GLIDEIN_ResourceName)])", "DAGNodeName": "401+401", "PeriodicRelease": "error", "JobRunCount": 1, "LastRemoteHost": "slot1@e406.chtc.wisc.edu", "JobPrio": 0, "LocalSysCpu": 0.0d, "ExecutableSize": 7, "RemoteSysCpu": 89.0d, "TransferInput": "/home/xguo23/finally_2/Simulation_condor/data/401/,/home/xguo23/finally_2/Simulation_condor/data/shared/", "PeriodicHold": false, "WantRemoteIO": true, "CommittedSuspensionTime": 0, "DAGParentNodeNames": "", "CompletionDate": 1446132649, "StreamErr": false, "RecentBlockReadKbytes": 0, "WantRemoteSyscalls": false, "NumShadowStarts": 1, "MachineAttrCpus0": 1, "DAGM
anNodesLog": "/home/xguo23/finally_2/Simulation_condor/model_3/./mydag.dag.nodes.log", "Owner": "xguo23", "Requirements": "undefined", "DiskUsage": 1250000, "JobLeaseDuration": 2400, "ClusterId": 49582065, "BufferSize": 524288, "IsCHTCSubmit": true, "RecentStatsLifetimeStarter": 1200, "CommittedSlotTime": 25607.0d, "Args": "--type=Matlab --version=R2011b --cmdtorun=simu_condor --unique=401 -- 3", "Environment": "", "LastPublicClaimId": "<128.105.245.206:27946>#1443991437#15826#...", "Iwd": "/home/xguo23/finally_2/Simulation_condor/model_3/401", "QDate": 1446105458, "CurrentHosts": 0, "User": "xguo23@chtc.wisc.edu", "StreamOut": false }
-{ "GlobalJobId": "submit-3.chtc.wisc.edu#49582094.0#1446105491", "StatsLifetimeStarter": 25168, "JobStartDate": 1446107489, "SubmitEventNotes": "DAG Node: 106+106", "JobStatus": 4, "LeaveJobInQueue": false, "StartdPrincipal": "execute-side@matchsession/128.104.55.83", "WantRHEL6": true, "OnExitRemove": true, "JobCurrentStartDate": 1446107489, "CoreSize": 0, "MATCH_EXP_JOBGLIDEIN_ResourceName": "wisc.edu", "Rank": 0.0d, "ExitStatus": 0, "ResidentSetSize": 75000, "RemoteWallClockTime": 25169.0d, "WantCheckpoint": false, "In": "/dev/null", "MaxHosts": 1, "RootDir": "/", "NumRestarts": 0, "RecentBlockWriteKbytes": 4, "DiskUsage_RAW": 1216669, "EnteredCurrentStatus": 1446132658, "ResidentSetSize_RAW": 72016, "RequestDisk": 1000000, "MyType": "Job", "PeriodicRemove": false, "Cmd": "/home/xguo23/finally_2/Simulation_condor/chtcjobwrapper", "CondorVersion": "$CondorVersion: 8.5.0 Sep 16 2015 BuildID: 341710 $", "ShouldTransferFiles": "YES", "TargetType": "Machine", "MinHosts": 1, "NumCkpts_
RAW": 0, "RequestCpus": 1, "RemoteUserCpu": 24949.0d, "BlockWrites": 1, "NiceUser": false, "Out": "process.out", "ImageSize_RAW": 119520, "BytesSent": 30486.0d, "CumulativeSuspensionTime": 0, "TransferIn": false, "NumCkpts": 0, "Err": "process.err", "RecentBlockWrites": 1, "JobFinishedHookDone": 1446132658, "ProcId": 0, "ImageSize": 125000, "JobUniverse": 5, "EncryptExecuteDirectory": false, "TransferInputSizeMB": 0, "RecentBlockReads": 86, "SpooledOutputFiles": "CURLTIME_122139,ChtcWrapper106.out,AuditLog.106,simu_3_106.txt,harvest.log,106.out", "BlockWriteKbytes": 4, "WhenToTransferOutput": "ON_EXIT", "JobCurrentStartExecutingDate": 1446107490, "ExitBySignal": false, "LastMatchTime": 1446107489, "OnExitHold": false, "OrigMaxHosts": 1, "RequestMemory": 1000, "NumJobStarts": 1, "TerminationPending": true, "TotalSuspensions": 0, "BlockReads": 665, "DAGManJobId": 49581933, "MemoryUsage": 73, "ExitCode": 0, "JobNotification": 0, "BlockReadKbytes": 26620, "NumJobMatches": 1, "LocalUserC
pu": 0.0d, "LastJobStatus": 2, "BufferBlockSize": 32768, "CommittedTime": 25169, "ExecutableSize_RAW": 6, "LastSuspensionTime": 0, "Matlab": "R2011b", "UserLog": "/home/xguo23/finally_2/Simulation_condor/model_3/106/process.log", "DAGManNodesMask": "0,1,2,4,5,7,9,10,11,12,13,16,17,24,27", "CumulativeSlotTime": 25169.0d, "LastJobLeaseRenewal": 1446132658, "MachineAttrSlotWeight0": 1, "NumSystemHolds": 0, "BytesRecvd": 285054.0d, "CondorPlatform": "$CondorPlatform: X86_64-RedHat_6.6 $", "JOBGLIDEIN_ResourceName": "$$([IfThenElse(IsUndefined(TARGET.GLIDEIN_ResourceName), IfThenElse(IsUndefined(TARGET.GLIDEIN_Site), \"wisc.edu\", TARGET.GLIDEIN_Site), TARGET.GLIDEIN_ResourceName)])", "DAGNodeName": "106+106", "PeriodicRelease": "error", "JobRunCount": 1, "LastRemoteHost": "slot1@c064.chtc.wisc.edu", "JobPrio": 0, "LocalSysCpu": 0.0d, "ExecutableSize": 7, "RemoteSysCpu": 204.0d, "TransferInput": "/home/xguo23/finally_2/Simulation_condor/data/106/,/home/xguo23/finally_2/Simulation_condor/
data/shared/", "PeriodicHold": false, "WantRemoteIO": true, "CommittedSuspensionTime": 0, "DAGParentNodeNames": "", "CompletionDate": 1446132658, "StreamErr": false, "RecentBlockReadKbytes": 960, "WantRemoteSyscalls": false, "NumShadowStarts": 1, "MachineAttrCpus0": 1, "DAGManNodesLog": "/home/xguo23/finally_2/Simulation_condor/model_3/./mydag.dag.nodes.log", "Owner": "xguo23", "Requirements": "undefined", "DiskUsage": 1250000, "JobLeaseDuration": 2400, "ClusterId": 49582094, "BufferSize": 524288, "IsCHTCSubmit": true, "RecentStatsLifetimeStarter": 1200, "CommittedSlotTime": 25169.0d, "Args": "--type=Matlab --version=R2011b --cmdtorun=simu_condor --unique=106 -- 3", "Environment": "", "LastPublicClaimId": "<128.104.55.83:25899>#1445308581#1240#...", "Iwd": "/home/xguo23/finally_2/Simulation_condor/model_3/106", "QDate": 1446105491, "CurrentHosts": 0, "User": "xguo23@chtc.wisc.edu", "StreamOut": false }
-{ "GlobalJobId": "submit-3.chtc.wisc.edu#49582098.0#1446105492", "StatsLifetimeStarter": 26020, "JobStartDate": 1446107489, "SubmitEventNotes": "DAG Node: 304+304", "JobStatus": 4, "LeaveJobInQueue": false, "StartdPrincipal": "execute-side@matchsession/128.105.245.223", "WantRHEL6": true, "OnExitRemove": true, "JobCurrentStartDate": 1446107489, "CoreSize": 0, "MATCH_EXP_JOBGLIDEIN_ResourceName": "wisc.edu", "Rank": 0.0d, "ExitStatus": 0, "ResidentSetSize": 150000, "RemoteWallClockTime": 26022.0d, "WantCheckpoint": false, "In": "/dev/null", "MaxHosts": 1, "RootDir": "/", "NumRestarts": 0, "RecentBlockWriteKbytes": 0, "DiskUsage_RAW": 1216669, "EnteredCurrentStatus": 1446133511, "ResidentSetSize_RAW": 128776, "RequestDisk": 1000000, "MyType": "Job", "PeriodicRemove": false, "Cmd": "/home/xguo23/finally_2/Simulation_condor/chtcjobwrapper", "CondorVersion": "$CondorVersion: 8.5.0 Sep 16 2015 BuildID: 341710 $", "ShouldTransferFiles": "YES", "TargetType": "Machine", "MinHosts": 1, "NumCk
pts_RAW": 0, "RequestCpus": 1, "RemoteUserCpu": 25844.0d, "BlockWrites": 0, "NiceUser": false, "Out": "process.out", "ImageSize_RAW": 811948, "BytesSent": 31801.0d, "CumulativeSuspensionTime": 0, "TransferIn": false, "NumCkpts": 0, "Err": "process.err", "RecentBlockWrites": 0, "JobFinishedHookDone": 1446133511, "ProcId": 0, "ImageSize": 1000000, "JobUniverse": 5, "EncryptExecuteDirectory": false, "TransferInputSizeMB": 0, "RecentBlockReads": 0, "SpooledOutputFiles": "CURLTIME_3651606,ChtcWrapper304.out,AuditLog.304,simu_3_304.txt,harvest.log,304.out", "BlockWriteKbytes": 0, "WhenToTransferOutput": "ON_EXIT", "JobCurrentStartExecutingDate": 1446107491, "ExitBySignal": false, "LastMatchTime": 1446107489, "OnExitHold": false, "OrigMaxHosts": 1, "RequestMemory": 1000, "NumJobStarts": 1, "TerminationPending": true, "TotalSuspensions": 0, "BlockReads": 0, "DAGManJobId": 49581933, "MemoryUsage": 146, "ExitCode": 0, "JobNotification": 0, "BlockReadKbytes": 0, "NumJobMatches": 1, "LocalUserC
pu": 0.0d, "LastJobStatus": 2, "BufferBlockSize": 32768, "CommittedTime": 26022, "ExecutableSize_RAW": 6, "LastSuspensionTime": 0, "Matlab": "R2011b", "UserLog": "/home/xguo23/finally_2/Simulation_condor/model_3/304/process.log", "DAGManNodesMask": "0,1,2,4,5,7,9,10,11,12,13,16,17,24,27", "CumulativeSlotTime": 26022.0d, "LastJobLeaseRenewal": 1446133511, "MachineAttrSlotWeight0": 1, "NumSystemHolds": 0, "BytesRecvd": 285054.0d, "CondorPlatform": "$CondorPlatform: X86_64-RedHat_6.6 $", "JOBGLIDEIN_ResourceName": "$$([IfThenElse(IsUndefined(TARGET.GLIDEIN_ResourceName), IfThenElse(IsUndefined(TARGET.GLIDEIN_Site), \"wisc.edu\", TARGET.GLIDEIN_Site), TARGET.GLIDEIN_ResourceName)])", "DAGNodeName": "304+304", "PeriodicRelease": "error", "JobRunCount": 1, "LastRemoteHost": "slot1@e423.chtc.wisc.edu", "JobPrio": 0, "LocalSysCpu": 0.0d, "ExecutableSize": 7, "RemoteSysCpu": 143.0d, "TransferInput": "/home/xguo23/finally_2/Simulation_condor/data/304/,/home/xguo23/finally_2/Simulation_condor/
data/shared/", "PeriodicHold": false, "WantRemoteIO": true, "CommittedSuspensionTime": 0, "DAGParentNodeNames": "", "CompletionDate": 1446133511, "StreamErr": false, "RecentBlockReadKbytes": 0, "WantRemoteSyscalls": false, "NumShadowStarts": 1, "MachineAttrCpus0": 1, "DAGManNodesLog": "/home/xguo23/finally_2/Simulation_condor/model_3/./mydag.dag.nodes.log", "Owner": "xguo23", "Requirements": "undefined", "DiskUsage": 1250000, "JobLeaseDuration": 2400, "ClusterId": 49582098, "BufferSize": 524288, "IsCHTCSubmit": true, "RecentStatsLifetimeStarter": 1200, "CommittedSlotTime": 26022.0d, "Args": "--type=Matlab --version=R2011b --cmdtorun=simu_condor --unique=304 -- 3", "Environment": "", "LastPublicClaimId": "<128.105.245.223:13467>#1444760039#6376#...", "Iwd": "/home/xguo23/finally_2/Simulation_condor/model_3/304", "QDate": 1446105492, "CurrentHosts": 0, "User": "xguo23@chtc.wisc.edu", "StreamOut": false }
-{ "GlobalJobId": "submit-3.chtc.wisc.edu#49582113.0#1446105509", "StatsLifetimeStarter": 26044, "JobStartDate": 1446107490, "SubmitEventNotes": "DAG Node: 206+206", "JobStatus": 4, "LeaveJobInQueue": false, "StartdPrincipal": "execute-side@matchsession/128.105.245.120", "WantRHEL6": true, "OnExitRemove": true, "JobCurrentStartDate": 1446107490, "CoreSize": 0, "MATCH_EXP_JOBGLIDEIN_ResourceName": "wisc.edu", "Rank": 0.0d, "ExitStatus": 0, "ResidentSetSize": 150000, "RemoteWallClockTime": 26045.0d, "WantCheckpoint": false, "In": "/dev/null", "MaxHosts": 1, "RootDir": "/", "NumRestarts": 0, "RecentBlockWriteKbytes": 0, "DiskUsage_RAW": 1216669, "EnteredCurrentStatus": 1446133535, "ResidentSetSize_RAW": 126460, "RequestDisk": 1000000, "MyType": "Job", "PeriodicRemove": false, "Cmd": "/home/xguo23/finally_2/Simulation_condor/chtcjobwrapper", "CondorVersion": "$CondorVersion: 8.5.0 Sep 16 2015 BuildID: 341710 $", "ShouldTransferFiles": "YES", "TargetType": "Machine", "MinHosts": 1, "NumCk
pts_RAW": 0, "RequestCpus": 1, "RemoteUserCpu": 25939.0d, "BlockWrites": 0, "NiceUser": false, "Out": "process.out", "ImageSize_RAW": 811948, "BytesSent": 30596.0d, "CumulativeSuspensionTime": 0, "TransferIn": false, "NumCkpts": 0, "Err": "process.err", "RecentBlockWrites": 0, "JobFinishedHookDone": 1446133535, "ProcId": 0, "ImageSize": 1000000, "JobUniverse": 5, "EncryptExecuteDirectory": false, "TransferInputSizeMB": 0, "RecentBlockReads": 0, "SpooledOutputFiles": "CURLTIME_522843,ChtcWrapper206.out,AuditLog.206,simu_3_206.txt,harvest.log,206.out", "BlockWriteKbytes": 0, "WhenToTransferOutput": "ON_EXIT", "JobCurrentStartExecutingDate": 1446107491, "ExitBySignal": false, "LastMatchTime": 1446107490, "OnExitHold": false, "OrigMaxHosts": 1, "RequestMemory": 1000, "NumJobStarts": 1, "TerminationPending": true, "TotalSuspensions": 0, "BlockReads": 0, "DAGManJobId": 49581933, "MemoryUsage": 146, "ExitCode": 0, "JobNotification": 0, "BlockReadKbytes": 0, "NumJobMatches": 1, "LocalUserCp
u": 0.0d, "LastJobStatus": 2, "BufferBlockSize": 32768, "CommittedTime": 26045, "ExecutableSize_RAW": 6, "LastSuspensionTime": 0, "Matlab": "R2011b", "UserLog": "/home/xguo23/finally_2/Simulation_condor/model_3/206/process.log", "DAGManNodesMask": "0,1,2,4,5,7,9,10,11,12,13,16,17,24,27", "CumulativeSlotTime": 26045.0d, "LastJobLeaseRenewal": 1446133535, "MachineAttrSlotWeight0": 1, "NumSystemHolds": 0, "BytesRecvd": 285054.0d, "CondorPlatform": "$CondorPlatform: X86_64-RedHat_6.6 $", "JOBGLIDEIN_ResourceName": "$$([IfThenElse(IsUndefined(TARGET.GLIDEIN_ResourceName), IfThenElse(IsUndefined(TARGET.GLIDEIN_Site), \"wisc.edu\", TARGET.GLIDEIN_Site), TARGET.GLIDEIN_ResourceName)])", "DAGNodeName": "206+206", "PeriodicRelease": "error", "JobRunCount": 1, "LastRemoteHost": "slot1@e320.chtc.wisc.edu", "JobPrio": 0, "LocalSysCpu": 0.0d, "ExecutableSize": 7, "RemoteSysCpu": 87.0d, "TransferInput": "/home/xguo23/finally_2/Simulation_condor/data/206/,/home/xguo23/finally_2/Simulation_condor/da
ta/shared/", "PeriodicHold": false, "WantRemoteIO": true, "CommittedSuspensionTime": 0, "DAGParentNodeNames": "", "CompletionDate": 1446133535, "StreamErr": false, "RecentBlockReadKbytes": 0, "WantRemoteSyscalls": false, "NumShadowStarts": 1, "MachineAttrCpus0": 1, "DAGManNodesLog": "/home/xguo23/finally_2/Simulation_condor/model_3/./mydag.dag.nodes.log", "Owner": "xguo23", "Requirements": "undefined", "DiskUsage": 1250000, "JobLeaseDuration": 2400, "ClusterId": 49582113, "BufferSize": 524288, "IsCHTCSubmit": true, "RecentStatsLifetimeStarter": 1200, "CommittedSlotTime": 26045.0d, "Args": "--type=Matlab --version=R2011b --cmdtorun=simu_condor --unique=206 -- 3", "Environment": "", "LastPublicClaimId": "<128.105.245.120:45185>#1443991409#14238#...", "Iwd": "/home/xguo23/finally_2/Simulation_condor/model_3/206", "QDate": 1446105509, "CurrentHosts": 0, "User": "xguo23@chtc.wisc.edu", "StreamOut": false }
-{ "GlobalJobId": "submit-3.chtc.wisc.edu#49582119.0#1446105519", "StatsLifetimeStarter": 24928, "JobStartDate": 1446107490, "SubmitEventNotes": "DAG Node: 152+152", "JobStatus": 4, "LeaveJobInQueue": false, "StartdPrincipal": "execute-side@matchsession/128.105.245.242", "WantRHEL6": true, "OnExitRemove": true, "JobCurrentStartDate": 1446107490, "CoreSize": 0, "MATCH_EXP_JOBGLIDEIN_ResourceName": "wisc.edu", "Rank": 0.0d, "ExitStatus": 0, "ResidentSetSize": 150000, "RemoteWallClockTime": 24930.0d, "WantCheckpoint": false, "In": "/dev/null", "MaxHosts": 1, "RootDir": "/", "NumRestarts": 0, "RecentBlockWriteKbytes": 0, "DiskUsage_RAW": 1216669, "EnteredCurrentStatus": 1446132420, "ResidentSetSize_RAW": 128972, "RequestDisk": 1000000, "MyType": "Job", "PeriodicRemove": false, "Cmd": "/home/xguo23/finally_2/Simulation_condor/chtcjobwrapper", "CondorVersion": "$CondorVersion: 8.5.0 Sep 16 2015 BuildID: 341710 $", "ShouldTransferFiles": "YES", "TargetType": "Machine", "MinHosts": 1, "NumCk
pts_RAW": 0, "RequestCpus": 1, "RemoteUserCpu": 24742.0d, "BlockWrites": 0, "NiceUser": false, "Out": "process.out", "ImageSize_RAW": 811948, "BytesSent": 30431.0d, "CumulativeSuspensionTime": 0, "TransferIn": false, "NumCkpts": 0, "Err": "process.err", "RecentBlockWrites": 0, "JobFinishedHookDone": 1446132420, "ProcId": 0, "ImageSize": 1000000, "JobUniverse": 5, "EncryptExecuteDirectory": false, "TransferInputSizeMB": 0, "RecentBlockReads": 0, "SpooledOutputFiles": "CURLTIME_674,ChtcWrapper152.out,AuditLog.152,simu_3_152.txt,harvest.log,152.out", "BlockWriteKbytes": 0, "WhenToTransferOutput": "ON_EXIT", "JobCurrentStartExecutingDate": 1446107491, "ExitBySignal": false, "LastMatchTime": 1446107490, "OnExitHold": false, "OrigMaxHosts": 1, "RequestMemory": 1000, "NumJobStarts": 1, "TerminationPending": true, "TotalSuspensions": 0, "BlockReads": 0, "DAGManJobId": 49581933, "MemoryUsage": 146, "ExitCode": 0, "JobNotification": 0, "BlockReadKbytes": 0, "NumJobMatches": 1, "LocalUserCpu":
0.0d, "LastJobStatus": 2, "BufferBlockSize": 32768, "CommittedTime": 24930, "ExecutableSize_RAW": 6, "LastSuspensionTime": 0, "Matlab": "R2011b", "UserLog": "/home/xguo23/finally_2/Simulation_condor/model_3/152/process.log", "DAGManNodesMask": "0,1,2,4,5,7,9,10,11,12,13,16,17,24,27", "CumulativeSlotTime": 24930.0d, "LastJobLeaseRenewal": 1446132420, "MachineAttrSlotWeight0": 1, "NumSystemHolds": 0, "BytesRecvd": 285054.0d, "CondorPlatform": "$CondorPlatform: X86_64-RedHat_6.6 $", "JOBGLIDEIN_ResourceName": "$$([IfThenElse(IsUndefined(TARGET.GLIDEIN_ResourceName), IfThenElse(IsUndefined(TARGET.GLIDEIN_Site), \"wisc.edu\", TARGET.GLIDEIN_Site), TARGET.GLIDEIN_ResourceName)])", "DAGNodeName": "152+152", "PeriodicRelease": "error", "JobRunCount": 1, "LastRemoteHost": "slot1@e442.chtc.WISC.EDU", "JobPrio": 0, "LocalSysCpu": 0.0d, "ExecutableSize": 7, "RemoteSysCpu": 156.0d, "TransferInput": "/home/xguo23/finally_2/Simulation_condor/data/152/,/home/xguo23/finally_2/Simulation_condor/data
/shared/", "PeriodicHold": false, "WantRemoteIO": true, "CommittedSuspensionTime": 0, "DAGParentNodeNames": "", "CompletionDate": 1446132420, "StreamErr": false, "RecentBlockReadKbytes": 0, "WantRemoteSyscalls": false, "NumShadowStarts": 1, "MachineAttrCpus0": 1, "DAGManNodesLog": "/home/xguo23/finally_2/Simulation_condor/model_3/./mydag.dag.nodes.log", "Owner": "xguo23", "Requirements": "undefined", "DiskUsage": 1250000, "JobLeaseDuration": 2400, "ClusterId": 49582119, "BufferSize": 524288, "IsCHTCSubmit": true, "RecentStatsLifetimeStarter": 1200, "CommittedSlotTime": 24930.0d, "Args": "--type=Matlab --version=R2011b --cmdtorun=simu_condor --unique=152 -- 3", "Environment": "", "LastPublicClaimId": "<128.105.245.242:38884>#1443991450#10374#...", "Iwd": "/home/xguo23/finally_2/Simulation_condor/model_3/152", "QDate": 1446105519, "CurrentHosts": 0, "User": "xguo23@chtc.wisc.edu", "StreamOut": false }
-{ "GlobalJobId": "submit-3.chtc.wisc.edu#49582124.0#1446105525", "StatsLifetimeStarter": 24745, "JobStartDate": 1446107685, "SubmitEventNotes": "DAG Node: 323+323", "JobStatus": 4, "LeaveJobInQueue": false, "AutoClusterId": 13, "StartdPrincipal": "execute-side@matchsession/128.104.55.89", "WantRHEL6": true, "OnExitRemove": true, "JobCurrentStartDate": 1446107685, "CoreSize": 0, "MATCH_EXP_JOBGLIDEIN_ResourceName": "wisc.edu", "ExitStatus": 0, "Rank": 0.0d, "ResidentSetSize": 75000, "RemoteWallClockTime": 24748.0d, "WantCheckpoint": false, "In": "/dev/null", "MaxHosts": 1, "RootDir": "/", "NumRestarts": 0, "RecentBlockWriteKbytes": 0, "DiskUsage_RAW": 1216669, "EnteredCurrentStatus": 1446132433, "ResidentSetSize_RAW": 71248, "RequestDisk": 1000000, "MyType": "Job", "PeriodicRemove": false, "Cmd": "/home/xguo23/finally_2/Simulation_condor/chtcjobwrapper", "CondorVersion": "$CondorVersion: 8.5.0 Sep 16 2015 BuildID: 341710 $", "ShouldTransferFiles": "YES", "TargetType": "Machine", "Min
Hosts": 1, "NumCkpts_RAW": 0, "RequestCpus": 1, "RemoteUserCpu": 21145.0d, "BlockWrites": 1, "NiceUser": false, "Out": "process.out", "ImageSize_RAW": 118000, "BytesSent": 30560.0d, "CumulativeSuspensionTime": 0, "TransferIn": false, "AutoClusterAttrs": "JobUniverse,LastCheckpointPlatform,NumCkpts,ClientMachine,_condor_RequestCpus,_condor_RequestDisk,_condor_RequestGPUs,_condor_RequestMemory,RequestCpus,RequestDisk,RequestGPUs,RequestMemory,BIOCHEM,MachineLastMatchTime,ConcurrencyLimits,NiceUser,Rank,Requirements,ImageSize,MemoryRequirements,User,RemoteGroup,SubmitterGroup,SubmitterUserPrio,Group,WIDsTheme,InteractiveJob,Is_Resumable,WantFlocking,WantGlidein,Scheduler,Owner,JobStart,MemoryUsage,IsExpressQueueJob,DiskUsage,HEP_VO,IsDesktop,OSG_VO,x509userproxysubject,PassedTest,IsLocalCMSJob,IsLocalCMSSlot,IsSAMSlot,IsSAMJob,MaxDiskTempC,IsDedicated,estimated_run_hours,IsCHTCSubmit,RequiresCVMFS,DynamicSlot,PartitionableSlot,Slot1_ExpectedMachineGracefulDrainingCompletion,Slot1_JobSt
arts,Slot1_SelfMonitorAge,Slot2_ExpectedMachineGracefulDrainingCompletion,Slot2_JobStarts,Slot2_SelfMonitorAge,Slot3_ExpectedMachineGracefulDrainingCompletion,Slot3_JobStarts,Slot3_SelfMonitorAge,Slot4_ExpectedMachineGracefulDrainingCompletion,Slot4_JobStarts,Slot4_SelfMonitorAge,Slot5_ExpectedMachineGracefulDrainingCompletion,Slot5_JobStarts,Slot5_SelfMonitorAge,Slot6_ExpectedMachineGracefulDrainingCompletion,Slot6_JobStarts,Slot6_SelfMonitorAge,Slot7_ExpectedMachineGracefulDrainingCompletion,Slot7_JobStarts,Slot7_SelfMonitorAge,Slot8_ExpectedMachineGracefulDrainingCompletion,Slot8_JobStarts,Slot8_SelfMonitorAge,Slot1_TotalTimeClaimedBusy,Slot1_TotalTimeUnclaimedIdle,Slot2_TotalTimeClaimedBusy,Slot2_TotalTimeUnclaimedIdle,Slot3_TotalTimeClaimedBusy,Slot3_TotalTimeUnclaimedIdle,Slot4_TotalTimeClaimedBusy,Slot4_TotalTimeUnclaimedIdle,Slot5_TotalTimeClaimedBusy,Slot5_TotalTimeUnclaimedIdle,Slot6_TotalTimeClaimedBusy,Slot6_TotalTimeUnclaimedIdle,Slot7_TotalTimeClaimedBusy,Slot7_TotalTi
meUnclaimedIdle,Slot8_TotalTimeClaimedBusy,Slot8_TotalTimeUnclaimedIdle,Slot10_ExpectedMachineGracefulDrainingCompletion,Slot10_JobStarts,Slot10_SelfMonitorAge,Slot11_ExpectedMachineGracefulDrainingCompletion,Slot11_JobStarts,Slot11_SelfMonitorAge,Slot12_ExpectedMachineGracefulDrainingCompletion,Slot12_JobStarts,Slot12_SelfMonitorAge,Slot9_ExpectedMachineGracefulDrainingCompletion,Slot9_JobStarts,Slot9_SelfMonitorAge,Slot12_TotalTimeClaimedBusy,Slot10_TotalTimeClaimedBusy,Slot10_TotalTimeUnclaimedIdle,Slot11_TotalTimeClaimedBusy,Slot11_TotalTimeUnclaimedIdle,Slot12_TotalTimeUnclaimedIdle,Slot9_TotalTimeClaimedBusy,Slot9_TotalTimeUnclaimedIdle,Slot13_ExpectedMachineGracefulDrainingCompletion,Slot13_JobStarts,Slot13_SelfMonitorAge,Slot14_ExpectedMachineGracefulDrainingCompletion,Slot14_JobStarts,Slot14_SelfMonitorAge,Slot15_ExpectedMachineGracefulDrainingCompletion,Slot15_JobStarts,Slot15_SelfMonitorAge,Slot16_ExpectedMachineGracefulDrainingCompletion,Slot16_JobStarts,Slot16_SelfMonit
orAge,IsResumable,WHEN_TO_TRANSFER_OUTPUT,_condor_Requestadmin_mutex_1,_condor_Requestadmin_mutex_2,_condor_Requestadmin_mutex_3,_condor_Requestmachine_token,Requestadmin_mutex_1,Requestadmin_mutex_2,Requestadmin_mutex_3,Requestmachine_token,nyehle,IsBuildJob,IsMatlabBuildJob,TotalJobRunTime,NodeOnline,Slot13_TotalTimeClaimedBusy,Slot13_TotalTimeUnclaimedIdle,Slot14_TotalTimeClaimedBusy,Slot14_TotalTimeUnclaimedIdle,Slot15_TotalTimeClaimedBusy,Slot15_TotalTimeUnclaimedIdle,Slot16_TotalTimeClaimedBusy,Slot16_TotalTimeUnclaimedIdle,TmpIsFull,trResumable,RequiresCMSFrontier,Slot17_ExpectedMachineGracefulDrainingCompletion,Slot17_JobStarts,Slot17_SelfMonitorAge,Slot17_TotalTimeClaimedBusy,Slot17_TotalTimeUnclaimedIdle,Slot18_ExpectedMachineGracefulDrainingCompletion,Slot18_JobStarts,Slot18_SelfMonitorAge,Slot18_TotalTimeClaimedBusy,Slot18_TotalTimeUnclaimedIdle,Slot19_ExpectedMachineGracefulDrainingCompletion,Slot19_JobStarts,Slot19_SelfMonitorAge,Slot19_TotalTimeClaimedBusy,Slot19_Tota
lTimeUnclaimedIdle,Slot20_ExpectedMachineGracefulDrainingCompletion,Slot20_JobStarts,Slot20_SelfMonitorAge,Slot20_TotalTimeClaimedBusy,Slot20_TotalTimeUnclaimedIdle,Slot21_ExpectedMachineGracefulDrainingCompletion,Slot21_JobStarts,Slot21_SelfMonitorAge,Slot21_TotalTimeClaimedBusy,Slot21_TotalTimeUnclaimedIdle,Slot22_ExpectedMachineGracefulDrainingCompletion,Slot22_JobStarts,Slot22_SelfMonitorAge,Slot22_TotalTimeClaimedBusy,Slot22_TotalTimeUnclaimedIdle,Slot23_ExpectedMachineGracefulDrainingCompletion,Slot23_JobStarts,Slot23_SelfMonitorAge,Slot23_TotalTimeClaimedBusy,Slot23_TotalTimeUnclaimedIdle,Slot24_ExpectedMachineGracefulDrainingCompletion,Slot24_JobStarts,Slot24_SelfMonitorAge,Slot24_TotalTimeClaimedBusy,Slot24_TotalTimeUnclaimedIdle,Slot25_ExpectedMachineGracefulDrainingCompletion,Slot25_JobStarts,Slot25_SelfMonitorAge,Slot25_TotalTimeClaimedBusy,Slot25_TotalTimeUnclaimedIdle,Slot26_ExpectedMachineGracefulDrainingCompletion,Slot26_JobStarts,Slot26_SelfMonitorAge,Slot26_TotalTi
meClaimedBusy,Slot26_TotalTimeUnclaimedIdle,Slot27_ExpectedMachineGracefulDrainingCompletion,Slot27_JobStarts,Slot27_SelfMonitorAge,Slot27_TotalTimeClaimedBusy,Slot27_TotalTimeUnclaimedIdle,Slot28_ExpectedMachineGracefulDrainingCompletion,Slot28_JobStarts,Slot28_SelfMonitorAge,Slot28_TotalTimeClaimedBusy,Slot28_TotalTimeUnclaimedIdle,Slot29_ExpectedMachineGracefulDrainingCompletion,Slot29_JobStarts,Slot29_SelfMonitorAge,Slot29_TotalTimeClaimedBusy,Slot29_TotalTimeUnclaimedIdle,Slot30_ExpectedMachineGracefulDrainingCompletion,Slot30_JobStarts,Slot30_SelfMonitorAge,Slot30_TotalTimeClaimedBusy,Slot30_TotalTimeUnclaimedIdle,Slot31_ExpectedMachineGracefulDrainingCompletion,Slot31_JobStarts,Slot31_SelfMonitorAge,Slot31_TotalTimeClaimedBusy,Slot31_TotalTimeUnclaimedIdle,Slot32_ExpectedMachineGracefulDrainingCompletion,Slot32_JobStarts,Slot32_SelfMonitorAge,Slot32_TotalTimeClaimedBusy,Slot32_TotalTimeUnclaimedIdle,ResidentSetSize", "NumCkpts": 0, "Err": "process.err", "RecentBlockWrites": 0
, "JobFinishedHookDone": 1446132434, "ProcId": 0, "ImageSize": 125000, "JobUniverse": 5, "EncryptExecuteDirectory": false, "TransferInputSizeMB": 0, "RecentBlockReads": 314, "SpooledOutputFiles": "harvest.log,CURLTIME_3853266,ChtcWrapper323.out,AuditLog.323,simu_3_323.txt,323.out", "BlockWriteKbytes": 4, "WhenToTransferOutput": "ON_EXIT", "JobCurrentStartExecutingDate": 1446107686, "ExitBySignal": false, "LastMatchTime": 1446107685, "OnExitHold": false, "OrigMaxHosts": 1, "RequestMemory": 1000, "NumJobStarts": 1, "TerminationPending": true, "TotalSuspensions": 0, "BlockReads": 1142, "DAGManJobId": 49581933, "MemoryUsage": 73, "ExitCode": 0, "JobNotification": 0, "BlockReadKbytes": 43788, "NumJobMatches": 1, "LocalUserCpu": 0.0d, "LastJobStatus": 2, "BufferBlockSize": 32768, "CommittedTime": 24748, "ExecutableSize_RAW": 6, "LastSuspensionTime": 0, "Matlab": "R2011b", "UserLog": "/home/xguo23/finally_2/Simulation_condor/model_3/323/process.log", "DAGManNodesMask": "0,1,2,4,5,7,9,10,11
,12,13,16,17,24,27", "CumulativeSlotTime": 24748.0d, "LastJobLeaseRenewal": 1446132433, "MachineAttrSlotWeight0": 1, "NumSystemHolds": 0, "BytesRecvd": 285054.0d, "CondorPlatform": "$CondorPlatform: X86_64-RedHat_6.6 $", "JOBGLIDEIN_ResourceName": "$$([IfThenElse(IsUndefined(TARGET.GLIDEIN_ResourceName), IfThenElse(IsUndefined(TARGET.GLIDEIN_Site), \"wisc.edu\", TARGET.GLIDEIN_Site), TARGET.GLIDEIN_ResourceName)])", "DAGNodeName": "323+323", "PeriodicRelease": "error", "JobRunCount": 1, "LastRemoteHost": "slot1@c070.chtc.wisc.edu", "JobPrio": 0, "LocalSysCpu": 0.0d, "ExecutableSize": 7, "RemoteSysCpu": 175.0d, "TransferInput": "/home/xguo23/finally_2/Simulation_condor/data/323/,/home/xguo23/finally_2/Simulation_condor/data/shared/", "PeriodicHold": false, "WantRemoteIO": true, "CommittedSuspensionTime": 0, "DAGParentNodeNames": "", "CompletionDate": 1446132433, "StreamErr": false, "RecentBlockReadKbytes": 4224, "WantRemoteSyscalls": false, "NumShadowStarts": 1, "MachineAttrCpus0": 1
, "DAGManNodesLog": "/home/xguo23/finally_2/Simulation_condor/model_3/./mydag.dag.nodes.log", "Owner": "xguo23", "Requirements": "undefined", "DiskUsage": 1250000, "JobLeaseDuration": 2400, "ClusterId": 49582124, "BufferSize": 524288, "IsCHTCSubmit": true, "RecentStatsLifetimeStarter": 1200, "CommittedSlotTime": 24748.0d, "Args": "--type=Matlab --version=R2011b --cmdtorun=simu_condor --unique=323 -- 3", "Environment": "", "LastPublicClaimId": "<128.104.55.89:32652>#1445371750#1302#...", "Iwd": "/home/xguo23/finally_2/Simulation_condor/model_3/323", "QDate": 1446105525, "CurrentHosts": 0, "User": "xguo23@chtc.wisc.edu", "StreamOut": false }
-{ "GlobalJobId": "submit-3.chtc.wisc.edu#49582148.0#1446105547", "StatsLifetimeStarter": 26230, "JobStartDate": 1446107686, "SubmitEventNotes": "DAG Node: 162+162", "JobStatus": 4, "LeaveJobInQueue": false, "AutoClusterId": 24, "StartdPrincipal": "execute-side@matchsession/128.105.245.170", "WantRHEL6": true, "OnExitRemove": true, "JobCurrentStartDate": 1446107686, "CoreSize": 0, "MATCH_EXP_JOBGLIDEIN_ResourceName": "wisc.edu", "ExitStatus": 0, "Rank": 0.0d, "ResidentSetSize": 150000, "RemoteWallClockTime": 26233.0d, "WantCheckpoint": false, "In": "/dev/null", "MaxHosts": 1, "RootDir": "/", "NumRestarts": 0, "RecentBlockWriteKbytes": 0, "DiskUsage_RAW": 1216669, "EnteredCurrentStatus": 1446133919, "ResidentSetSize_RAW": 126384, "RequestDisk": 1000000, "MyType": "Job", "PeriodicRemove": false, "Cmd": "/home/xguo23/finally_2/Simulation_condor/chtcjobwrapper", "CondorVersion": "$CondorVersion: 8.5.0 Sep 16 2015 BuildID: 341710 $", "ShouldTransferFiles": "YES", "TargetType": "Machine",
"MinHosts": 1, "NumCkpts_RAW": 0, "RequestCpus": 1, "RemoteUserCpu": 26088.0d, "BlockWrites": 0, "NiceUser": false, "Out": "process.out", "ImageSize_RAW": 811948, "BytesSent": 30612.0d, "CumulativeSuspensionTime": 0, "TransferIn": false, "AutoClusterAttrs": "JobUniverse,LastCheckpointPlatform,NumCkpts,ClientMachine,_condor_RequestCpus,_condor_RequestDisk,_condor_RequestGPUs,_condor_RequestMemory,RequestCpus,RequestDisk,RequestGPUs,RequestMemory,BIOCHEM,MachineLastMatchTime,ConcurrencyLimits,NiceUser,Rank,Requirements,ImageSize,MemoryRequirements,User,RemoteGroup,SubmitterGroup,SubmitterUserPrio,Group,WIDsTheme,InteractiveJob,Is_Resumable,WantFlocking,WantGlidein,Scheduler,Owner,JobStart,MemoryUsage,IsExpressQueueJob,DiskUsage,HEP_VO,IsDesktop,OSG_VO,x509userproxysubject,PassedTest,IsLocalCMSJob,IsLocalCMSSlot,IsSAMSlot,IsSAMJob,MaxDiskTempC,IsDedicated,estimated_run_hours,IsCHTCSubmit,RequiresCVMFS,DynamicSlot,PartitionableSlot,Slot1_ExpectedMachineGracefulDrainingCompletion,Slot1_J
obStarts,Slot1_SelfMonitorAge,Slot2_ExpectedMachineGracefulDrainingCompletion,Slot2_JobStarts,Slot2_SelfMonitorAge,Slot3_ExpectedMachineGracefulDrainingCompletion,Slot3_JobStarts,Slot3_SelfMonitorAge,Slot4_ExpectedMachineGracefulDrainingCompletion,Slot4_JobStarts,Slot4_SelfMonitorAge,Slot5_ExpectedMachineGracefulDrainingCompletion,Slot5_JobStarts,Slot5_SelfMonitorAge,Slot6_ExpectedMachineGracefulDrainingCompletion,Slot6_JobStarts,Slot6_SelfMonitorAge,Slot7_ExpectedMachineGracefulDrainingCompletion,Slot7_JobStarts,Slot7_SelfMonitorAge,Slot8_ExpectedMachineGracefulDrainingCompletion,Slot8_JobStarts,Slot8_SelfMonitorAge,Slot1_TotalTimeClaimedBusy,Slot1_TotalTimeUnclaimedIdle,Slot2_TotalTimeClaimedBusy,Slot2_TotalTimeUnclaimedIdle,Slot3_TotalTimeClaimedBusy,Slot3_TotalTimeUnclaimedIdle,Slot4_TotalTimeClaimedBusy,Slot4_TotalTimeUnclaimedIdle,Slot5_TotalTimeClaimedBusy,Slot5_TotalTimeUnclaimedIdle,Slot6_TotalTimeClaimedBusy,Slot6_TotalTimeUnclaimedIdle,Slot7_TotalTimeClaimedBusy,Slot7_Tot
alTimeUnclaimedIdle,Slot8_TotalTimeClaimedBusy,Slot8_TotalTimeUnclaimedIdle,Slot10_ExpectedMachineGracefulDrainingCompletion,Slot10_JobStarts,Slot10_SelfMonitorAge,Slot11_ExpectedMachineGracefulDrainingCompletion,Slot11_JobStarts,Slot11_SelfMonitorAge,Slot12_ExpectedMachineGracefulDrainingCompletion,Slot12_JobStarts,Slot12_SelfMonitorAge,Slot9_ExpectedMachineGracefulDrainingCompletion,Slot9_JobStarts,Slot9_SelfMonitorAge,Slot12_TotalTimeClaimedBusy,Slot10_TotalTimeClaimedBusy,Slot10_TotalTimeUnclaimedIdle,Slot11_TotalTimeClaimedBusy,Slot11_TotalTimeUnclaimedIdle,Slot12_TotalTimeUnclaimedIdle,Slot9_TotalTimeClaimedBusy,Slot9_TotalTimeUnclaimedIdle,Slot13_ExpectedMachineGracefulDrainingCompletion,Slot13_JobStarts,Slot13_SelfMonitorAge,Slot14_ExpectedMachineGracefulDrainingCompletion,Slot14_JobStarts,Slot14_SelfMonitorAge,Slot15_ExpectedMachineGracefulDrainingCompletion,Slot15_JobStarts,Slot15_SelfMonitorAge,Slot16_ExpectedMachineGracefulDrainingCompletion,Slot16_JobStarts,Slot16_SelfM
onitorAge,IsResumable,WHEN_TO_TRANSFER_OUTPUT,_condor_Requestadmin_mutex_1,_condor_Requestadmin_mutex_2,_condor_Requestadmin_mutex_3,_condor_Requestmachine_token,Requestadmin_mutex_1,Requestadmin_mutex_2,Requestadmin_mutex_3,Requestmachine_token,nyehle,IsBuildJob,IsMatlabBuildJob,TotalJobRunTime,NodeOnline,Slot13_TotalTimeClaimedBusy,Slot13_TotalTimeUnclaimedIdle,Slot14_TotalTimeClaimedBusy,Slot14_TotalTimeUnclaimedIdle,Slot15_TotalTimeClaimedBusy,Slot15_TotalTimeUnclaimedIdle,Slot16_TotalTimeClaimedBusy,Slot16_TotalTimeUnclaimedIdle,TmpIsFull,trResumable,RequiresCMSFrontier,Slot17_ExpectedMachineGracefulDrainingCompletion,Slot17_JobStarts,Slot17_SelfMonitorAge,Slot17_TotalTimeClaimedBusy,Slot17_TotalTimeUnclaimedIdle,Slot18_ExpectedMachineGracefulDrainingCompletion,Slot18_JobStarts,Slot18_SelfMonitorAge,Slot18_TotalTimeClaimedBusy,Slot18_TotalTimeUnclaimedIdle,Slot19_ExpectedMachineGracefulDrainingCompletion,Slot19_JobStarts,Slot19_SelfMonitorAge,Slot19_TotalTimeClaimedBusy,Slot19_
TotalTimeUnclaimedIdle,Slot20_ExpectedMachineGracefulDrainingCompletion,Slot20_JobStarts,Slot20_SelfMonitorAge,Slot20_TotalTimeClaimedBusy,Slot20_TotalTimeUnclaimedIdle,Slot21_ExpectedMachineGracefulDrainingCompletion,Slot21_JobStarts,Slot21_SelfMonitorAge,Slot21_TotalTimeClaimedBusy,Slot21_TotalTimeUnclaimedIdle,Slot22_ExpectedMachineGracefulDrainingCompletion,Slot22_JobStarts,Slot22_SelfMonitorAge,Slot22_TotalTimeClaimedBusy,Slot22_TotalTimeUnclaimedIdle,Slot23_ExpectedMachineGracefulDrainingCompletion,Slot23_JobStarts,Slot23_SelfMonitorAge,Slot23_TotalTimeClaimedBusy,Slot23_TotalTimeUnclaimedIdle,Slot24_ExpectedMachineGracefulDrainingCompletion,Slot24_JobStarts,Slot24_SelfMonitorAge,Slot24_TotalTimeClaimedBusy,Slot24_TotalTimeUnclaimedIdle,Slot25_ExpectedMachineGracefulDrainingCompletion,Slot25_JobStarts,Slot25_SelfMonitorAge,Slot25_TotalTimeClaimedBusy,Slot25_TotalTimeUnclaimedIdle,Slot26_ExpectedMachineGracefulDrainingCompletion,Slot26_JobStarts,Slot26_SelfMonitorAge,Slot26_Tot
alTimeClaimedBusy,Slot26_TotalTimeUnclaimedIdle,Slot27_ExpectedMachineGracefulDrainingCompletion,Slot27_JobStarts,Slot27_SelfMonitorAge,Slot27_TotalTimeClaimedBusy,Slot27_TotalTimeUnclaimedIdle,Slot28_ExpectedMachineGracefulDrainingCompletion,Slot28_JobStarts,Slot28_SelfMonitorAge,Slot28_TotalTimeClaimedBusy,Slot28_TotalTimeUnclaimedIdle,Slot29_ExpectedMachineGracefulDrainingCompletion,Slot29_JobStarts,Slot29_SelfMonitorAge,Slot29_TotalTimeClaimedBusy,Slot29_TotalTimeUnclaimedIdle,Slot30_ExpectedMachineGracefulDrainingCompletion,Slot30_JobStarts,Slot30_SelfMonitorAge,Slot30_TotalTimeClaimedBusy,Slot30_TotalTimeUnclaimedIdle,Slot31_ExpectedMachineGracefulDrainingCompletion,Slot31_JobStarts,Slot31_SelfMonitorAge,Slot31_TotalTimeClaimedBusy,Slot31_TotalTimeUnclaimedIdle,Slot32_ExpectedMachineGracefulDrainingCompletion,Slot32_JobStarts,Slot32_SelfMonitorAge,Slot32_TotalTimeClaimedBusy,Slot32_TotalTimeUnclaimedIdle,ResidentSetSize", "NumCkpts": 0, "Err": "process.err", "RecentBlockWrites
": 0, "JobFinishedHookDone": 1446133919, "ProcId": 0, "ImageSize": 1000000, "JobUniverse": 5, "EncryptExecuteDirectory": false, "TransferInputSizeMB": 0, "RecentBlockReads": 0, "SpooledOutputFiles": "CURLTIME_1114551,ChtcWrapper162.out,AuditLog.162,simu_3_162.txt,harvest.log,162.out", "BlockWriteKbytes": 0, "WhenToTransferOutput": "ON_EXIT", "JobCurrentStartExecutingDate": 1446107688, "ExitBySignal": false, "LastMatchTime": 1446107686, "OnExitHold": false, "OrigMaxHosts": 1, "RequestMemory": 1000, "NumJobStarts": 1, "TerminationPending": true, "TotalSuspensions": 0, "BlockReads": 0, "DAGManJobId": 49581933, "MemoryUsage": 146, "ExitCode": 0, "JobNotification": 0, "BlockReadKbytes": 0, "NumJobMatches": 1, "LocalUserCpu": 0.0d, "LastJobStatus": 2, "BufferBlockSize": 32768, "CommittedTime": 26233, "ExecutableSize_RAW": 6, "LastSuspensionTime": 0, "Matlab": "R2011b", "UserLog": "/home/xguo23/finally_2/Simulation_condor/model_3/162/process.log", "DAGManNodesMask": "0,1,2,4,5,7,9,10,11,12
,13,16,17,24,27", "CumulativeSlotTime": 26233.0d, "LastJobLeaseRenewal": 1446133919, "MachineAttrSlotWeight0": 1, "NumSystemHolds": 0, "BytesRecvd": 285054.0d, "CondorPlatform": "$CondorPlatform: X86_64-RedHat_6.6 $", "JOBGLIDEIN_ResourceName": "$$([IfThenElse(IsUndefined(TARGET.GLIDEIN_ResourceName), IfThenElse(IsUndefined(TARGET.GLIDEIN_Site), \"wisc.edu\", TARGET.GLIDEIN_Site), TARGET.GLIDEIN_ResourceName)])", "DAGNodeName": "162+162", "PeriodicRelease": "error", "JobRunCount": 1, "LastRemoteHost": "slot1@e370.chtc.WISC.EDU", "JobPrio": 0, "LocalSysCpu": 0.0d, "ExecutableSize": 7, "RemoteSysCpu": 96.0d, "TransferInput": "/home/xguo23/finally_2/Simulation_condor/data/162/,/home/xguo23/finally_2/Simulation_condor/data/shared/", "PeriodicHold": false, "WantRemoteIO": true, "CommittedSuspensionTime": 0, "DAGParentNodeNames": "", "CompletionDate": 1446133919, "StreamErr": false, "RecentBlockReadKbytes": 0, "WantRemoteSyscalls": false, "NumShadowStarts": 1, "MachineAttrCpus0": 1, "DAGM
anNodesLog": "/home/xguo23/finally_2/Simulation_condor/model_3/./mydag.dag.nodes.log", "Owner": "xguo23", "Requirements": "undefined", "DiskUsage": 1250000, "JobLeaseDuration": 2400, "ClusterId": 49582148, "BufferSize": 524288, "IsCHTCSubmit": true, "RecentStatsLifetimeStarter": 1200, "CommittedSlotTime": 26233.0d, "Args": "--type=Matlab --version=R2011b --cmdtorun=simu_condor --unique=162 -- 3", "Environment": "", "LastPublicClaimId": "<128.105.245.170:9482>#1443991414#13008#...", "Iwd": "/home/xguo23/finally_2/Simulation_condor/model_3/162", "QDate": 1446105547, "CurrentHosts": 0, "User": "xguo23@chtc.wisc.edu", "StreamOut": false }
-{ "GlobalJobId": "submit-3.chtc.wisc.edu#49582154.0#1446105553", "StatsLifetimeStarter": 25874, "JobStartDate": 1446107686, "SubmitEventNotes": "DAG Node: 333+333", "JobStatus": 4, "LeaveJobInQueue": false, "StartdPrincipal": "execute-side@matchsession/128.105.245.120", "WantRHEL6": true, "OnExitRemove": true, "JobCurrentStartDate": 1446107686, "CoreSize": 0, "MATCH_EXP_JOBGLIDEIN_ResourceName": "wisc.edu", "Rank": 0.0d, "ExitStatus": 0, "ResidentSetSize": 150000, "RemoteWallClockTime": 25876.0d, "WantCheckpoint": false, "In": "/dev/null", "MaxHosts": 1, "RootDir": "/", "NumRestarts": 0, "RecentBlockWriteKbytes": 0, "DiskUsage_RAW": 1216669, "EnteredCurrentStatus": 1446133562, "ResidentSetSize_RAW": 125740, "RequestDisk": 1000000, "MyType": "Job", "PeriodicRemove": false, "Cmd": "/home/xguo23/finally_2/Simulation_condor/chtcjobwrapper", "CondorVersion": "$CondorVersion: 8.5.0 Sep 16 2015 BuildID: 341710 $", "ShouldTransferFiles": "YES", "TargetType": "Machine", "MinHosts": 1, "NumCk
pts_RAW": 0, "RequestCpus": 1, "RemoteUserCpu": 25692.0d, "BlockWrites": 0, "NiceUser": false, "Out": "process.out", "ImageSize_RAW": 811948, "BytesSent": 30542.0d, "CumulativeSuspensionTime": 0, "TransferIn": false, "NumCkpts": 0, "Err": "process.err", "RecentBlockWrites": 0, "JobFinishedHookDone": 1446133562, "ProcId": 0, "ImageSize": 1000000, "JobUniverse": 5, "EncryptExecuteDirectory": false, "TransferInputSizeMB": 0, "RecentBlockReads": 0, "SpooledOutputFiles": "CURLTIME_523030,ChtcWrapper333.out,AuditLog.333,simu_3_333.txt,harvest.log,333.out", "BlockWriteKbytes": 0, "WhenToTransferOutput": "ON_EXIT", "JobCurrentStartExecutingDate": 1446107688, "ExitBySignal": false, "LastMatchTime": 1446107686, "OnExitHold": false, "OrigMaxHosts": 1, "RequestMemory": 1000, "NumJobStarts": 1, "TerminationPending": true, "TotalSuspensions": 0, "BlockReads": 0, "DAGManJobId": 49581933, "MemoryUsage": 146, "ExitCode": 0, "JobNotification": 0, "BlockReadKbytes": 0, "NumJobMatches": 1, "LocalUserCp
u": 0.0d, "LastJobStatus": 2, "BufferBlockSize": 32768, "CommittedTime": 25876, "ExecutableSize_RAW": 6, "LastSuspensionTime": 0, "Matlab": "R2011b", "UserLog": "/home/xguo23/finally_2/Simulation_condor/model_3/333/process.log", "DAGManNodesMask": "0,1,2,4,5,7,9,10,11,12,13,16,17,24,27", "CumulativeSlotTime": 25876.0d, "LastJobLeaseRenewal": 1446133562, "MachineAttrSlotWeight0": 1, "NumSystemHolds": 0, "BytesRecvd": 285054.0d, "CondorPlatform": "$CondorPlatform: X86_64-RedHat_6.6 $", "JOBGLIDEIN_ResourceName": "$$([IfThenElse(IsUndefined(TARGET.GLIDEIN_ResourceName), IfThenElse(IsUndefined(TARGET.GLIDEIN_Site), \"wisc.edu\", TARGET.GLIDEIN_Site), TARGET.GLIDEIN_ResourceName)])", "DAGNodeName": "333+333", "PeriodicRelease": "error", "JobRunCount": 1, "LastRemoteHost": "slot1@e320.chtc.wisc.edu", "JobPrio": 0, "LocalSysCpu": 0.0d, "ExecutableSize": 7, "RemoteSysCpu": 157.0d, "TransferInput": "/home/xguo23/finally_2/Simulation_condor/data/333/,/home/xguo23/finally_2/Simulation_condor/d
ata/shared/", "PeriodicHold": false, "WantRemoteIO": true, "CommittedSuspensionTime": 0, "DAGParentNodeNames": "", "CompletionDate": 1446133562, "StreamErr": false, "RecentBlockReadKbytes": 0, "WantRemoteSyscalls": false, "NumShadowStarts": 1, "MachineAttrCpus0": 1, "DAGManNodesLog": "/home/xguo23/finally_2/Simulation_condor/model_3/./mydag.dag.nodes.log", "Owner": "xguo23", "Requirements": "undefined", "DiskUsage": 1250000, "JobLeaseDuration": 2400, "ClusterId": 49582154, "BufferSize": 524288, "IsCHTCSubmit": true, "RecentStatsLifetimeStarter": 1200, "CommittedSlotTime": 25876.0d, "Args": "--type=Matlab --version=R2011b --cmdtorun=simu_condor --unique=333 -- 3", "Environment": "", "LastPublicClaimId": "<128.105.245.120:45185>#1443991409#14242#...", "Iwd": "/home/xguo23/finally_2/Simulation_condor/model_3/333", "QDate": 1446105553, "CurrentHosts": 0, "User": "xguo23@chtc.wisc.edu", "StreamOut": false }
-{ "GlobalJobId": "submit-3.chtc.wisc.edu#49582177.0#1446105581", "StatsLifetimeStarter": 25025, "JobStartDate": 1446108665, "SubmitEventNotes": "DAG Node: 145+145", "JobStatus": 4, "LeaveJobInQueue": false, "StartdPrincipal": "execute-side@matchsession/128.104.55.57", "WantRHEL6": true, "OnExitRemove": true, "JobCurrentStartDate": 1446108665, "CoreSize": 0, "MATCH_EXP_JOBGLIDEIN_ResourceName": "wisc.edu", "Rank": 0.0d, "ExitStatus": 0, "ResidentSetSize": 75000, "RemoteWallClockTime": 25026.0d, "WantCheckpoint": false, "In": "/dev/null", "MaxHosts": 1, "RootDir": "/", "NumRestarts": 0, "RecentBlockWriteKbytes": 4, "DiskUsage_RAW": 1216669, "EnteredCurrentStatus": 1446133691, "ResidentSetSize_RAW": 73308, "RequestDisk": 1000000, "MyType": "Job", "PeriodicRemove": false, "Cmd": "/home/xguo23/finally_2/Simulation_condor/chtcjobwrapper", "CondorVersion": "$CondorVersion: 8.5.0 Sep 16 2015 BuildID: 341710 $", "ShouldTransferFiles": "YES", "TargetType": "Machine", "MinHosts": 1, "NumCkpts_
RAW": 0, "RequestCpus": 1, "RemoteUserCpu": 24770.0d, "BlockWrites": 1, "NiceUser": false, "Out": "process.out", "ImageSize_RAW": 120972, "BytesSent": 28290.0d, "CumulativeSuspensionTime": 0, "TransferIn": false, "NumCkpts": 0, "Err": "process.err", "RecentBlockWrites": 1, "JobFinishedHookDone": 1446133691, "ProcId": 0, "ImageSize": 125000, "JobUniverse": 5, "EncryptExecuteDirectory": false, "TransferInputSizeMB": 0, "RecentBlockReads": 146, "SpooledOutputFiles": "CURLTIME_4179033,ChtcWrapper145.out,AuditLog.145,simu_3_145.txt,harvest.log,145.out", "BlockWriteKbytes": 4, "WhenToTransferOutput": "ON_EXIT", "JobCurrentStartExecutingDate": 1446108666, "ExitBySignal": false, "LastMatchTime": 1446108665, "OnExitHold": false, "OrigMaxHosts": 1, "RequestMemory": 1000, "NumJobStarts": 1, "TerminationPending": true, "TotalSuspensions": 0, "BlockReads": 796, "DAGManJobId": 49581933, "MemoryUsage": 73, "ExitCode": 0, "JobNotification": 0, "BlockReadKbytes": 28476, "NumJobMatches": 1, "LocalUse
rCpu": 0.0d, "LastJobStatus": 2, "BufferBlockSize": 32768, "CommittedTime": 25026, "ExecutableSize_RAW": 6, "LastSuspensionTime": 0, "Matlab": "R2011b", "UserLog": "/home/xguo23/finally_2/Simulation_condor/model_3/145/process.log", "DAGManNodesMask": "0,1,2,4,5,7,9,10,11,12,13,16,17,24,27", "CumulativeSlotTime": 25026.0d, "LastJobLeaseRenewal": 1446133691, "MachineAttrSlotWeight0": 1, "NumSystemHolds": 0, "BytesRecvd": 285054.0d, "CondorPlatform": "$CondorPlatform: X86_64-RedHat_6.6 $", "JOBGLIDEIN_ResourceName": "$$([IfThenElse(IsUndefined(TARGET.GLIDEIN_ResourceName), IfThenElse(IsUndefined(TARGET.GLIDEIN_Site), \"wisc.edu\", TARGET.GLIDEIN_Site), TARGET.GLIDEIN_ResourceName)])", "DAGNodeName": "145+145", "PeriodicRelease": "error", "JobRunCount": 1, "LastRemoteHost": "slot1@c038.chtc.wisc.edu", "JobPrio": 0, "LocalSysCpu": 0.0d, "ExecutableSize": 7, "RemoteSysCpu": 217.0d, "TransferInput": "/home/xguo23/finally_2/Simulation_condor/data/145/,/home/xguo23/finally_2/Simulation_condo
r/data/shared/", "PeriodicHold": false, "WantRemoteIO": true, "CommittedSuspensionTime": 0, "DAGParentNodeNames": "", "CompletionDate": 1446133691, "StreamErr": false, "RecentBlockReadKbytes": 1932, "WantRemoteSyscalls": false, "NumShadowStarts": 1, "MachineAttrCpus0": 1, "DAGManNodesLog": "/home/xguo23/finally_2/Simulation_condor/model_3/./mydag.dag.nodes.log", "Owner": "xguo23", "Requirements": "undefined", "DiskUsage": 1250000, "JobLeaseDuration": 2400, "ClusterId": 49582177, "BufferSize": 524288, "IsCHTCSubmit": true, "RecentStatsLifetimeStarter": 1200, "CommittedSlotTime": 25026.0d, "Args": "--type=Matlab --version=R2011b --cmdtorun=simu_condor --unique=145 -- 3", "Environment": "", "LastPublicClaimId": "<128.104.55.57:49793>#1445322694#1541#...", "Iwd": "/home/xguo23/finally_2/Simulation_condor/model_3/145", "QDate": 1446105581, "CurrentHosts": 0, "User": "xguo23@chtc.wisc.edu", "StreamOut": false }
-{ "GlobalJobId": "submit-3.chtc.wisc.edu#49582178.0#1446105581", "StatsLifetimeStarter": 24871, "JobStartDate": 1446108666, "SubmitEventNotes": "DAG Node: 154+154", "JobStatus": 4, "LeaveJobInQueue": false, "StartdPrincipal": "execute-side@matchsession/128.105.245.158", "WantRHEL6": true, "OnExitRemove": true, "JobCurrentStartDate": 1446108666, "CoreSize": 0, "MATCH_EXP_JOBGLIDEIN_ResourceName": "wisc.edu", "Rank": 0.0d, "ExitStatus": 0, "ResidentSetSize": 150000, "RemoteWallClockTime": 24874.0d, "WantCheckpoint": false, "In": "/dev/null", "MaxHosts": 1, "RootDir": "/", "NumRestarts": 0, "RecentBlockWriteKbytes": 0, "DiskUsage_RAW": 1216669, "EnteredCurrentStatus": 1446133540, "ResidentSetSize_RAW": 125792, "RequestDisk": 1000000, "MyType": "Job", "PeriodicRemove": false, "Cmd": "/home/xguo23/finally_2/Simulation_condor/chtcjobwrapper", "CondorVersion": "$CondorVersion: 8.5.0 Sep 16 2015 BuildID: 341710 $", "ShouldTransferFiles": "YES", "TargetType": "Machine", "MinHosts": 1, "NumCk
pts_RAW": 0, "RequestCpus": 1, "RemoteUserCpu": 24626.0d, "BlockWrites": 0, "NiceUser": false, "Out": "process.out", "ImageSize_RAW": 811948, "BytesSent": 30559.0d, "CumulativeSuspensionTime": 0, "TransferIn": false, "NumCkpts": 0, "Err": "process.err", "RecentBlockWrites": 0, "JobFinishedHookDone": 1446133540, "ProcId": 0, "ImageSize": 1000000, "JobUniverse": 5, "EncryptExecuteDirectory": false, "TransferInputSizeMB": 0, "RecentBlockReads": 0, "SpooledOutputFiles": "CURLTIME_1382128,ChtcWrapper154.out,AuditLog.154,simu_3_154.txt,harvest.log,154.out", "BlockWriteKbytes": 0, "WhenToTransferOutput": "ON_EXIT", "JobCurrentStartExecutingDate": 1446108668, "ExitBySignal": false, "LastMatchTime": 1446108666, "OnExitHold": false, "OrigMaxHosts": 1, "RequestMemory": 1000, "NumJobStarts": 1, "TerminationPending": true, "TotalSuspensions": 0, "BlockReads": 0, "DAGManJobId": 49581933, "MemoryUsage": 146, "ExitCode": 0, "JobNotification": 0, "BlockReadKbytes": 0, "NumJobMatches": 1, "LocalUserC
pu": 0.0d, "LastJobStatus": 2, "BufferBlockSize": 32768, "CommittedTime": 24874, "ExecutableSize_RAW": 6, "LastSuspensionTime": 0, "Matlab": "R2011b", "UserLog": "/home/xguo23/finally_2/Simulation_condor/model_3/154/process.log", "DAGManNodesMask": "0,1,2,4,5,7,9,10,11,12,13,16,17,24,27", "CumulativeSlotTime": 24874.0d, "LastJobLeaseRenewal": 1446133540, "MachineAttrSlotWeight0": 1, "NumSystemHolds": 0, "BytesRecvd": 285054.0d, "CondorPlatform": "$CondorPlatform: X86_64-RedHat_6.6 $", "JOBGLIDEIN_ResourceName": "$$([IfThenElse(IsUndefined(TARGET.GLIDEIN_ResourceName), IfThenElse(IsUndefined(TARGET.GLIDEIN_Site), \"wisc.edu\", TARGET.GLIDEIN_Site), TARGET.GLIDEIN_ResourceName)])", "DAGNodeName": "154+154", "PeriodicRelease": "error", "JobRunCount": 1, "LastRemoteHost": "slot1@e358.chtc.wisc.edu", "JobPrio": 0, "LocalSysCpu": 0.0d, "ExecutableSize": 7, "RemoteSysCpu": 183.0d, "TransferInput": "/home/xguo23/f
<TRUNCATED>
[03/13] incubator-asterixdb git commit: Improve Error Handling in
Local Directory Feeds
Posted by am...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/FunctionalTester.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/FunctionalTester.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/FunctionalTester.java
index c9ce47c..75b2800 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/FunctionalTester.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/FunctionalTester.java
@@ -38,6 +38,7 @@ import org.apache.asterix.external.classad.Lexer.TokenType;
import org.apache.asterix.external.classad.PrettyPrint;
import org.apache.asterix.external.classad.StringLexerSource;
import org.apache.asterix.external.classad.Value;
+import org.apache.asterix.external.classad.object.pool.ClassAdObjectPool;
import org.apache.asterix.external.library.ClassAdParser;
import org.apache.asterix.om.base.AMutableString;
import org.apache.hyracks.api.exceptions.HyracksDataException;
@@ -54,15 +55,15 @@ public class FunctionalTester {
public ExprTreeHolder tree;
public Value value;
- public Variable(String name, ExprTree tree) {
+ public Variable(String name, ExprTree tree, ClassAdObjectPool objectPool) {
this.name = name;
- this.tree = new ExprTreeHolder(tree);
+ this.tree = new ExprTreeHolder(tree, objectPool);
this.isTree = true;
}
- public Variable() {
+ public Variable(ClassAdObjectPool objectPool) {
this.name = null;
- this.tree = new ExprTreeHolder(null);
+ this.tree = new ExprTreeHolder(objectPool);
this.isTree = false;
}
@@ -73,8 +74,9 @@ public class FunctionalTester {
this.tree = null;
}
- public void getStringRepresentation(AMutableCharArrayString representation) throws HyracksDataException {
- ClassAdUnParser unparser = new PrettyPrint();
+ public void getStringRepresentation(AMutableCharArrayString representation, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ ClassAdUnParser unparser = new PrettyPrint(objectPool);
if (isTree) {
unparser.unparse(representation, tree);
@@ -179,7 +181,7 @@ public class FunctionalTester {
*
*--------------------------------------------------------------------*/
- public static int test(int argc, String[] argv) throws IOException {
+ public static int test(int argc, String[] argv, ClassAdObjectPool objectPool) throws IOException {
// here
boolean quit;
AMutableString line = new AMutableString(null);
@@ -194,10 +196,10 @@ public class FunctionalTester {
boolean good_line;
Command command;
- good_line = replace_variables(line, state, parameters);
+ good_line = replace_variables(line, state, parameters, objectPool);
if (good_line) {
command = get_command(line, parameters);
- quit = handle_command(command, line, state, parameters);
+ quit = handle_command(command, line, state, parameters, objectPool);
}
}
print_final_state(state);
@@ -327,14 +329,14 @@ public class FunctionalTester {
*
* @throws HyracksDataException
*********************************************************************/
- public static boolean replace_variables(AMutableString mutableLine, State state, Parameters parameters)
- throws HyracksDataException {
+ public static boolean replace_variables(AMutableString mutableLine, State state, Parameters parameters,
+ ClassAdObjectPool objectPool) throws HyracksDataException {
boolean good_line;
String error;
good_line = true;
error = "";
- Variable var = new Variable();
+ Variable var = new Variable(objectPool);
for (;;) {
int dollar;
int current_position;
@@ -366,7 +368,7 @@ public class FunctionalTester {
error += "'";
break;
}
- var.getStringRepresentation(variable_value);
+ var.getStringRepresentation(variable_value, objectPool);
// We have to be careful with substr() because with gcc 2.96, it likes to
// assert/except if you give it values that are too large.
@@ -457,8 +459,8 @@ public class FunctionalTester {
*
* @throws IOException
*********************************************************************/
- public static boolean handle_command(Command command, AMutableString line, State state, Parameters parameters)
- throws IOException {
+ public static boolean handle_command(Command command, AMutableString line, State state, Parameters parameters,
+ ClassAdObjectPool objectPool) throws IOException {
boolean quit = false;
switch (command) {
@@ -469,31 +471,31 @@ public class FunctionalTester {
print_error_message("Unknown command on line", state);
break;
case cmd_Let:
- handle_let(line, state, parameters);
+ handle_let(line, state, parameters, objectPool);
break;
case cmd_Eval:
- handle_eval(line, state, parameters);
+ handle_eval(line, state, parameters, objectPool);
break;
case cmd_Print:
- handle_print(line, state, parameters);
+ handle_print(line, state, parameters, objectPool);
break;
case cmd_Same:
- handle_same(line, state, parameters);
+ handle_same(line, state, parameters, objectPool);
break;
case cmd_Sameq:
- handle_sameq(line, state, parameters);
+ handle_sameq(line, state, parameters, objectPool);
break;
case cmd_Diff:
- handle_diff(line, state, parameters);
+ handle_diff(line, state, parameters, objectPool);
break;
case cmd_Diffq:
- handle_diffq(line, state, parameters);
+ handle_diffq(line, state, parameters, objectPool);
break;
case cmd_Set:
- handle_set(line, state, parameters);
+ handle_set(line, state, parameters, objectPool);
break;
case cmd_Show:
- handle_show(line, state, parameters);
+ handle_show(line, state, parameters, objectPool);
break;
case cmd_Writexml:
// handle_writexml(line, state, parameters);
@@ -520,18 +522,19 @@ public class FunctionalTester {
*
* @throws IOException
*********************************************************************/
- public static void handle_let(AMutableString line, State state, Parameters parameters) throws IOException {
+ public static void handle_let(AMutableString line, State state, Parameters parameters, ClassAdObjectPool objectPool)
+ throws IOException {
AMutableString variable_name = new AMutableString(null);
ExprTree tree;
Variable variable;
if (get_variable_name(line, true, variable_name, state, parameters)) {
- tree = get_expr(line, state, parameters);
+ tree = get_expr(line, state, parameters, objectPool);
if (tree != null) {
- variable = new Variable(variable_name.getStringValue(), tree);
+ variable = new Variable(variable_name.getStringValue(), tree, objectPool);
variables.put(variable_name.getStringValue(), variable);
if (parameters.interactive) {
- print_expr(tree, state, parameters);
+ print_expr(tree, state, parameters, objectPool);
}
}
}
@@ -544,16 +547,17 @@ public class FunctionalTester {
*
* @throws IOException
*********************************************************************/
- public static void handle_eval(AMutableString line, State state, Parameters parameters) throws IOException {
+ public static void handle_eval(AMutableString line, State state, Parameters parameters,
+ ClassAdObjectPool objectPool) throws IOException {
AMutableString variable_name = new AMutableString("");
ExprTree tree;
Variable variable;
if (get_variable_name(line, true, variable_name, state, parameters)) {
- tree = get_expr(line, state, parameters);
+ tree = get_expr(line, state, parameters, objectPool);
if (tree != null) {
- Value value = new Value();
- if (!evaluate_expr(tree, value, parameters)) {
+ Value value = new Value(objectPool);
+ if (!evaluate_expr(tree, value, parameters, objectPool)) {
print_error_message("Couldn't evaluate rvalue", state);
} else {
variable = new Variable(variable_name.getStringValue(), value);
@@ -570,11 +574,12 @@ public class FunctionalTester {
*
* @throws IOException
*********************************************************************/
- public static void handle_print(AMutableString line, State state, Parameters parameters) throws IOException {
+ public static void handle_print(AMutableString line, State state, Parameters parameters,
+ ClassAdObjectPool objectPool) throws IOException {
ExprTree tree;
- tree = get_expr(line, state, parameters);
+ tree = get_expr(line, state, parameters, objectPool);
if (tree != null) {
- print_expr(tree, state, parameters);
+ print_expr(tree, state, parameters, objectPool);
}
}
@@ -584,25 +589,26 @@ public class FunctionalTester {
*
* @throws IOException
*********************************************************************/
- public static void handle_same(AMutableString line, State state, Parameters parameters) throws IOException {
- ExprTreeHolder tree = new ExprTreeHolder();
- ExprTreeHolder tree2 = new ExprTreeHolder();
- Value value1 = new Value();
- Value value2 = new Value();
+ public static void handle_same(AMutableString line, State state, Parameters parameters,
+ ClassAdObjectPool objectPool) throws IOException {
+ ExprTreeHolder tree = new ExprTreeHolder(objectPool);
+ ExprTreeHolder tree2 = new ExprTreeHolder(objectPool);
+ Value value1 = new Value(objectPool);
+ Value value2 = new Value(objectPool);
try {
- get_two_exprs(line, tree, tree2, state, parameters);
+ get_two_exprs(line, tree, tree2, state, parameters, objectPool);
if (tree.getInnerTree() != null || tree2.getInnerTree() != null) {
if (parameters.debug) {
System.out.println("Sameeval has two trees:");
System.out.print(" ");
- print_expr(tree, state, parameters);
+ print_expr(tree, state, parameters, objectPool);
System.out.print(" ");
- print_expr(tree2, state, parameters);
+ print_expr(tree2, state, parameters, objectPool);
}
- if (!evaluate_expr(tree, value1, parameters)) {
+ if (!evaluate_expr(tree, value1, parameters, objectPool)) {
print_error_message("Couldn't evaluate first expression.\n", state);
- } else if (!evaluate_expr(tree2, value2, parameters)) {
+ } else if (!evaluate_expr(tree2, value2, parameters, objectPool)) {
print_error_message("Couldn't evaluate second expressions.\n", state);
} else if (!value1.sameAs(value2)) {
print_error_message("the expressions are different.", state);
@@ -624,14 +630,16 @@ public class FunctionalTester {
/*********************************************************************
* Function: handle_sameq
* Purpose:
+ * @param objectPool
*
* @throws IOException
*********************************************************************/
- public static void handle_sameq(AMutableString line, State state, Parameters parameters) throws IOException {
- ExprTreeHolder tree = new ExprTreeHolder();
- ExprTreeHolder tree2 = new ExprTreeHolder();
+ public static void handle_sameq(AMutableString line, State state, Parameters parameters,
+ ClassAdObjectPool objectPool) throws IOException {
+ ExprTreeHolder tree = new ExprTreeHolder(objectPool);
+ ExprTreeHolder tree2 = new ExprTreeHolder(objectPool);
- get_two_exprs(line, tree, tree2, state, parameters);
+ get_two_exprs(line, tree, tree2, state, parameters, objectPool);
if (tree.getInnerTree() != null || tree2.getInnerTree() != null) {
if (!tree.sameAs(tree2)) {
print_error_message("the expressions are different.", state);
@@ -646,17 +654,18 @@ public class FunctionalTester {
*
* @throws IOException
*********************************************************************/
- public static void handle_diff(AMutableString line, State state, Parameters parameters) throws IOException {
- ExprTreeHolder tree = new ExprTreeHolder();
- ExprTreeHolder tree2 = new ExprTreeHolder();
- Value value1 = new Value();
- Value value2 = new Value();
-
- get_two_exprs(line, tree, tree2, state, parameters);
+ public static void handle_diff(AMutableString line, State state, Parameters parameters,
+ ClassAdObjectPool objectPool) throws IOException {
+ ExprTreeHolder tree = new ExprTreeHolder(objectPool);
+ ExprTreeHolder tree2 = new ExprTreeHolder(objectPool);
+ Value value1 = new Value(objectPool);
+ Value value2 = new Value(objectPool);
+
+ get_two_exprs(line, tree, tree2, state, parameters, objectPool);
if (tree.getInnerTree() != null || tree2.getInnerTree() != null) {
- if (!evaluate_expr(tree, value1, parameters)) {
+ if (!evaluate_expr(tree, value1, parameters, objectPool)) {
print_error_message("Couldn't evaluate first expression.\n", state);
- } else if (!evaluate_expr(tree2, value2, parameters)) {
+ } else if (!evaluate_expr(tree2, value2, parameters, objectPool)) {
print_error_message("Couldn't evaluate second expressions.\n", state);
} else if (value1.sameAs(value2)) {
print_error_message("the expressions are the same.", state);
@@ -669,14 +678,16 @@ public class FunctionalTester {
/*********************************************************************
* Function: handle_diffq
* Purpose:
+ * @param objectPool
*
* @throws IOException
*********************************************************************/
- public static void handle_diffq(AMutableString line, State state, Parameters parameters) throws IOException {
- ExprTreeHolder tree = new ExprTreeHolder();
- ExprTreeHolder tree2 = new ExprTreeHolder();
+ public static void handle_diffq(AMutableString line, State state, Parameters parameters,
+ ClassAdObjectPool objectPool) throws IOException {
+ ExprTreeHolder tree = new ExprTreeHolder(objectPool);
+ ExprTreeHolder tree2 = new ExprTreeHolder(objectPool);
- get_two_exprs(line, tree, tree2, state, parameters);
+ get_two_exprs(line, tree, tree2, state, parameters, objectPool);
if (tree.getInnerTree() != null || tree2.getInnerTree() != null) {
if (tree.sameAs(tree2)) {
print_error_message("the expressions are the same.", state);
@@ -688,8 +699,10 @@ public class FunctionalTester {
/*********************************************************************
* Function: handle_set
* Purpose:
+ * @param objectPool
*********************************************************************/
- public static void handle_set(AMutableString line, State state, Parameters parameters) {
+ public static void handle_set(AMutableString line, State state, Parameters parameters,
+ ClassAdObjectPool objectPool) {
AMutableString option_name = new AMutableString(null);
AMutableString option_value = new AMutableString(null);
@@ -718,8 +731,10 @@ public class FunctionalTester {
/*********************************************************************
* Function: handle_show
* Purpose:
+ * @param objectPool
*********************************************************************/
- public static void handle_show(AMutableString line, State state, Parameters parameters) {
+ public static void handle_show(AMutableString line, State state, Parameters parameters,
+ ClassAdObjectPool objectPool) {
AMutableString option_name = new AMutableString(null);
if (get_variable_name(line, false, option_name, state, parameters)) {
@@ -906,10 +921,11 @@ public class FunctionalTester {
*
* @throws IOException
*********************************************************************/
- public static ExprTree get_expr(AMutableString line, State state, Parameters parameters) throws IOException {
+ public static ExprTree get_expr(AMutableString line, State state, Parameters parameters,
+ ClassAdObjectPool objectPool) throws IOException {
int offset;
ExprTree tree;
- ClassAdParser parser = new ClassAdParser(null, false, true, false, null, null, null);
+ ClassAdParser parser = new ClassAdParser(objectPool);
StringLexerSource lexer_source = new StringLexerSource(line.getStringValue());
tree = parser.parseExpression(lexer_source, false);
@@ -930,9 +946,9 @@ public class FunctionalTester {
* @throws IOException
*********************************************************************/
public static void get_two_exprs(AMutableString line, ExprTreeHolder tree1, ExprTreeHolder tree2, State state,
- Parameters parameters) throws IOException {
+ Parameters parameters, ClassAdObjectPool objectPool) throws IOException {
int offset;
- ClassAdParser parser = new ClassAdParser(null, false, true, false, null, null, null);
+ ClassAdParser parser = new ClassAdParser(objectPool);
StringLexerSource lexer_source = new StringLexerSource(line.getStringValue());
tree1.setInnerTree(parser.parseExpression(lexer_source, false));
@@ -943,7 +959,7 @@ public class FunctionalTester {
} else {
if (parameters.debug) {
System.out.print("# Tree1: ");
- print_expr(tree1, state, parameters);
+ print_expr(tree1, state, parameters, objectPool);
}
if (parser.peekToken() != TokenType.LEX_COMMA) {
@@ -961,9 +977,9 @@ public class FunctionalTester {
throw new IOException();
} else if (parameters.debug) {
System.out.print("# Tree2: ");
- print_expr(tree2, state, parameters);
+ print_expr(tree2, state, parameters, objectPool);
System.out.print("# Tree1: ");
- print_expr(tree1, state, parameters);
+ print_expr(tree1, state, parameters, objectPool);
System.out.println();
}
}
@@ -978,14 +994,15 @@ public class FunctionalTester {
*
* @throws HyracksDataException
*********************************************************************/
- public static void print_expr(ExprTree tree, State state, Parameters parameters) throws HyracksDataException {
+ public static void print_expr(ExprTree tree, State state, Parameters parameters, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
AMutableCharArrayString output = new AMutableCharArrayString();
if (state.format == PrintFormat.print_Compact) {
- ClassAdUnParser unparser = new ClassAdUnParser();
+ ClassAdUnParser unparser = new ClassAdUnParser(objectPool);
unparser.unparse(output, tree);
} else if (state.format == PrintFormat.print_Pretty) {
- PrettyPrint unparser = new PrettyPrint();
+ PrettyPrint unparser = new PrettyPrint(objectPool);
unparser.unparse(output, tree);
} else if (state.format == PrintFormat.print_XML) {/*
* ClassAdXMLUnParser unparser = new
@@ -1009,8 +1026,9 @@ public class FunctionalTester {
*
* @throws HyracksDataException
*********************************************************************/
- public static boolean evaluate_expr(ExprTree tree, Value value, Parameters parameters) throws HyracksDataException {
- ClassAd classad = new ClassAd();
+ public static boolean evaluate_expr(ExprTree tree, Value value, Parameters parameters, ClassAdObjectPool objectPool)
+ throws HyracksDataException {
+ ClassAd classad = new ClassAd(objectPool);
boolean success = false;;
classad.insert("internal___", tree);
success = classad.evaluateAttr("internal___", value);
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/test/java/org/apache/asterix/external/library/ClassAdParser.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/library/ClassAdParser.java b/asterix-external-data/src/test/java/org/apache/asterix/external/library/ClassAdParser.java
index d6db6c2..5fd6f21 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/library/ClassAdParser.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/library/ClassAdParser.java
@@ -59,15 +59,7 @@ import org.apache.asterix.external.classad.StringLexerSource;
import org.apache.asterix.external.classad.TokenValue;
import org.apache.asterix.external.classad.Value;
import org.apache.asterix.external.classad.Value.NumberFactor;
-import org.apache.asterix.external.classad.object.pool.AttributeReferencePool;
-import org.apache.asterix.external.classad.object.pool.BitSetPool;
-import org.apache.asterix.external.classad.object.pool.ClassAdPool;
-import org.apache.asterix.external.classad.object.pool.ExprHolderPool;
-import org.apache.asterix.external.classad.object.pool.ExprListPool;
-import org.apache.asterix.external.classad.object.pool.LiteralPool;
-import org.apache.asterix.external.classad.object.pool.OperationPool;
-import org.apache.asterix.external.classad.object.pool.TokenValuePool;
-import org.apache.asterix.external.classad.object.pool.ValuePool;
+import org.apache.asterix.external.classad.object.pool.ClassAdObjectPool;
import org.apache.asterix.external.parser.AbstractDataParser;
import org.apache.asterix.om.base.ABoolean;
import org.apache.asterix.om.base.AMutableInt32;
@@ -94,16 +86,7 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars
private Lexer lexer = new Lexer();
private LexerSource currentSource = null;
private boolean isExpr = false;
- // object pools
- private final ExprHolderPool mutableExprPool = new ExprHolderPool();
- private final TokenValuePool tokenValuePool = new TokenValuePool();
- private final ClassAdPool classAdPool = new ClassAdPool();
- private final ExprListPool exprListPool = new ExprListPool();
- private final ValuePool valuePool = new ValuePool();
- private final LiteralPool literalPool = new LiteralPool();
- private final BitSetPool bitSetPool = new BitSetPool();
- private final OperationPool operationPool = new OperationPool();
- private final AttributeReferencePool attrRefPool = new AttributeReferencePool();
+ private final ClassAdObjectPool objectPool;
// asterix objects
private ARecordType recordType;
private IObjectPool<IARecordBuilder, ATypeTag> recordBuilderPool = new ListObjectPool<IARecordBuilder, ATypeTag>(
@@ -112,17 +95,22 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars
new ListBuilderFactory());
private IObjectPool<IMutableValueStorage, ATypeTag> abvsBuilderPool = new ListObjectPool<IMutableValueStorage, ATypeTag>(
new AbvsBuilderFactory());
- private ClassAd rootAd = new ClassAd(false, true);
+ private final ClassAd rootAd;
private String exprPrefix = "expr=";
private String exprSuffix = "";
private boolean evaluateExpr = true;
private String exprFieldNameSuffix = "Expr";
private boolean keepBoth = true;
- private boolean oldFormat = false;
+ private boolean oldFormat = true;
private StringLexerSource stringLexerSource = new StringLexerSource("");
public ClassAdParser(ARecordType recordType, boolean oldFormat, boolean evaluateExpr, boolean keepBoth,
- String exprPrefix, String exprSuffix, String exprFieldNameSuffix) {
+ String exprPrefix, String exprSuffix, String exprFieldNameSuffix, ClassAdObjectPool objectPool) {
+ if (objectPool == null) {
+ System.out.println();
+ }
+ this.objectPool = objectPool;
+ this.rootAd = new ClassAd(objectPool);
this.recordType = recordType;
this.currentSource = new CharArrayLexerSource();
this.recordType = recordType;
@@ -137,6 +125,15 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars
this.exprFieldNameSuffix = exprFieldNameSuffix;
}
+ public ClassAdParser(ClassAdObjectPool objectPool) {
+ if (objectPool == null) {
+ System.out.println();
+ }
+ this.objectPool = objectPool;
+ this.currentSource = new CharArrayLexerSource();
+ rootAd = null;
+ }
+
/***********************************
* AsterixDB Specific begin
*
@@ -160,8 +157,8 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars
return false;
}
tt = lexer.peekToken();
- TokenValue tv = tokenValuePool.get();
- ExprTreeHolder tree = mutableExprPool.get();
+ TokenValue tv = objectPool.tokenValuePool.get();
+ ExprTreeHolder tree = objectPool.mutableExprPool.get();
while (tt != TokenType.LEX_CLOSE_BOX) {
// Get the name of the expression
tv.reset();
@@ -200,8 +197,8 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars
// we will store a string representation of the expression
int len = lexer.getLexSource().getPosition() - positionBefore;
// add it as it is to the classAd
- Literal lit = literalPool.get();
- Value exprVal = valuePool.get();
+ Literal lit = objectPool.literalPool.get();
+ Value exprVal = objectPool.valuePool.get();
exprVal.setStringValue(exprPrefix
+ String.valueOf(lexer.getLexSource().getBuffer(), positionBefore, len) + exprSuffix);
Literal.createLiteral(lit, exprVal, NumberFactor.NO_FACTOR);
@@ -263,15 +260,7 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars
listBuilderPool.reset();
recordBuilderPool.reset();
abvsBuilderPool.reset();
- mutableExprPool.reset();
- tokenValuePool.reset();
- classAdPool.reset();
- exprListPool.reset();
- valuePool.reset();
- literalPool.reset();
- bitSetPool.reset();
- operationPool.reset();
- attrRefPool.reset();
+ objectPool.reset();
}
private ATypeTag getTargetTypeTag(ATypeTag expectedTypeTag, IAType aObjectType) throws IOException {
@@ -384,7 +373,7 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars
case EXPR_LIST_NODE:
case FN_CALL_NODE:
case OP_NODE:
- val = valuePool.get();
+ val = objectPool.valuePool.get();
if (pAd.evaluateAttr(name, val)) {
} else {
@@ -393,7 +382,7 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars
}
break;
case LITERAL_NODE:
- val = ((Literal) tree).getValue();
+ val = ((Literal) tree.getTree()).getValue();
break;
default:
throw new HyracksDataException("Unknown Expression type detected: " + tree.getKind());
@@ -574,7 +563,7 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars
}
private BitSet getBitSet() {
- return bitSetPool.get();
+ return objectPool.bitSetPool.get();
}
public static int checkNullConstraints(ARecordType recType, BitSet nulls) {
@@ -710,7 +699,7 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars
public ClassAd parseClassAd(LexerSource lexer_source, boolean full) throws IOException {
System.out.println("Don't use this call. instead, pass a mutable classad instance");
- ClassAd ad = classAdPool.get();
+ ClassAd ad = objectPool.classAdPool.get();
if (lexer.initialize(lexer_source)) {
if (!parseClassAd(ad, full)) {
return null;
@@ -804,7 +793,7 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars
*/
public ExprTree parseExpression(String buffer, boolean full) throws IOException {
stringLexerSource.setNewSource(buffer);
- ExprTreeHolder mutableExpr = mutableExprPool.get();
+ ExprTreeHolder mutableExpr = objectPool.mutableExprPool.get();
if (lexer.initialize(stringLexerSource)) {
parseExpression(mutableExpr, full);
}
@@ -816,7 +805,7 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars
}
public ExprTree parseExpression(LexerSource lexer_source, boolean full) throws IOException {
- ExprTreeHolder mutableExpr = mutableExprPool.get();
+ ExprTreeHolder mutableExpr = objectPool.mutableExprPool.get();
if (lexer.initialize(lexer_source)) {
parseExpression(mutableExpr, full);
}
@@ -827,7 +816,7 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars
if (!lexer.wasInitialized()) {
return null;
} else {
- ExprTreeHolder expr = mutableExprPool.get();
+ ExprTreeHolder expr = objectPool.mutableExprPool.get();
parseExpression(expr, false);
ExprTree innerTree = expr.getInnerTree();
return innerTree;
@@ -849,20 +838,21 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars
private boolean parseExpression(ExprTreeHolder tree, boolean full) throws IOException {
TokenType tt;
- if (!parseLogicalORExpression(tree))
+ if (!parseLogicalORExpression(tree)) {
return false;
+ }
if ((tt = lexer.peekToken()) == TokenType.LEX_QMARK) {
lexer.consumeToken();
ExprTreeHolder treeL = tree;
- ExprTreeHolder treeM = mutableExprPool.get();
- ExprTreeHolder treeR = mutableExprPool.get();
+ ExprTreeHolder treeM = objectPool.mutableExprPool.get();
+ ExprTreeHolder treeR = objectPool.mutableExprPool.get();
parseExpression(treeM);
if ((tt = lexer.consumeToken()) != TokenType.LEX_COLON) {
throw new HyracksDataException("expected LEX_COLON, but got " + Lexer.strLexToken(tt));
}
parseExpression(treeR);
if (treeL.getInnerTree() != null && treeM.getInnerTree() != null && treeR.getInnerTree() != null) {
- Operation newTree = operationPool.get();
+ Operation newTree = objectPool.operationPool.get();
Operation.createOperation(Operation.OpKind_TERNARY_OP, treeL, treeM, treeR, newTree);
tree.setInnerTree(newTree);
return (true);
@@ -882,15 +872,16 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars
// | LogicalORExpression '||' LogicalANDExpression
private boolean parseLogicalORExpression(ExprTreeHolder tree) throws IOException {
- if (!parseLogicalANDExpression(tree))
+ if (!parseLogicalANDExpression(tree)) {
return false;
+ }
while ((lexer.peekToken()) == TokenType.LEX_LOGICAL_OR) {
ExprTreeHolder treeL = tree;
- ExprTreeHolder treeR = mutableExprPool.get();
+ ExprTreeHolder treeR = objectPool.mutableExprPool.get();
lexer.consumeToken();
parseLogicalANDExpression(treeR);
if (treeL.getInnerTree() != null && treeR.getInnerTree() != null) {
- Operation newTree = operationPool.get();
+ Operation newTree = objectPool.operationPool.get();
Operation.createOperation(Operation.OpKind_LOGICAL_OR_OP, treeL, treeR, null, newTree);
tree.setInnerTree(newTree);
} else {
@@ -904,15 +895,16 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars
// LogicalANDExpression .= InclusiveORExpression
// | LogicalANDExpression '&&' InclusiveORExpression
private boolean parseLogicalANDExpression(ExprTreeHolder tree) throws IOException {
- if (!parseInclusiveORExpression(tree))
+ if (!parseInclusiveORExpression(tree)) {
return false;
+ }
while ((lexer.peekToken()) == TokenType.LEX_LOGICAL_AND) {
ExprTreeHolder treeL = tree;
- ExprTreeHolder treeR = mutableExprPool.get();
+ ExprTreeHolder treeR = objectPool.mutableExprPool.get();
lexer.consumeToken();
parseInclusiveORExpression(treeR);
if (treeL.getInnerTree() != null && treeR.getInnerTree() != null) {
- Operation newTree = operationPool.get();
+ Operation newTree = objectPool.operationPool.get();
Operation.createOperation(Operation.OpKind_LOGICAL_AND_OP, treeL, treeR, null, newTree);
tree.setInnerTree(newTree);
} else {
@@ -926,15 +918,16 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars
// InclusiveORExpression .= ExclusiveORExpression
// | InclusiveORExpression '|' ExclusiveORExpression
public boolean parseInclusiveORExpression(ExprTreeHolder tree) throws IOException {
- if (!parseExclusiveORExpression(tree))
+ if (!parseExclusiveORExpression(tree)) {
return false;
+ }
while ((lexer.peekToken()) == TokenType.LEX_BITWISE_OR) {
ExprTreeHolder treeL = tree;
- ExprTreeHolder treeR = mutableExprPool.get();
+ ExprTreeHolder treeR = objectPool.mutableExprPool.get();
lexer.consumeToken();
parseExclusiveORExpression(treeR);
if (treeL.getInnerTree() != null && treeR.getInnerTree() != null) {
- Operation newTree = operationPool.get();
+ Operation newTree = objectPool.operationPool.get();
Operation.createOperation(Operation.OpKind_BITWISE_OR_OP, treeL, treeR, null, newTree);
tree.setInnerTree(newTree);
} else {
@@ -948,15 +941,16 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars
// ExclusiveORExpression .= ANDExpression
// | ExclusiveORExpression '^' ANDExpression
private boolean parseExclusiveORExpression(ExprTreeHolder tree) throws IOException {
- if (!parseANDExpression(tree))
+ if (!parseANDExpression(tree)) {
return false;
+ }
while ((lexer.peekToken()) == TokenType.LEX_BITWISE_XOR) {
lexer.consumeToken();
ExprTreeHolder treeL = tree;
- ExprTreeHolder treeR = mutableExprPool.get();
+ ExprTreeHolder treeR = objectPool.mutableExprPool.get();
parseANDExpression(treeR);
if (treeL.getInnerTree() != null && treeR.getInnerTree() != null) {
- Operation newTree = operationPool.get();
+ Operation newTree = objectPool.operationPool.get();
Operation.createOperation(Operation.OpKind_BITWISE_XOR_OP, treeL, treeR, null, newTree);
tree.setInnerTree(newTree);
} else {
@@ -970,15 +964,16 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars
// ANDExpression .= EqualityExpression
// | ANDExpression '&' EqualityExpression
private boolean parseANDExpression(ExprTreeHolder tree) throws IOException {
- if (!parseEqualityExpression(tree))
+ if (!parseEqualityExpression(tree)) {
return false;
+ }
while ((lexer.peekToken()) == TokenType.LEX_BITWISE_AND) {
ExprTreeHolder treeL = tree;
- ExprTreeHolder treeR = mutableExprPool.get();
+ ExprTreeHolder treeR = objectPool.mutableExprPool.get();
lexer.consumeToken();
parseEqualityExpression(treeR);
if (treeL.getInnerTree() != null && treeR.getInnerTree() != null) {
- Operation newTree = operationPool.get();
+ Operation newTree = objectPool.operationPool.get();
Operation.createOperation(Operation.OpKind_BITWISE_AND_OP, treeL, treeR, null, newTree);
tree.setInnerTree(newTree);
} else {
@@ -997,13 +992,14 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars
private boolean parseEqualityExpression(ExprTreeHolder tree) throws IOException {
TokenType tt;
int op = Operation.OpKind_NO_OP;
- if (!parseRelationalExpression(tree))
+ if (!parseRelationalExpression(tree)) {
return false;
+ }
tt = lexer.peekToken();
while (tt == TokenType.LEX_EQUAL || tt == TokenType.LEX_NOT_EQUAL || tt == TokenType.LEX_META_EQUAL
|| tt == TokenType.LEX_META_NOT_EQUAL) {
ExprTreeHolder treeL = tree;
- ExprTreeHolder treeR = mutableExprPool.get();
+ ExprTreeHolder treeR = objectPool.mutableExprPool.get();
lexer.consumeToken();
parseRelationalExpression(treeR);
switch (tt) {
@@ -1023,7 +1019,7 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars
throw new HyracksDataException("ClassAd: Should not reach here");
}
if (treeL.getInnerTree() != null && treeR.getInnerTree() != null) {
- Operation newTree = operationPool.get();
+ Operation newTree = objectPool.operationPool.get();
Operation.createOperation(op, treeL, treeR, null, newTree);
tree.setInnerTree(newTree);
} else {
@@ -1042,14 +1038,15 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars
// | RelationalExpression '>=' ShiftExpression
private boolean parseRelationalExpression(ExprTreeHolder tree) throws IOException {
TokenType tt;
- if (!parseShiftExpression(tree))
+ if (!parseShiftExpression(tree)) {
return false;
+ }
tt = lexer.peekToken();
while (tt == TokenType.LEX_LESS_THAN || tt == TokenType.LEX_GREATER_THAN || tt == TokenType.LEX_LESS_OR_EQUAL
|| tt == TokenType.LEX_GREATER_OR_EQUAL) {
int op = Operation.OpKind_NO_OP;
ExprTreeHolder treeL = tree;
- ExprTreeHolder treeR = mutableExprPool.get();
+ ExprTreeHolder treeR = objectPool.mutableExprPool.get();
lexer.consumeToken();
parseShiftExpression(treeR);
switch (tt) {
@@ -1069,7 +1066,7 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars
throw new HyracksDataException("ClassAd: Should not reach here");
}
if (treeL.getInnerTree() != null && treeR.getInnerTree() != null) {
- Operation newTree = operationPool.get();
+ Operation newTree = objectPool.operationPool.get();
Operation.createOperation(op, treeL, treeR, null, newTree);
tree.setInnerTree(newTree);
} else {
@@ -1086,13 +1083,14 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars
// | ShiftExpression '>>' AdditiveExpression
// | ShiftExpression '>>>' AditiveExpression
private boolean parseShiftExpression(ExprTreeHolder tree) throws IOException {
- if (!parseAdditiveExpression(tree))
+ if (!parseAdditiveExpression(tree)) {
return false;
+ }
TokenType tt = lexer.peekToken();
while (tt == TokenType.LEX_LEFT_SHIFT || tt == TokenType.LEX_RIGHT_SHIFT || tt == TokenType.LEX_URIGHT_SHIFT) {
ExprTreeHolder treeL = tree;
- ExprTreeHolder treeR = mutableExprPool.get();
+ ExprTreeHolder treeR = objectPool.mutableExprPool.get();
int op;
lexer.consumeToken();
parseAdditiveExpression(treeR);
@@ -1112,7 +1110,7 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars
}
if (treeL.getInnerTree() != null && treeR.getInnerTree() != null) {
- Operation newTree = operationPool.get();
+ Operation newTree = objectPool.operationPool.get();
Operation.createOperation(op, treeL, treeR, null, newTree);
tree.setInnerTree(newTree);
} else {
@@ -1128,17 +1126,18 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars
// | AdditiveExpression '+' MultiplicativeExpression
// | AdditiveExpression '-' MultiplicativeExpression
private boolean parseAdditiveExpression(ExprTreeHolder tree) throws IOException {
- if (!parseMultiplicativeExpression(tree))
+ if (!parseMultiplicativeExpression(tree)) {
return false;
+ }
TokenType tt = lexer.peekToken();
while (tt == TokenType.LEX_PLUS || tt == TokenType.LEX_MINUS) {
ExprTreeHolder treeL = tree;
- ExprTreeHolder treeR = mutableExprPool.get();
+ ExprTreeHolder treeR = objectPool.mutableExprPool.get();
lexer.consumeToken();
parseMultiplicativeExpression(treeR);
if (treeL.getInnerTree() != null && treeR.getInnerTree() != null) {
- Operation newTree = operationPool.get();
+ Operation newTree = objectPool.operationPool.get();
Operation.createOperation(
(tt == TokenType.LEX_PLUS) ? Operation.OpKind_ADDITION_OP : Operation.OpKind_SUBTRACTION_OP,
treeL, treeR, null, newTree);
@@ -1157,13 +1156,14 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars
// | MultiplicativeExpression '/' UnaryExpression
// | MultiplicativeExpression '%' UnaryExpression
private boolean parseMultiplicativeExpression(ExprTreeHolder tree) throws IOException {
- if (!parseUnaryExpression(tree))
+ if (!parseUnaryExpression(tree)) {
return false;
+ }
TokenType tt = lexer.peekToken();
while (tt == TokenType.LEX_MULTIPLY || tt == TokenType.LEX_DIVIDE || tt == TokenType.LEX_MODULUS) {
ExprTreeHolder treeL = tree;
- ExprTreeHolder treeR = mutableExprPool.get();
+ ExprTreeHolder treeR = objectPool.mutableExprPool.get();
int op;
lexer.consumeToken();
parseUnaryExpression(treeR);
@@ -1182,7 +1182,7 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars
throw new HyracksDataException("ClassAd: Should not reach here");
}
if (treeL.getInnerTree() != null && treeR.getInnerTree() != null) {
- Operation newTree = operationPool.get();
+ Operation newTree = objectPool.operationPool.get();
Operation.createOperation(op, treeL, treeR, null, newTree);
tree.setInnerTree(newTree);
} else {
@@ -1202,7 +1202,7 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars
if (tt == TokenType.LEX_MINUS || tt == TokenType.LEX_PLUS || tt == TokenType.LEX_BITWISE_NOT
|| tt == TokenType.LEX_LOGICAL_NOT) {
lexer.consumeToken();
- ExprTreeHolder treeM = mutableExprPool.get();
+ ExprTreeHolder treeM = objectPool.mutableExprPool.get();
int op = Operation.OpKind_NO_OP;
parseUnaryExpression(treeM);
switch (tt) {
@@ -1222,7 +1222,7 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars
throw new HyracksDataException("ClassAd: Shouldn't Get here");
}
if (treeM.getInnerTree() != null) {
- Operation newTree = operationPool.get();
+ Operation newTree = objectPool.operationPool.get();
Operation.createOperation(op, treeM, null, null, newTree);
tree.setInnerTree(newTree);
} else {
@@ -1240,18 +1240,19 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars
// | PostfixExpression '[' Expression ']'
private boolean parsePostfixExpression(ExprTreeHolder tree) throws IOException {
TokenType tt;
- if (!parsePrimaryExpression(tree))
+ if (!parsePrimaryExpression(tree)) {
return false;
+ }
while ((tt = lexer.peekToken()) == TokenType.LEX_OPEN_BOX || tt == TokenType.LEX_SELECTION) {
ExprTreeHolder treeL = tree;
- ExprTreeHolder treeR = mutableExprPool.get();
- TokenValue tv = tokenValuePool.get();
+ ExprTreeHolder treeR = objectPool.mutableExprPool.get();
+ TokenValue tv = objectPool.tokenValuePool.get();
lexer.consumeToken();
if (tt == TokenType.LEX_OPEN_BOX) {
// subscript operation
parseExpression(treeR);
if (treeL.getInnerTree() != null && treeR.getInnerTree() != null) {
- Operation newTree = operationPool.get();
+ Operation newTree = objectPool.operationPool.get();
Operation.createOperation(Operation.OpKind_SUBSCRIPT_OP, treeL, treeR, null, newTree);
if (lexer.consumeToken() == TokenType.LEX_CLOSE_BOX) {
tree.setInnerTree(newTree);
@@ -1266,7 +1267,7 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars
throw new HyracksDataException("second argument of selector must be an " + "identifier (got"
+ String.valueOf(Lexer.strLexToken(tt)) + ")");
}
- AttributeReference newTree = attrRefPool.get();
+ AttributeReference newTree = objectPool.attrRefPool.get();
AttributeReference.createAttributeReference(treeL, tv.getStrValue(), false, newTree);
tree.setInnerTree(newTree);
}
@@ -1285,7 +1286,7 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars
// ( ArgumentList non-terminal includes parentheses )
private boolean parsePrimaryExpression(ExprTreeHolder tree) throws IOException {
ExprTreeHolder treeL;
- TokenValue tv = tokenValuePool.get();
+ TokenValue tv = objectPool.tokenValuePool.get();
TokenType tt;
tree.setInnerTree(null);
switch ((tt = lexer.peekToken(tv))) {
@@ -1295,22 +1296,24 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars
lexer.consumeToken();
// check for funcion call
if ((tt = lexer.peekToken()) == TokenType.LEX_OPEN_PAREN) {
- ExprList argList = exprListPool.get();
+ ExprList argList = objectPool.exprListPool.get();
if (!parseArgumentList(argList)) {
tree.setInnerTree(null);
return false;
- };
+ } ;
// special case function-calls should be converted
// into a literal expression if the argument is a
// string literal
if (shouldEvaluateAtParseTime(tv.getStrValue().toString(), argList)) {
tree.setInnerTree(evaluateFunction(tv.getStrValue().toString(), argList));
} else {
- tree.setInnerTree(FunctionCall.createFunctionCall(tv.getStrValue().toString(), argList));
+ tree.setInnerTree(
+ FunctionCall.createFunctionCall(tv.getStrValue().toString(), argList, objectPool));
}
} else {
// I don't think this is ever hit
- tree.setInnerTree(AttributeReference.createAttributeReference(null, tv.getStrValue(), false));
+ tree.setInnerTree(
+ AttributeReference.createAttributeReference(null, tv.getStrValue(), false, objectPool));
}
return (tree.getInnerTree() != null);
case LEX_SELECTION:
@@ -1318,7 +1321,8 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars
lexer.consumeToken();
if ((tt = lexer.consumeToken(tv)) == TokenType.LEX_IDENTIFIER) {
// the boolean final arg signifies that reference is absolute
- tree.setInnerTree(AttributeReference.createAttributeReference(null, tv.getStrValue(), true));
+ tree.setInnerTree(
+ AttributeReference.createAttributeReference(null, tv.getStrValue(), true, objectPool));
return (tree.size() != 0);
}
// not an identifier following the '.'
@@ -1328,7 +1332,7 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars
case LEX_OPEN_PAREN: {
isExpr = true;
lexer.consumeToken();
- treeL = mutableExprPool.get();
+ treeL = objectPool.mutableExprPool.get();
parseExpression(treeL);
if (treeL.getInnerTree() == null) {
tree.resetExprTree(null);
@@ -1341,13 +1345,13 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars
// return false;
}
// assume make operation will return a new tree
- tree.setInnerTree(Operation.createOperation(Operation.OpKind_PARENTHESES_OP, treeL));
+ tree.setInnerTree(Operation.createOperation(Operation.OpKind_PARENTHESES_OP, treeL, objectPool));
return (tree.size() != 0);
}
- // constants
+ // constants
case LEX_OPEN_BOX: {
isExpr = true;
- ClassAd newAd = classAdPool.get();
+ ClassAd newAd = objectPool.classAdPool.get();
if (!parseClassAd(newAd)) {
tree.resetExprTree(null);
return false;
@@ -1358,7 +1362,7 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars
case LEX_OPEN_BRACE: {
isExpr = true;
- ExprList newList = exprListPool.get();
+ ExprList newList = objectPool.exprListPool.get();
if (!parseExprList(newList)) {
tree.setInnerTree(null);
return false;
@@ -1368,66 +1372,66 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars
return true;
case LEX_UNDEFINED_VALUE: {
- Value val = valuePool.get();
+ Value val = objectPool.valuePool.get();
lexer.consumeToken();
val.setUndefinedValue();
- tree.setInnerTree(Literal.createLiteral(val));
+ tree.setInnerTree(Literal.createLiteral(val, objectPool));
return (tree.getInnerTree() != null);
}
case LEX_ERROR_VALUE: {
- Value val = valuePool.get();
+ Value val = objectPool.valuePool.get();
lexer.consumeToken();
val.setErrorValue();
- tree.setInnerTree(Literal.createLiteral(val));
+ tree.setInnerTree(Literal.createLiteral(val, objectPool));
return (tree.getInnerTree() != null);
}
case LEX_BOOLEAN_VALUE: {
- Value val = valuePool.get();
+ Value val = objectPool.valuePool.get();
MutableBoolean b = new MutableBoolean();
tv.getBoolValue(b);
lexer.consumeToken();
val.setBooleanValue(b);
- tree.setInnerTree(Literal.createLiteral(val));
+ tree.setInnerTree(Literal.createLiteral(val, objectPool));
return (tree.getInnerTree() != null);
}
case LEX_INTEGER_VALUE: {
- Value val = valuePool.get();
+ Value val = objectPool.valuePool.get();
lexer.consumeToken();
val.setIntegerValue(tv.getIntValue());
- tree.setInnerTree(Literal.createLiteral(val, tv.getFactor()));
+ tree.setInnerTree(Literal.createLiteral(val, tv.getFactor(), objectPool));
return (tree.getInnerTree() != null);
}
case LEX_REAL_VALUE: {
- Value val = valuePool.get();
+ Value val = objectPool.valuePool.get();
lexer.consumeToken();
val.setRealValue(tv.getRealValue());
- tree.setInnerTree(Literal.createLiteral(val, tv.getFactor()));
+ tree.setInnerTree(Literal.createLiteral(val, tv.getFactor(), objectPool));
return (tree.getInnerTree() != null);
}
case LEX_STRING_VALUE: {
- Value val = valuePool.get();
+ Value val = objectPool.valuePool.get();
lexer.consumeToken();
val.setStringValue(tv.getStrValue());
- tree.setInnerTree(Literal.createLiteral(val));
+ tree.setInnerTree(Literal.createLiteral(val, objectPool));
return (tree.getInnerTree() != null);
}
case LEX_ABSOLUTE_TIME_VALUE: {
- Value val = valuePool.get();
+ Value val = objectPool.valuePool.get();
lexer.consumeToken();
val.setAbsoluteTimeValue(tv.getTimeValue());
- tree.setInnerTree(Literal.createLiteral(val));
+ tree.setInnerTree(Literal.createLiteral(val, objectPool));
return (tree.getInnerTree() != null);
}
case LEX_RELATIVE_TIME_VALUE: {
- Value val = valuePool.get();
+ Value val = objectPool.valuePool.get();
lexer.consumeToken();
val.setRelativeTimeValue(tv.getTimeValue().getRelativeTime());
- tree.setInnerTree(Literal.createLiteral(val));
+ tree.setInnerTree(Literal.createLiteral(val, objectPool));
return (tree.getInnerTree() != null);
}
@@ -1448,7 +1452,7 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars
// return false;
}
tt = lexer.peekToken();
- ExprTreeHolder tree = mutableExprPool.get();
+ ExprTreeHolder tree = objectPool.mutableExprPool.get();
while (tt != TokenType.LEX_CLOSE_PAREN) {
// parse the expression
tree.reset();
@@ -1462,9 +1466,9 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars
// the next token must be a ',' or a ')'
// or it can be a ';' if using old ClassAd semantics
tt = lexer.peekToken();
- if (tt == TokenType.LEX_COMMA || (tt == TokenType.LEX_SEMICOLON && false))
+ if (tt == TokenType.LEX_COMMA || (tt == TokenType.LEX_SEMICOLON && false)) {
lexer.consumeToken();
- else if (tt != TokenType.LEX_CLOSE_PAREN) {
+ } else if (tt != TokenType.LEX_CLOSE_PAREN) {
argList.clear();
throw new HyracksDataException(
"expected LEX_COMMA or LEX_CLOSE_PAREN but got " + String.valueOf(Lexer.strLexToken(tt)));
@@ -1490,11 +1494,12 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars
public boolean parseClassAd(ClassAd ad, boolean full) throws IOException {
TokenType tt;
ad.clear();
- if ((tt = lexer.consumeToken()) != TokenType.LEX_OPEN_BOX)
+ if ((tt = lexer.consumeToken()) != TokenType.LEX_OPEN_BOX) {
return false;
+ }
tt = lexer.peekToken();
- TokenValue tv = tokenValuePool.get();
- ExprTreeHolder tree = mutableExprPool.get();
+ TokenValue tv = objectPool.tokenValuePool.get();
+ ExprTreeHolder tree = objectPool.mutableExprPool.get();
while (tt != TokenType.LEX_CLOSE_BOX) {
// Get the name of the expression
tv.reset();
@@ -1567,8 +1572,8 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars
public boolean parseExprList(ExprList list, boolean full) throws IOException {
TokenType tt;
- ExprTreeHolder tree = new ExprTreeHolder();
- ExprList loe = new ExprList();
+ ExprTreeHolder tree = objectPool.mutableExprPool.get();
+ ExprList loe = objectPool.exprListPool.get();
if ((tt = lexer.consumeToken()) != TokenType.LEX_OPEN_BRACE) {
throw new HyracksDataException(
@@ -1589,16 +1594,16 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars
// the next token must be a ',' or a '}'
tt = lexer.peekToken();
- if (tt == TokenType.LEX_COMMA)
+ if (tt == TokenType.LEX_COMMA) {
lexer.consumeToken();
- else if (tt != TokenType.LEX_CLOSE_BRACE) {
+ } else if (tt != TokenType.LEX_CLOSE_BRACE) {
throw new HyracksDataException("while parsing expression list: expected "
+ "LEX_CLOSE_BRACE or LEX_COMMA but got " + Lexer.strLexToken(tt));
}
}
lexer.consumeToken();
- list.setValue(ExprList.createExprList(loe));
+ list.setValue(ExprList.createExprList(loe, objectPool));
// if a full parse was requested, ensure that input is exhausted
if (full && (lexer.consumeToken() != TokenType.LEX_END_OF_INPUT)) {
@@ -1613,8 +1618,8 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars
boolean should_eval = false;
if (functionName.equalsIgnoreCase("absTime") || functionName.equalsIgnoreCase("relTime")) {
if (argList.size() == 1 && argList.get(0).getKind() == NodeKind.LITERAL_NODE) {
- Value val = new Value();
- AMutableNumberFactor factor = new AMutableNumberFactor();
+ Value val = objectPool.valuePool.get();
+ AMutableNumberFactor factor = objectPool.numFactorPool.get();
((Literal) argList.get(0)).getComponents(val, factor);
if (val.isStringValue()) {
should_eval = true;
@@ -1625,22 +1630,22 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars
}
public ExprTree evaluateFunction(String functionName, ExprList argList) throws HyracksDataException {
- Value val = new Value();
- AMutableNumberFactor factor = new AMutableNumberFactor();
- ExprTreeHolder tree = new ExprTreeHolder();
+ Value val = objectPool.valuePool.get();
+ AMutableNumberFactor factor = objectPool.numFactorPool.get();
+ ExprTreeHolder tree = objectPool.mutableExprPool.get();
((Literal) argList.get(0)).getComponents(val, factor);
- AMutableCharArrayString string_value = new AMutableCharArrayString();
+ AMutableCharArrayString string_value = objectPool.strPool.get();
if (val.isStringValue(string_value)) {
if (functionName.equalsIgnoreCase("absTime")) {
- tree.setInnerTree(Literal.createAbsTime(string_value));
+ tree.setInnerTree(Literal.createAbsTime(string_value, objectPool));
} else if (functionName.equalsIgnoreCase("relTime")) {
- tree.setInnerTree(Literal.createRelTime(string_value));
+ tree.setInnerTree(Literal.createRelTime(string_value, objectPool));
} else {
- tree.setInnerTree(FunctionCall.createFunctionCall(functionName, argList));
+ tree.setInnerTree(FunctionCall.createFunctionCall(functionName, argList, objectPool));
}
} else {
- tree.setInnerTree(FunctionCall.createFunctionCall(functionName, argList));
+ tree.setInnerTree(FunctionCall.createFunctionCall(functionName, argList, objectPool));
}
return tree;
}
@@ -1688,17 +1693,10 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars
this.currentSource = lexerSource;
}
- public void reset() {
- resetPools();
- }
-
- public Literal getLiteral() {
- return literalPool.get();
- }
-
@Override
public void parse(IRawRecord<? extends char[]> record, DataOutput out) throws IOException {
try {
+ resetPools();
if (oldFormat) {
int maxOffset = record.size();
rootAd.clear();
@@ -1718,7 +1716,6 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars
line = readLine(buffer, aInt32, maxOffset);
}
} else {
- resetPools();
currentSource.setNewSource(record.get());
rootAd.reset();
asterixParseClassAd(rootAd);
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/test/java/org/apache/asterix/external/library/ClassAdParserFactory.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/library/ClassAdParserFactory.java b/asterix-external-data/src/test/java/org/apache/asterix/external/library/ClassAdParserFactory.java
index 5064992..17b83c2 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/library/ClassAdParserFactory.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/library/ClassAdParserFactory.java
@@ -25,6 +25,7 @@ import java.util.Map;
import org.apache.asterix.external.api.IRecordDataParser;
import org.apache.asterix.external.api.IRecordDataParserFactory;
+import org.apache.asterix.external.classad.object.pool.ClassAdObjectPool;
import org.apache.asterix.external.util.ExternalDataConstants;
import org.apache.asterix.om.types.ARecordType;
import org.apache.hyracks.api.context.IHyracksTaskContext;
@@ -42,8 +43,8 @@ public class ClassAdParserFactory implements IRecordDataParserFactory<char[]> {
private ARecordType recordType;
private Map<String, String> configuration;
private boolean oldFormat = false;
- private boolean evaluateExpr = true;
- private boolean keepBoth;
+ private boolean evaluateExpr = false;
+ private boolean keepBoth = false;
private String exprPrefix;
private String exprSuffix;
private String exprFieldNameSuffix;
@@ -77,14 +78,12 @@ public class ClassAdParserFactory implements IRecordDataParserFactory<char[]> {
}
parserConfig = configuration.get(ClassAdParserFactory.KEY_EVALUATE);
- if ((parserConfig != null) && parserConfig.equalsIgnoreCase("false")) {
- evaluateExpr = false;
- keepBoth = false;
+ if ((parserConfig != null) && parserConfig.equalsIgnoreCase(ExternalDataConstants.TRUE)) {
+ evaluateExpr = true;
}
parserConfig = configuration.get(ClassAdParserFactory.KEY_KEEP_EXPR);
- if ((parserConfig != null) && parserConfig.equalsIgnoreCase("false")) {
- keepBoth = false;
- evaluateExpr = true;
+ if ((parserConfig != null) && parserConfig.equalsIgnoreCase(ExternalDataConstants.TRUE)) {
+ keepBoth = true;
}
parserConfig = configuration.get(ClassAdParserFactory.KEY_EXPR_PREFIX);
@@ -110,7 +109,7 @@ public class ClassAdParserFactory implements IRecordDataParserFactory<char[]> {
@Override
public IRecordDataParser<char[]> createRecordParser(IHyracksTaskContext ctx) throws HyracksDataException {
return new ClassAdParser(recordType, oldFormat, evaluateExpr, keepBoth, exprPrefix, exprSuffix,
- exprFieldNameSuffix);
+ exprFieldNameSuffix, new ClassAdObjectPool());
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/test/java/org/apache/asterix/external/parser/test/RecordWithMetaTest.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/parser/test/RecordWithMetaTest.java b/asterix-external-data/src/test/java/org/apache/asterix/external/parser/test/RecordWithMetaTest.java
index 521cee0..851a7e0 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/parser/test/RecordWithMetaTest.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/parser/test/RecordWithMetaTest.java
@@ -28,7 +28,7 @@ import java.util.List;
import org.apache.asterix.external.api.IRawRecord;
import org.apache.asterix.external.input.record.converter.CSVToRecordWithMetadataAndPKConverter;
import org.apache.asterix.external.input.record.reader.stream.QuotedLineRecordReader;
-import org.apache.asterix.external.input.stream.LocalFileSystemInputStream;
+import org.apache.asterix.external.input.stream.LocalFSInputStream;
import org.apache.asterix.external.parser.ADMDataParser;
import org.apache.asterix.external.parser.RecordWithMetadataParser;
import org.apache.asterix.external.util.ExternalDataConstants;
@@ -42,7 +42,9 @@ import org.apache.hyracks.algebricks.data.IPrinter;
import org.apache.hyracks.algebricks.data.IPrinterFactory;
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+import org.apache.hyracks.dataflow.std.file.FileSplit;
import org.junit.Assert;
import org.junit.Test;
@@ -78,8 +80,10 @@ public class RecordWithMetaTest {
int[] pkIndicators = { 1 };
// create input stream
- LocalFileSystemInputStream inputStream = new LocalFileSystemInputStream(
- Paths.get(getClass().getResource("/beer.csv").toURI()).toAbsolutePath(), null, false);
+ LocalFSInputStream inputStream = new LocalFSInputStream(
+ new FileSplit[] { new FileSplit(null,
+ new FileReference(Paths.get(getClass().getResource("/beer.csv").toURI()).toFile())) },
+ null, null, 0, null, false);
// create reader record reader
QuotedLineRecordReader lineReader = new QuotedLineRecordReader(true, inputStream, null,
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/test/resources/functional_tests.txt
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/test/resources/functional_tests.txt b/asterix-external-data/src/test/resources/functional_tests.txt
index 42c0e95..3aaae46 100644
--- a/asterix-external-data/src/test/resources/functional_tests.txt
+++ b/asterix-external-data/src/test/resources/functional_tests.txt
@@ -1,3 +1,14 @@
+echo Testing select on lists...
+eval x = {
+ [a = 3; b = "I see London"],
+ [a = 2; b = "I see France"],
+ [a = 1; b = "I see Alain's funky pants"]
+ }
+same $x.a, {3, 2, 1}
+same $x.b, {"I see London", "I see France", "I see Alain's funky pants"}
+same $x.c, {undefined, undefined, undefined}
+same {}.a, {}
+
/////////////////////////////////
echo Testing basic math...
eval x = (1 + 2)
@@ -74,16 +85,7 @@ same isundefined($x.e), false
// same $y, $z[1]
/////////////////////////////////
-echo Testing select on lists...
-eval x = {
- [a = 3; b = "I see London"],
- [a = 2; b = "I see France"],
- [a = 1; b = "I see Alain's funky pants"]
- }
-same $x.a, {3, 2, 1}
-same $x.b, {"I see London", "I see France", "I see Alain's funky pants"}
-same $x.c, {undefined, undefined, undefined}
-same {}.a, {}
+
/////////////////////////////////
echo Testing subscripts
[13/13] incubator-asterixdb git commit: Improve Error Handling in
Local Directory Feeds
Posted by am...@apache.org.
Improve Error Handling in Local Directory Feeds
This change improves handling of two error types for filesystem
based feeds. The first one is the handling of IO Errors which
causes the input stream to be closed, and the second one is
reacting to missed filesystem events. In both cases, we scan the
directory and compare it with the history we have in order to
resume from where we last left off.
In addition, this change includes some refactoring in external
data. Particularly, we get rid of the stream provider layer and
instead, stream factories create input streams directly. This
is consistent with record reader factories which create readers
directly without reader providers.
Change-Id: I08d89229e33c91532b1038ba9f7a372f7ca1fdb5
Reviewed-on: https://asterix-gerrit.ics.uci.edu/720
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Murtadha Hubail <hu...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/commit/121e1d9a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/tree/121e1d9a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/diff/121e1d9a
Branch: refs/heads/master
Commit: 121e1d9abd54c60acf595b678955b38b63d38100
Parents: cf39bc9
Author: Abdullah Alamoudi <ba...@gmail.com>
Authored: Sat Mar 26 23:02:39 2016 +0300
Committer: abdullah alamoudi <ba...@gmail.com>
Committed: Sat Mar 26 13:21:49 2016 -0700
----------------------------------------------------------------------
.gitattributes | 3 +
.gitignore | 1 +
.../optimizer/rules/UnnestToDataScanRule.java | 4 +-
.../apache/asterix/app/external/FeedJoint.java | 19 +-
.../asterix/aql/translator/QueryTranslator.java | 16 +-
.../queries/feeds/feeds_08/feeds_08.1.ddl.aql | 10 +-
.../feeds/feeds_08/feeds_08.2.update.aql | 8 +-
.../queries/feeds/feeds_08/feeds_08.3.query.aql | 8 +-
.../queries/feeds/feeds_08/feeds_08.4.sleep.aql | 31 ++
.../classad-parser/classad-parser.1.adm | 200 +++----
.../feed-with-external-parser.1.adm | 198 +++----
.../src/test/resources/runtimets/testsuite.xml | 54 +-
...erixLSMInsertDeleteOperatorNodePushable.java | 7 +-
.../external/api/AsterixInputStream.java | 44 ++
.../asterix/external/api/IExternalIndexer.java | 2 +-
.../external/api/IIndexingDatasource.java | 27 +
.../external/api/IInputStreamFactory.java | 32 ++
.../external/api/IInputStreamProvider.java | 29 -
.../api/IInputStreamProviderFactory.java | 33 --
.../asterix/external/api/IRecordReader.java | 8 +-
.../dataflow/FeedRecordDataFlowController.java | 20 +-
.../dataflow/FeedStreamDataFlowController.java | 9 +-
.../external/dataflow/FeedTupleForwarder.java | 2 +-
.../external/feed/api/IFeedFrameHandler.java | 4 +-
.../asterix/external/feed/api/IFeedJoint.java | 4 +-
.../feed/api/IFeedLifecycleListener.java | 6 -
.../external/feed/api/IMessageReceiver.java | 2 +-
.../dataflow/DistributeFeedFrameWriter.java | 4 +-
.../FeedCollectRuntimeInputHandler.java | 13 +-
.../feed/dataflow/FeedFrameHandlers.java | 6 +-
.../feed/dataflow/FeedRuntimeInputHandler.java | 7 +-
.../feed/dataflow/FrameDistributor.java | 123 +++--
.../feed/management/FeedConnectionRequest.java | 9 +-
.../external/feed/message/MessageReceiver.java | 12 +-
.../external/feed/runtime/IngestionRuntime.java | 2 +-
.../external/indexing/FileOffsetIndexer.java | 16 +-
.../indexing/RecordColumnarIndexer.java | 4 +-
.../external/input/HDFSDataSourceFactory.java | 16 +-
.../external/input/record/CharArrayRecord.java | 45 +-
...questToRecordWithMetadataAndPKConverter.java | 2 +-
.../record/reader/TestAsterixMembersReader.java | 9 +-
.../hdfs/AbstractCharRecordLookupReader.java | 2 +-
.../record/reader/hdfs/HDFSRecordReader.java | 14 +-
.../input/record/reader/kv/KVReader.java | 18 +-
.../input/record/reader/kv/KVTestReader.java | 9 +-
.../record/reader/rss/RSSRecordReader.java | 9 +-
.../stream/AbstractStreamRecordReader.java | 41 +-
.../AbstractStreamRecordReaderFactory.java | 17 +-
.../stream/EmptyLineSeparatedRecordReader.java | 4 +-
.../EmptyLineSeparatedRecordReaderFactory.java | 4 +-
.../record/reader/stream/LineRecordReader.java | 4 +-
.../reader/stream/LineRecordReaderFactory.java | 4 +-
.../reader/stream/QuotedLineRecordReader.java | 6 +-
.../stream/SemiStructuredRecordReader.java | 16 +-
.../SemiStructuredRecordReaderFactory.java | 4 +-
.../reader/twitter/TwitterPullRecordReader.java | 9 +-
.../reader/twitter/TwitterPushRecordReader.java | 9 +-
.../external/input/stream/AInputStream.java | 36 --
.../input/stream/AInputStreamReader.java | 114 ----
.../input/stream/AsterixInputStreamReader.java | 120 ++++
.../external/input/stream/BasicInputStream.java | 17 +-
.../external/input/stream/HDFSInputStream.java | 237 ++++++++
.../input/stream/LocalFSInputStream.java | 182 +++++++
.../stream/LocalFileSystemInputStream.java | 137 -----
.../input/stream/SocketClientInputStream.java | 61 +++
.../input/stream/SocketServerInputStream.java | 28 +-
.../stream/TwitterFirehoseInputStream.java | 158 ++++++
.../factory/LocalFSInputStreamFactory.java | 158 ++++++
.../LocalFSInputStreamProviderFactory.java | 152 ------
.../factory/SocketClientInputStreamFactory.java | 84 +++
.../SocketClientInputStreamProviderFactory.java | 80 ---
.../factory/SocketServerInputStreamFactory.java | 141 +++++
.../SocketServerInputStreamProviderFactory.java | 141 -----
.../factory/TwitterFirehoseStreamFactory.java | 102 ++++
.../TwitterFirehoseStreamProviderFactory.java | 98 ----
.../provider/HDFSInputStreamProvider.java | 127 -----
.../provider/LocalFSInputStreamProvider.java | 58 --
.../SocketClientInputStreamProvider.java | 98 ----
.../SocketServerInputStreamProvider.java | 43 --
.../TwitterFirehoseInputStreamProvider.java | 202 -------
.../FeedCollectOperatorDescriptor.java | 52 +-
.../FeedCollectOperatorNodePushable.java | 17 +-
.../external/parser/AbstractDataParser.java | 2 +-
.../provider/DataflowControllerProvider.java | 15 +-
.../provider/DatasourceFactoryProvider.java | 22 +-
.../external/util/ExternalDataConstants.java | 5 +-
.../external/util/ExternalDataUtils.java | 6 +-
.../external/util/FileSystemWatcher.java | 52 +-
.../apache/asterix/external/util/HDFSUtils.java | 4 +-
.../external/classad/AttributeReference.java | 149 ++---
.../classad/BuiltinClassAdFunctions.java | 543 ++++++++++--------
.../external/classad/CaseInsensitiveString.java | 12 +-
.../asterix/external/classad/ClassAd.java | 411 +++++++-------
.../asterix/external/classad/ClassAdFunc.java | 4 +-
.../asterix/external/classad/ClassAdTime.java | 12 +
.../external/classad/ClassAdUnParser.java | 76 +--
.../asterix/external/classad/EvalState.java | 30 +-
.../asterix/external/classad/ExprList.java | 81 +--
.../asterix/external/classad/ExprTree.java | 56 +-
.../external/classad/ExprTreeHolder.java | 29 +-
.../asterix/external/classad/FunctionCall.java | 119 ++--
.../apache/asterix/external/classad/Lexer.java | 6 +-
.../asterix/external/classad/Literal.java | 143 ++---
.../asterix/external/classad/Operation.java | 545 ++++++++++---------
.../asterix/external/classad/PrettyPrint.java | 9 +-
.../apache/asterix/external/classad/Value.java | 149 ++---
.../pool/AMutableCharArrayStringPool.java | 35 ++
.../object/pool/AttributeReferencePool.java | 9 +-
.../object/pool/CaseInsensitiveStringPool.java | 23 +-
.../classad/object/pool/ClassAdObjectPool.java | 103 ++++
.../classad/object/pool/ClassAdParserPool.java | 39 ++
.../classad/object/pool/ClassAdPool.java | 8 +-
.../classad/object/pool/ClassAdTimePool.java | 35 ++
.../classad/object/pool/DoublePool.java | 34 ++
.../classad/object/pool/EvalStatePool.java | 41 ++
.../classad/object/pool/ExprHolderPool.java | 9 +-
.../classad/object/pool/ExprListPool.java | 8 +-
.../classad/object/pool/FunctionCallPool.java | 41 ++
.../classad/object/pool/HashMapPool.java | 34 ++
.../external/classad/object/pool/Int32Pool.java | 35 ++
.../external/classad/object/pool/Int64Pool.java | 34 ++
.../classad/object/pool/LiteralPool.java | 8 +-
.../classad/object/pool/MutableBooleanPool.java | 35 ++
.../object/pool/MutableNumberFactorPool.java | 36 ++
.../classad/object/pool/OperationPool.java | 8 +-
.../classad/object/pool/PrettyPrintPool.java | 40 ++
.../object/pool/StringArrayListPool.java | 35 ++
.../classad/object/pool/TreeSetPool.java | 35 ++
.../external/classad/object/pool/ValuePool.java | 11 +-
.../classad/test/ClassAdFunctionalTest.java | 9 +-
.../classad/test/ClassAdParserTest.java | 12 +-
.../external/classad/test/ClassAdToADMTest.java | 20 +-
.../external/classad/test/ClassAdUnitTest.java | 7 +-
.../classad/test/ClassAdUnitTester.java | 80 +--
.../external/classad/test/FunctionalTester.java | 170 +++---
.../asterix/external/library/ClassAdParser.java | 277 +++++-----
.../external/library/ClassAdParserFactory.java | 17 +-
.../parser/test/RecordWithMetaTest.java | 10 +-
.../src/test/resources/functional_tests.txt | 22 +-
.../record-parser/record-parser.1.adm | 200 +++----
.../metadata/declared/AqlMetadataProvider.java | 14 +-
.../metadata/declared/FeedDataSource.java | 8 +-
.../serde/AStringSerializerDeserializer.java | 15 +-
.../AqlSerializerDeserializerProvider.java | 9 +
.../serde/SimpleSerializerDeserializerTest.java | 3 +-
145 files changed, 4395 insertions(+), 3516 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/.gitattributes
----------------------------------------------------------------------
diff --git a/.gitattributes b/.gitattributes
index 2451378..5eb0c00 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -18,3 +18,6 @@
# Explicitly declare binary files
*testdata.txt binary
+
+#Unset sample_01.cvs
+asterix-app/data/csv/sample_01.csv text=unset
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index 1b129d6..8d0e894 100644
--- a/.gitignore
+++ b/.gitignore
@@ -16,6 +16,7 @@ asterix-app/opttest/
asterix-app/parserts/
asterix-app/opt_parserts/
asterix-app/runtime_parserts/
+asterix-app/data/csv/beer.csv
asterix-installer/ittest/
asterix-installer/repliationtest/
asterix-installer/src/test/resources/clusterts/asterix-installer-0.8.9-SNAPSHOT-binary-assembly/
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/UnnestToDataScanRule.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/UnnestToDataScanRule.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/UnnestToDataScanRule.java
index c3062ed..a0060b4 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/UnnestToDataScanRule.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/UnnestToDataScanRule.java
@@ -22,7 +22,7 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.asterix.common.config.DatasetConfig.DatasetType;
-import org.apache.asterix.external.feed.api.IFeedLifecycleListener.ConnectionLocation;
+import org.apache.asterix.external.feed.api.IFeedRuntime.FeedRuntimeType;
import org.apache.asterix.external.feed.watch.FeedActivity.FeedActivityDetails;
import org.apache.asterix.external.util.ExternalDataUtils;
import org.apache.asterix.external.util.FeedUtils;
@@ -268,7 +268,7 @@ public class UnnestToDataScanRule implements IAlgebraicRewriteRule {
}
FeedDataSource feedDataSource = new FeedDataSource(sourceFeed, aqlId, targetDataset, feedOutputType, metaType,
pkTypes, partitioningKeys, keyAccessScalarFunctionCallExpression, sourceFeed.getFeedId(),
- sourceFeed.getFeedType(), ConnectionLocation.valueOf(subscriptionLocation), locations.split(","),
+ sourceFeed.getFeedType(), FeedRuntimeType.valueOf(subscriptionLocation), locations.split(","),
context.getComputationNodeDomain());
feedDataSource.getProperties().put(BuiltinFeedPolicies.CONFIG_FEED_POLICY_KEY, feedPolicy);
return feedDataSource;
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-app/src/main/java/org/apache/asterix/app/external/FeedJoint.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/org/apache/asterix/app/external/FeedJoint.java b/asterix-app/src/main/java/org/apache/asterix/app/external/FeedJoint.java
index e650a5b..3f0bc54 100644
--- a/asterix-app/src/main/java/org/apache/asterix/app/external/FeedJoint.java
+++ b/asterix-app/src/main/java/org/apache/asterix/app/external/FeedJoint.java
@@ -24,7 +24,7 @@ import java.util.logging.Level;
import java.util.logging.Logger;
import org.apache.asterix.external.feed.api.IFeedJoint;
-import org.apache.asterix.external.feed.api.IFeedLifecycleListener.ConnectionLocation;
+import org.apache.asterix.external.feed.api.IFeedRuntime.FeedRuntimeType;
import org.apache.asterix.external.feed.management.FeedConnectionId;
import org.apache.asterix.external.feed.management.FeedConnectionRequest;
import org.apache.asterix.external.feed.management.FeedId;
@@ -49,13 +49,13 @@ public class FeedJoint implements IFeedJoint {
/** A list of feed subscription requests submitted for subscribing to the FeedPoint's data **/
private final List<FeedConnectionRequest> connectionRequests;
- private final ConnectionLocation connectionLocation;
+ private final FeedRuntimeType connectionLocation;
private final FeedJointType type;
private FeedConnectionId provider;
- public FeedJoint(FeedJointKey key, FeedId ownerFeedId, ConnectionLocation subscriptionLocation, FeedJointType type,
+ public FeedJoint(FeedJointKey key, FeedId ownerFeedId, FeedRuntimeType subscriptionLocation, FeedJointType type,
FeedConnectionId provider) {
this.key = key;
this.ownerFeedId = ownerFeedId;
@@ -72,14 +72,17 @@ public class FeedJoint implements IFeedJoint {
return key.hashCode();
}
+ @Override
public void addReceiver(FeedConnectionId connectionId) {
receivers.add(connectionId);
}
+ @Override
public void removeReceiver(FeedConnectionId connectionId) {
receivers.remove(connectionId);
}
+ @Override
public synchronized void addConnectionRequest(FeedConnectionRequest request) {
connectionRequests.add(request);
if (state.equals(State.ACTIVE)) {
@@ -87,6 +90,7 @@ public class FeedJoint implements IFeedJoint {
}
}
+ @Override
public synchronized void setState(State state) {
if (this.state.equals(state)) {
return;
@@ -121,6 +125,7 @@ public class FeedJoint implements IFeedJoint {
connectionRequests.clear();
}
+ @Override
public FeedConnectionId getReceiver(FeedConnectionId connectionId) {
for (FeedConnectionId cid : receivers) {
if (cid.equals(connectionId)) {
@@ -149,18 +154,22 @@ public class FeedJoint implements IFeedJoint {
return ((FeedJoint) o).getFeedJointKey().equals(this.key);
}
+ @Override
public FeedId getOwnerFeedId() {
return ownerFeedId;
}
+ @Override
public List<FeedConnectionRequest> getConnectionRequests() {
return connectionRequests;
}
- public ConnectionLocation getConnectionLocation() {
+ @Override
+ public FeedRuntimeType getConnectionLocation() {
return connectionLocation;
}
+ @Override
public FeedJointType getType() {
return type;
}
@@ -170,6 +179,7 @@ public class FeedJoint implements IFeedJoint {
return provider;
}
+ @Override
public List<FeedConnectionId> getReceivers() {
return receivers;
}
@@ -178,6 +188,7 @@ public class FeedJoint implements IFeedJoint {
return key;
}
+ @Override
public synchronized State getState() {
return state;
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java b/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
index 67bf3bb..3786413 100644
--- a/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
+++ b/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
@@ -63,7 +63,7 @@ import org.apache.asterix.external.feed.api.IFeedJoint;
import org.apache.asterix.external.feed.api.IFeedJoint.FeedJointType;
import org.apache.asterix.external.feed.api.IFeedLifecycleEventSubscriber;
import org.apache.asterix.external.feed.api.IFeedLifecycleEventSubscriber.FeedLifecycleEvent;
-import org.apache.asterix.external.feed.api.IFeedLifecycleListener.ConnectionLocation;
+import org.apache.asterix.external.feed.api.IFeedRuntime.FeedRuntimeType;
import org.apache.asterix.external.feed.management.FeedConnectionId;
import org.apache.asterix.external.feed.management.FeedConnectionRequest;
import org.apache.asterix.external.feed.management.FeedId;
@@ -863,7 +863,7 @@ public class QueryTranslator extends AbstractLangTranslator {
if (subType.isOpen()) {
isOpen = true;
break;
- };
+ } ;
}
}
if (fieldExpr.second == null) {
@@ -1951,7 +1951,7 @@ public class QueryTranslator extends AbstractLangTranslator {
private JobSpecification rewriteCompileQuery(AqlMetadataProvider metadataProvider, Query query,
ICompiledDmlStatement stmt)
- throws AsterixException, RemoteException, AlgebricksException, JSONException, ACIDException {
+ throws AsterixException, RemoteException, AlgebricksException, JSONException, ACIDException {
//Query Rewriting (happens under the same ongoing metadata transaction)
Pair<Query, Integer> reWrittenQuery = apiFramework.reWriteQuery(declaredFunctions, metadataProvider, query,
@@ -2257,7 +2257,7 @@ public class QueryTranslator extends AbstractLangTranslator {
*/
private Triple<FeedConnectionRequest, Boolean, List<IFeedJoint>> getFeedConnectionRequest(String dataverse,
Feed feed, String dataset, FeedPolicyEntity feedPolicy, MetadataTransactionContext mdTxnCtx)
- throws MetadataException {
+ throws MetadataException {
IFeedJoint sourceFeedJoint = null;
FeedConnectionRequest request = null;
List<String> functionsToApply = new ArrayList<String>();
@@ -2265,13 +2265,13 @@ public class QueryTranslator extends AbstractLangTranslator {
List<IFeedJoint> jointsToRegister = new ArrayList<IFeedJoint>();
FeedConnectionId connectionId = new FeedConnectionId(feed.getFeedId(), dataset);
- ConnectionLocation connectionLocation = null;
+ FeedRuntimeType connectionLocation = null;
FeedJointKey feedJointKey = getFeedJointKey(feed, mdTxnCtx);
boolean isFeedJointAvailable = FeedLifecycleListener.INSTANCE.isFeedJointAvailable(feedJointKey);
if (!isFeedJointAvailable) {
sourceFeedJoint = FeedLifecycleListener.INSTANCE.getAvailableFeedJoint(feedJointKey);
if (sourceFeedJoint == null) { //the feed is currently not being ingested, i.e., it is unavailable.
- connectionLocation = ConnectionLocation.SOURCE_FEED_INTAKE_STAGE;
+ connectionLocation = FeedRuntimeType.INTAKE;
FeedId sourceFeedId = feedJointKey.getFeedId(); // the root/primary feedId
Feed primaryFeed = MetadataManager.INSTANCE.getFeed(mdTxnCtx, dataverse, sourceFeedId.getFeedName());
FeedJointKey intakeFeedJointKey = new FeedJointKey(sourceFeedId, new ArrayList<String>());
@@ -2295,7 +2295,7 @@ public class QueryTranslator extends AbstractLangTranslator {
if (!functionsToApply.isEmpty()) {
FeedJointKey computeFeedJointKey = new FeedJointKey(feed.getFeedId(), functionsToApply);
IFeedJoint computeFeedJoint = new FeedJoint(computeFeedJointKey, feed.getFeedId(),
- ConnectionLocation.SOURCE_FEED_COMPUTE_STAGE, FeedJointType.COMPUTE, connectionId);
+ FeedRuntimeType.COMPUTE, FeedJointType.COMPUTE, connectionId);
jointsToRegister.add(computeFeedJoint);
}
} else {
@@ -2924,7 +2924,7 @@ public class QueryTranslator extends AbstractLangTranslator {
private void prepareRunExternalRuntime(AqlMetadataProvider metadataProvider, IHyracksClientConnection hcc,
RunStatement pregelixStmt, String dataverseNameFrom, String dataverseNameTo, String datasetNameFrom,
String datasetNameTo, MetadataTransactionContext mdTxnCtx)
- throws AlgebricksException, AsterixException, Exception {
+ throws AlgebricksException, AsterixException, Exception {
//Validates the source/sink dataverses and datasets.
Dataset fromDataset = metadataProvider.findDataset(dataverseNameFrom, datasetNameFrom);
if (fromDataset == null) {
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.1.ddl.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.1.ddl.aql
index af15e99..658487b 100644
--- a/asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.1.ddl.aql
+++ b/asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.1.ddl.aql
@@ -17,12 +17,14 @@
* under the License.
*/
/*
- * Description : Create a feed using the synthetic feed simulator adapter.
+ * Description : Create a feed using the synthetic feed simulator adapter.
Create a dataset that has an associated ngram index.
- The synthetic feed simulator uses the Social-Data generator to generate data and simulate a feed.
- The feed lasts a configured duration with data arriving at a configured rate (tweets per second).
+ The synthetic feed simulator uses the Social-Data generator
+ to generate data and simulate a feed.
+ The feed lasts a configured duration with data arriving at
+ a configured rate (tweets per second).
Verify the existence of data after the feed finishes.
- * Issue : 711
+ * Issue : 711
* Expected Res : Success
* Date : 8th Feb 2014
*/
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.2.update.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.2.update.aql
index b821bc7..4a1a6a4 100644
--- a/asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.2.update.aql
+++ b/asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.2.update.aql
@@ -17,10 +17,12 @@
* under the License.
*/
/*
- * Description : Create a feed using the synthetic feed simulator adapter.
+ * Description : Create a feed using the synthetic feed simulator adapter.
Create a dataset that has an associated ngram index.
- The synthetic feed simulator uses the Social-Data generator to generate data and simulate a feed.
- The feed lasts a configured duration with data arriving at a configured rate (tweets per second).
+ The synthetic feed simulator uses the Social-Data generator
+ to generate data and simulate a feed.
+ The feed lasts a configured duration with data arriving at
+ a configured rate (tweets per second).
Verify the existence of data after the feed finishes.
* Issue : 711
* Expected Res : Success
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.3.query.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.3.query.aql b/asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.3.query.aql
index 57b8ecd..177bd98 100644
--- a/asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.3.query.aql
+++ b/asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.3.query.aql
@@ -17,10 +17,12 @@
* under the License.
*/
/*
- * Description : Create a feed using the synthetic feed simulator adapter.
+ * Description : Create a feed using the synthetic feed simulator adapter.
Create a dataset that has an associated ngram index.
- The synthetic feed simulator uses the Social-Data generator to generate data and simulate a feed.
- The feed lasts a configured duration with data arriving at a configured rate (tweets per second).
+ The synthetic feed simulator uses the Social-Data generator
+ to generate data and simulate a feed.
+ The feed lasts a configured duration with data arriving at
+ a configured rate (tweets per second).
Verify the existence of data after the feed finishes.
* Issue : 711
* Expected Res : Success
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.4.sleep.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.4.sleep.aql b/asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.4.sleep.aql
new file mode 100644
index 0000000..6b7f502
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.4.sleep.aql
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Create a feed using the synthetic feed simulator adapter.
+ Create a dataset that has an associated ngram index.
+ The synthetic feed simulator uses the Social-Data generator
+ to generate data and simulate a feed.
+ The feed lasts a configured duration with data arriving at
+ a configured rate (tweets per second).
+ Verify the existence of data after the feed finishes.
+ * Issue : 711
+ * Expected Res : Success
+ * Date : 8th Feb 2014
+ */
+2000
\ No newline at end of file