You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2017/02/18 23:48:21 UTC
incubator-systemml git commit: [MINOR] Fix flaky frame converter test
(codegen and OOM issues)
Repository: incubator-systemml
Updated Branches:
refs/heads/master c360304eb -> 7d3a50d26
[MINOR] Fix flaky frame converter test (codegen and OOM issues)
This frame converter test shows code generation issues related to
SPARK-16845, which are caused by a large number of DataSet columns.
Apart from confusing exceptions, this also led to OOMs that affect
subsequent tests when ran through junit in a single jvm process. In
order to allow for smooth local testing, this fix reduces the number of
columns without affecting the tested internal code path.
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/7d3a50d2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/7d3a50d2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/7d3a50d2
Branch: refs/heads/master
Commit: 7d3a50d269c8e037fcd2614195101cec81637e6f
Parents: c360304
Author: Matthias Boehm <mb...@gmail.com>
Authored: Sat Feb 18 15:48:29 2017 -0800
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Sat Feb 18 15:48:29 2017 -0800
----------------------------------------------------------------------
.../functions/frame/FrameConverterTest.java | 48 ++++++++------------
1 file changed, 18 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7d3a50d2/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameConverterTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameConverterTest.java b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameConverterTest.java
index f626813..74a1ad4 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameConverterTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameConverterTest.java
@@ -25,6 +25,7 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.List;
+import org.apache.commons.collections.CollectionUtils;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.spark.api.java.JavaPairRDD;
@@ -76,40 +77,29 @@ public class FrameConverterTest extends AutomatedTestBase
private final static String TEST_NAME = "FrameConv";
private final static String TEST_CLASS_DIR = TEST_DIR + FrameConverterTest.class.getSimpleName() + "/";
-
private final static int rows = 1593;
+
private final static ValueType[] schemaStrings = new ValueType[]{ValueType.STRING, ValueType.STRING, ValueType.STRING};
private final static ValueType[] schemaMixed = new ValueType[]{ValueType.STRING, ValueType.DOUBLE, ValueType.INT, ValueType.BOOLEAN};
- private final static List<ValueType> schemaMixedLargeListStr = Collections.nCopies(600, ValueType.STRING);
- private final static List<ValueType> schemaMixedLargeListDble = Collections.nCopies(600, ValueType.DOUBLE);
- private final static List<ValueType> schemaMixedLargeListInt = Collections.nCopies(600, ValueType.INT);
- private final static List<ValueType> schemaMixedLargeListBool = Collections.nCopies(600, ValueType.BOOLEAN);
- private static List<ValueType> schemaMixedLargeList = null;
- static {
- schemaMixedLargeList = new ArrayList<ValueType>(schemaMixedLargeListStr);
- schemaMixedLargeList.addAll(schemaMixedLargeListDble);
- schemaMixedLargeList.addAll(schemaMixedLargeListInt);
- schemaMixedLargeList.addAll(schemaMixedLargeListBool);
- }
-
- private static ValueType[] schemaMixedLarge = new ValueType[schemaMixedLargeList.size()];
- static {
- schemaMixedLarge = (ValueType[]) schemaMixedLargeList.toArray(schemaMixedLarge);
- }
+ private final static List<ValueType> schemaMixedLargeListStr = Collections.nCopies(200, ValueType.STRING);
+ private final static List<ValueType> schemaMixedLargeListDble = Collections.nCopies(200, ValueType.DOUBLE);
+ private final static List<ValueType> schemaMixedLargeListInt = Collections.nCopies(200, ValueType.INT);
+ private final static List<ValueType> schemaMixedLargeListBool = Collections.nCopies(200, ValueType.BOOLEAN);
- private static List<ValueType> schemaMixedLargeListDFrame = null;
- static {
- schemaMixedLargeListDFrame = new ArrayList<ValueType>(schemaMixedLargeListStr.subList(0, 300));
- schemaMixedLargeListDFrame.addAll(schemaMixedLargeListDble.subList(0, 300));
- schemaMixedLargeListDFrame.addAll(schemaMixedLargeListInt.subList(0, 300));
- schemaMixedLargeListDFrame.addAll(schemaMixedLargeListBool.subList(0, 300));
- }
+ @SuppressWarnings("unchecked")
+ private static final List<ValueType> schemaMixedLargeList = new ArrayList<ValueType>(CollectionUtils.union(
+ CollectionUtils.union(schemaMixedLargeListStr, schemaMixedLargeListDble),
+ CollectionUtils.union(schemaMixedLargeListInt, schemaMixedLargeListBool)));
+ private static final ValueType[] schemaMixedLarge = schemaMixedLargeList.toArray(new ValueType[0]);
+
+ @SuppressWarnings("unchecked")
+ private static final List<ValueType> schemaMixedLargeListDFrame = new ArrayList<ValueType>(CollectionUtils.union(
+ CollectionUtils.union(schemaMixedLargeListStr.subList(0, 100), schemaMixedLargeListDble.subList(0, 100)),
+ CollectionUtils.union(schemaMixedLargeListInt.subList(0, 100), schemaMixedLargeListBool.subList(0, 100))));
+ private static final ValueType[] schemaMixedLargeDFrame = schemaMixedLargeListDFrame.toArray(new ValueType[0]);
+ //NOTE: moderate number of columns to workaround https://issues.apache.org/jira/browse/SPARK-16845
- private static ValueType[] schemaMixedLargeDFrame = new ValueType[schemaMixedLargeListDFrame.size()];
- static {
- schemaMixedLargeDFrame = (ValueType[]) schemaMixedLargeListDFrame.toArray(schemaMixedLargeDFrame);
- }
private enum ConvType {
CSV2BIN,
@@ -202,13 +192,11 @@ public class FrameConverterTest extends AutomatedTestBase
@Test
public void testFrameMixedDFrameBinSpark() {
- // TODO https://issues.apache.org/jira/browse/SPARK-16845
runFrameConverterTest(schemaMixedLargeDFrame, ConvType.DFRM2BIN);
}
@Test
public void testFrameMixedBinDFrameSpark() {
- // TODO https://issues.apache.org/jira/browse/SPARK-16845
runFrameConverterTest(schemaMixedLargeDFrame, ConvType.BIN2DFRM);
}