You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@falcon.apache.org by ar...@apache.org on 2014/08/21 19:30:45 UTC
[03/18] git commit: FALCON-581 merlin: Refactor code for cross
product and make it a method. Contributed by Raghav Kumar Gautam
FALCON-581 merlin: Refactor code for cross product and make it a method. Contributed by Raghav Kumar Gautam
Project: http://git-wip-us.apache.org/repos/asf/incubator-falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-falcon/commit/d6b5f417
Tree: http://git-wip-us.apache.org/repos/asf/incubator-falcon/tree/d6b5f417
Diff: http://git-wip-us.apache.org/repos/asf/incubator-falcon/diff/d6b5f417
Branch: refs/heads/FALCON-585
Commit: d6b5f417ad84740b3370e26e69cb4a70fd9c1e25
Parents: aee323d
Author: arpit <ar...@apache.org>
Authored: Thu Aug 14 08:13:51 2014 -0700
Committer: arpit <ar...@apache.org>
Committed: Thu Aug 14 08:13:51 2014 -0700
----------------------------------------------------------------------
CHANGES.txt | 3 +
.../falcon/regression/core/util/MathUtil.java | 60 ++++++++++++++++++++
.../apache/falcon/regression/NewRetryTest.java | 26 ++++-----
.../regression/hcat/HCatRetentionTest.java | 19 +------
.../regression/prism/PrismSubmitTest.java | 10 ----
.../falcon/regression/prism/RetentionTest.java | 27 ++-------
6 files changed, 81 insertions(+), 64 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/d6b5f417/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index e2c5df3..085773b 100755
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -21,6 +21,9 @@ Trunk (Unreleased)
FALCON-263 API to get workflow parameters. (pavan kumar kolamuri via Shwetha GS)
IMPROVEMENTS
+ FALCON-581 merlin: Refactor code for cross product and make it a method
+ (Raghav Kumar Gautam via Arpit Gupta)
+
FALCON-597 String logged at the start and end of the merlin test are slightly
mismatched (Raghav Kumar Gautam via Arpit Gupta)
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/d6b5f417/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/MathUtil.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/MathUtil.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/MathUtil.java
new file mode 100644
index 0000000..1bbe8d9
--- /dev/null
+++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/MathUtil.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.falcon.regression.core.util;
+
+public class MathUtil {
+ private MathUtil() {
+ throw new AssertionError("Instantiating utility class...");
+ }
+
+ /**
+ * Cross product many arrays
+ * @param firstArray first array that you want to cross product
+ * @param otherArrays other arrays that you want to cross product
+ * @return cross product
+ */
+ public static Object[][] crossProduct(Object[] firstArray, Object[]... otherArrays) {
+ if(otherArrays == null || otherArrays.length == 0) {
+ Object[][] result = new Object[firstArray.length][1];
+ for (int i = 0; i < firstArray.length; ++i) {
+ result[i][0] = firstArray[i];
+ }
+ return result;
+ }
+ // computing cross product for the rest of the arrays
+ Object[][] restArray = new Object[otherArrays.length-1][];
+ System.arraycopy(otherArrays, 1, restArray, 0, otherArrays.length - 1);
+ Object[][] restCrossProduct = crossProduct(otherArrays[0], restArray);
+ //creating and initializing result array
+ Object[][] result = new Object[firstArray.length * restCrossProduct.length][];
+ for(int i = 0; i < result.length; ++i) {
+ result[i] = new Object[otherArrays.length + 1];
+ }
+ //doing the final cross product
+ for (int i = 0; i < firstArray.length; ++i) {
+ for (int j = 0; j < restCrossProduct.length; ++j) {
+ //computing one row of result
+ final int rowIdx = i * restCrossProduct.length + j;
+ result[rowIdx][0] = firstArray[i];
+ System.arraycopy(restCrossProduct[j], 0, result[rowIdx], 1, otherArrays.length);
+ }
+ }
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/d6b5f417/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/NewRetryTest.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/NewRetryTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/NewRetryTest.java
index 65cf14e..5ab3dfe 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/NewRetryTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/NewRetryTest.java
@@ -31,6 +31,7 @@ import org.apache.falcon.regression.core.util.AssertUtil;
import org.apache.falcon.regression.core.util.BundleUtil;
import org.apache.falcon.regression.core.util.HadoopUtil;
import org.apache.falcon.regression.core.util.InstanceUtil;
+import org.apache.falcon.regression.core.util.MathUtil;
import org.apache.falcon.regression.core.util.OSUtil;
import org.apache.falcon.regression.core.util.OozieUtil;
import org.apache.falcon.regression.core.util.TimeUtil;
@@ -996,27 +997,20 @@ public class NewRetryTest extends BaseTestClass {
public Object[][] getData() {
String[] retryTypes = new String[]{"periodic", "exp-backoff"};//,"exp-backoff"
- int[] delays = new int[]{2,
+ Integer[] delays = new Integer[]{2,
0};//removing -1 since this should be checked at validation level while setting
String[] delayUnits = new String[]{"minutes"};
Integer[] retryAttempts = new Integer[]{2, 0, 3};//0,-1,2
- Object[][] testData = new Object[retryTypes.length * delays.length * delayUnits.length *
- retryAttempts.length][1];
-
- int i = 0;
-
- for (String retryType : retryTypes) {
- for (int delay : delays) {
- for (String delayUnit : delayUnits) {
- for (int retry : retryAttempts) {
- testData[i][0] = getRetry(delay, delayUnit, retryType, retry);
- i++;
- }
- }
- }
+ Object[][] crossProd = MathUtil.crossProduct(delays, delayUnits, retryTypes, retryAttempts);
+ Object[][] testData = new Object[crossProd.length][1];
+ for (int i = 0; i < crossProd.length; ++i) {
+ final Integer delay = (Integer) crossProd[i][0];
+ final String delayUnit = (String) crossProd[i][1];
+ final String retryType = (String) crossProd[i][2];
+ final Integer retry = (Integer) crossProd[i][3];
+ testData[i][0] = getRetry(delay, delayUnit, retryType, retry);
}
-
return testData;
}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/d6b5f417/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatRetentionTest.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatRetentionTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatRetentionTest.java
index 1d5fb1d..cce2860 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatRetentionTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatRetentionTest.java
@@ -27,6 +27,7 @@ import org.apache.falcon.regression.core.helpers.ColoHelper;
import org.apache.falcon.regression.core.util.AssertUtil;
import org.apache.falcon.regression.core.util.BundleUtil;
import org.apache.falcon.regression.core.util.HCatUtil;
+import org.apache.falcon.regression.core.util.MathUtil;
import org.apache.falcon.regression.core.util.OSUtil;
import org.apache.falcon.regression.core.util.OozieUtil;
import org.apache.falcon.regression.core.util.HadoopUtil;
@@ -276,28 +277,14 @@ public class HCatRetentionTest extends BaseTestClass {
public Object[][] getTestData(Method m) {
RetentionUnit[] retentionUnits = new RetentionUnit[]{RetentionUnit.HOURS, RetentionUnit.DAYS,
RetentionUnit.MONTHS};// "minutes","years",
- int[] periods = new int[]{7, 824, 43}; // a negative value like -4 should be covered
+ Integer[] periods = new Integer[]{7, 824, 43}; // a negative value like -4 should be covered
// in validation scenarios.
FeedType[] dataTypes =
new FeedType[]{
//disabling since falcon has support is for only for single hcat partition
//FeedType.DAILY, FeedType.MINUTELY, FeedType.HOURLY, FeedType.MONTHLY,
FeedType.YEARLY};
- Object[][] testData = new Object[retentionUnits.length * periods.length * dataTypes.length][3];
-
- int i = 0;
-
- for (RetentionUnit retentionUnit : retentionUnits) {
- for (int period : periods) {
- for (FeedType dataType : dataTypes) {
- testData[i][0] = period;
- testData[i][1] = retentionUnit;
- testData[i][2] = dataType;
- i++;
- }
- }
- }
- return testData;
+ return MathUtil.crossProduct(periods, retentionUnits, dataTypes);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/d6b5f417/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismSubmitTest.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismSubmitTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismSubmitTest.java
index 8156937..8f2b160 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismSubmitTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismSubmitTest.java
@@ -33,7 +33,6 @@ import org.testng.Assert;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
import java.lang.reflect.Method;
@@ -590,13 +589,4 @@ public class PrismSubmitTest extends BaseTestClass {
AssertUtil.compareDataStoreStates(beforeSubmitCluster2, afterSubmitCluster2, 0);
}
- @DataProvider(name = "errorDP")
- public Object[][] getTestData(Method m) {
- Object[][] testData = new Object[2][1];
- testData[0][0] = "EmptyInputTagProcess";
- testData[1][0] = "EmptyOutputTagProcess";
-
- return testData;
- }
-
}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/d6b5f417/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/RetentionTest.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/RetentionTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/RetentionTest.java
index c8cef02..4937361 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/RetentionTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/RetentionTest.java
@@ -30,6 +30,7 @@ import org.apache.falcon.regression.core.supportClasses.JmsMessageConsumer;
import org.apache.falcon.regression.core.util.AssertUtil;
import org.apache.falcon.regression.core.util.BundleUtil;
import org.apache.falcon.regression.core.util.HadoopUtil;
+import org.apache.falcon.regression.core.util.MathUtil;
import org.apache.falcon.regression.core.util.OozieUtil;
import org.apache.falcon.regression.core.util.TimeUtil;
import org.apache.falcon.regression.core.util.Util;
@@ -226,32 +227,14 @@ public class RetentionTest extends BaseTestClass {
@DataProvider(name = "betterDP")
public Object[][] getTestData(Method m) {
// a negative value like -4 should be covered in validation scenarios.
- int[] retentionPeriods = new int[]{0, 10080, 60, 8, 24};
+ Integer[] retentionPeriods = new Integer[]{0, 10080, 60, 8, 24};
RetentionUnit[] retentionUnits = new RetentionUnit[]{RetentionUnit.HOURS,
RetentionUnit.DAYS};// "minutes","hours", "days",
- boolean[] gaps = new boolean[]{false, true};
+ Boolean[] gaps = new Boolean[]{false, true};
FeedType[] feedTypes = new FeedType[]{FeedType.DAILY, FeedType.YEARLY, FeedType.MONTHLY};
- Object[][] testData = new Object[retentionPeriods.length * retentionUnits.length *
- gaps.length * feedTypes.length][5];
-
- int i = 0;
-
- for (RetentionUnit retentionUnit : retentionUnits) {
- for (int period : retentionPeriods) {
- for (boolean gap : gaps) {
- for (FeedType feedType : feedTypes) {
- testData[i][0] = period;
- testData[i][1] = retentionUnit;
- testData[i][2] = gap;
- testData[i][3] = feedType;
- testData[i][4] = true;
- i++;
- }
- }
- }
- }
+ final Boolean[] withData = new Boolean[]{true};
- return testData;
+ return MathUtil.crossProduct(retentionPeriods, retentionUnits, gaps, feedTypes, withData);
}
}