You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@falcon.apache.org by ar...@apache.org on 2014/08/21 19:30:46 UTC
[04/18] git commit: FALCON-606 hcat tests should stop using root dir
contributed by Raghav Gautam
FALCON-606 hcat tests should stop using root dir contributed by Raghav Gautam
Project: http://git-wip-us.apache.org/repos/asf/incubator-falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-falcon/commit/3fc1e8db
Tree: http://git-wip-us.apache.org/repos/asf/incubator-falcon/tree/3fc1e8db
Diff: http://git-wip-us.apache.org/repos/asf/incubator-falcon/diff/3fc1e8db
Branch: refs/heads/FALCON-585
Commit: 3fc1e8dbbf791b59425230169be850bc49fde735
Parents: d6b5f41
Author: Samarth Gupta <sa...@inmobi.com>
Authored: Mon Aug 18 16:05:10 2014 +0530
Committer: Samarth Gupta <sa...@inmobi.com>
Committed: Mon Aug 18 16:05:10 2014 +0530
----------------------------------------------------------------------
.../apache/falcon/regression/core/bundle/Bundle.java | 14 ++++++++++++++
.../falcon/regression/core/util/BundleUtil.java | 12 ++++++++----
.../falcon/regression/core/util/HadoopUtil.java | 4 ++++
.../falcon/regression/testHelper/BaseTestClass.java | 11 ++++++-----
.../falcon/regression/hcat/HCatProcessTest.java | 2 +-
.../falcon/regression/hcat/HCatReplicationTest.java | 2 +-
.../falcon/regression/hcat/HCatRetentionTest.java | 3 ++-
.../merlin/src/test/resources/hcat/cluster-0.1.xml | 4 ++--
.../merlin/src/test/resources/hcat/data/_SUCCESS | 13 -------------
.../merlin/src/test/resources/hcat_2/cluster-0.1.xml | 4 ++--
10 files changed, 40 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/3fc1e8db/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/bundle/Bundle.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/bundle/Bundle.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/bundle/Bundle.java
index 57c7292..35219c7 100644
--- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/bundle/Bundle.java
+++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/bundle/Bundle.java
@@ -44,6 +44,7 @@ import org.apache.falcon.regression.Entities.ProcessMerlin;
import org.apache.falcon.regression.core.helpers.ColoHelper;
import org.apache.falcon.regression.core.response.ServiceResponse;
import org.apache.falcon.regression.core.util.AssertUtil;
+import org.apache.falcon.regression.core.util.HadoopUtil;
import org.apache.falcon.regression.core.util.TimeUtil;
import org.apache.falcon.regression.core.util.Util;
import org.apache.falcon.regression.core.util.Util.URLS;
@@ -1031,4 +1032,17 @@ public class Bundle {
return feedObject.getName();
}
+ public void updateTestPath(String testPath) {
+ List<String> newClusters = new ArrayList<String>();
+ for (String cluster : clusters) {
+ ClusterMerlin clusterMerlin = new ClusterMerlin(cluster);
+ for (org.apache.falcon.entity.v0.cluster.Location location : clusterMerlin
+ .getLocations().getLocations()) {
+ location.setPath(HadoopUtil.stitchPath(testPath, location.getPath()));
+ }
+ newClusters.add(clusterMerlin.toString());
+ }
+ clusters = newClusters;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/3fc1e8db/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/BundleUtil.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/BundleUtil.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/BundleUtil.java
index 1f73523..3067d88 100644
--- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/BundleUtil.java
+++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/BundleUtil.java
@@ -60,12 +60,16 @@ public final class BundleUtil {
return readBundleFromFolder("ELbundle");
}
- public static Bundle readHCatBundle() throws IOException {
- return readBundleFromFolder("hcat");
+ public static Bundle readHCatBundle(String appPath, String testName) throws IOException {
+ final Bundle bundle = readBundleFromFolder("hcat");
+ bundle.updateTestPath(HadoopUtil.stitchPath(appPath, testName));
+ return bundle;
}
- public static Bundle readHCat2Bundle() throws IOException {
- return readBundleFromFolder("hcat_2");
+ public static Bundle readHCat2Bundle(String appPath, String testName) throws IOException {
+ final Bundle bundle = readBundleFromFolder("hcat_2");
+ bundle.updateTestPath(HadoopUtil.stitchPath(appPath, testName));
+ return bundle;
}
public static Bundle readLocalDCBundle() throws IOException {
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/3fc1e8db/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/HadoopUtil.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/HadoopUtil.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/HadoopUtil.java
index c33700c..04193d2 100644
--- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/HadoopUtil.java
+++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/HadoopUtil.java
@@ -412,4 +412,8 @@ public final class HadoopUtil {
}
}
}
+
+ public static String stitchPath(String path1, String path2) {
+ return path1.replaceAll("/*$", "") + "/" + path2.replaceAll("^/*", "");
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/3fc1e8db/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/testHelper/BaseTestClass.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/testHelper/BaseTestClass.java b/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/testHelper/BaseTestClass.java
index c64ddad..a474f34 100644
--- a/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/testHelper/BaseTestClass.java
+++ b/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/testHelper/BaseTestClass.java
@@ -45,11 +45,12 @@ public class BaseTestClass {
}
}
- public ColoHelper prism;
- public List<ColoHelper> servers;
- public List<FileSystem> serverFS;
- public List<OozieClient> serverOC;
- public String baseHDFSDir = "/tmp/falcon-regression";
+ public final ColoHelper prism;
+ public final List<ColoHelper> servers;
+ public final List<FileSystem> serverFS;
+ public final List<OozieClient> serverOC;
+ public final String baseHDFSDir = Config.getProperty("falcon.regression.test.dir");
+ public final String baseAppHDFSDir = Config.getProperty("falcon.regression.app.dir");
public static final String PRISM_PREFIX = "prism";
protected Bundle[] bundles;
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/3fc1e8db/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatProcessTest.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatProcessTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatProcessTest.java
index 32062ba..d62ddde 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatProcessTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatProcessTest.java
@@ -103,7 +103,7 @@ public class HCatProcessTest extends BaseTestClass {
@BeforeMethod(alwaysRun = true)
public void setUp() throws Exception {
clusterHC = cluster.getClusterHelper().getHCatClient();
- bundles[0] = BundleUtil.readHCatBundle();
+ bundles[0] = BundleUtil.readHCatBundle(baseAppHDFSDir, this.getClass().getSimpleName());
bundles[0] = new Bundle(bundles[0], cluster);
bundles[0].generateUniqueBundle();
bundles[0].setProcessWorkflow(hiveScriptFile, EngineType.HIVE);
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/3fc1e8db/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatReplicationTest.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatReplicationTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatReplicationTest.java
index 860e680..2f11d5c 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatReplicationTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatReplicationTest.java
@@ -94,7 +94,7 @@ public class HCatReplicationTest extends BaseTestClass {
@BeforeMethod(alwaysRun = true)
public void setUp() throws Exception {
- Bundle bundle = BundleUtil.readHCatBundle();
+ Bundle bundle = BundleUtil.readHCatBundle(baseAppHDFSDir, this.getClass().getSimpleName());
bundles[0] = new Bundle(bundle, cluster.getPrefix());
bundles[0].generateUniqueBundle();
bundles[0].setClusterInterface(Interfacetype.REGISTRY,
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/3fc1e8db/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatRetentionTest.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatRetentionTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatRetentionTest.java
index cce2860..557bd43 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatRetentionTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatRetentionTest.java
@@ -78,7 +78,8 @@ public class HCatRetentionTest extends BaseTestClass {
public void setUp() throws Exception {
HadoopUtil.recreateDir(clusterFS, baseTestHDFSDir);
cli = cluster.getClusterHelper().getHCatClient();
- bundle = new Bundle(BundleUtil.readHCat2Bundle(), cluster);
+ bundle = BundleUtil.readHCat2Bundle(baseAppHDFSDir, this.getClass().getSimpleName());
+ bundle = new Bundle(bundle, cluster);
bundle.generateUniqueBundle();
bundle.submitClusters(prism);
}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/3fc1e8db/falcon-regression/merlin/src/test/resources/hcat/cluster-0.1.xml
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/hcat/cluster-0.1.xml b/falcon-regression/merlin/src/test/resources/hcat/cluster-0.1.xml
index 5396835..7b4c006 100644
--- a/falcon-regression/merlin/src/test/resources/hcat/cluster-0.1.xml
+++ b/falcon-regression/merlin/src/test/resources/hcat/cluster-0.1.xml
@@ -36,9 +36,9 @@
</interfaces>
<locations>
- <location name="staging" path="/projects/ivory/staging" />
+ <location name="staging" path="/staging" />
<location name="temp" path="/tmp" />
- <location name="working" path="/projectsTest/ivory/working" />
+ <location name="working" path="/working" />
</locations>
<properties>
<property name="hive.metastore.client.socket.timeout" value="120"/>
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/3fc1e8db/falcon-regression/merlin/src/test/resources/hcat/data/_SUCCESS
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/hcat/data/_SUCCESS b/falcon-regression/merlin/src/test/resources/hcat/data/_SUCCESS
deleted file mode 100644
index 004c62e..0000000
--- a/falcon-regression/merlin/src/test/resources/hcat/data/_SUCCESS
+++ /dev/null
@@ -1,13 +0,0 @@
-Copyright [yyyy] [name of copyright owner]
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/3fc1e8db/falcon-regression/merlin/src/test/resources/hcat_2/cluster-0.1.xml
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/hcat_2/cluster-0.1.xml b/falcon-regression/merlin/src/test/resources/hcat_2/cluster-0.1.xml
index 9be1b2c..3cf5d9e 100644
--- a/falcon-regression/merlin/src/test/resources/hcat_2/cluster-0.1.xml
+++ b/falcon-regression/merlin/src/test/resources/hcat_2/cluster-0.1.xml
@@ -36,9 +36,9 @@
</interfaces>
<locations>
- <location name="staging" path="/projects/ivory/staging" />
+ <location name="staging" path="/staging" />
<location name="temp" path="/tmp" />
- <location name="working" path="/projectsTest/ivory/working" />
+ <location name="working" path="/working" />
</locations>
<properties>
<property name="hive.metastore.client.socket.timeout" value="120"/>