You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@falcon.apache.org by sr...@apache.org on 2013/09/18 10:23:38 UTC
git commit: FALCON-127 Fix test issues after FALCON-107. Contributed
by Shwetha GS
Updated Branches:
refs/heads/master fa2bc8e5c -> 518dc8952
FALCON-127 Fix test issues after FALCON-107. Contributed by Shwetha GS
Project: http://git-wip-us.apache.org/repos/asf/incubator-falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-falcon/commit/518dc895
Tree: http://git-wip-us.apache.org/repos/asf/incubator-falcon/tree/518dc895
Diff: http://git-wip-us.apache.org/repos/asf/incubator-falcon/diff/518dc895
Branch: refs/heads/master
Commit: 518dc89528b73d5c65248db56a87518a255446b7
Parents: fa2bc8e
Author: srikanth.sundarrajan <sr...@inmobi.com>
Authored: Wed Sep 18 13:53:12 2013 +0530
Committer: srikanth.sundarrajan <sr...@inmobi.com>
Committed: Wed Sep 18 13:53:12 2013 +0530
----------------------------------------------------------------------
CHANGES.txt | 3 +++
docs/src/site/twiki/InstallationSteps.twiki | 10 +++++---
.../falcon/converter/OozieFeedMapperTest.java | 11 +++------
.../converter/AbstractOozieEntityMapper.java | 13 ++++++----
.../converter/OozieProcessMapperTest.java | 26 +++++++++-----------
.../falcon/retention/FeedEvictorTest.java | 16 ++++++------
6 files changed, 42 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/518dc895/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index ddb3418..48e19d0 100755
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -28,6 +28,9 @@ Trunk (Unreleased)
Srikanth Sundarrajan)
BUG FIXES
+ FALCON-127 Fix test issues after FALCON-107. (Shwetha GS via Srikanth
+ Sundarrajan)
+
FALCON-76 Use cluster readonly endpoint for replication source.
(Suhas Vasu via Shweth GS)
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/518dc895/docs/src/site/twiki/InstallationSteps.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/InstallationSteps.twiki b/docs/src/site/twiki/InstallationSteps.twiki
index c459fb0..ab23d49 100644
--- a/docs/src/site/twiki/InstallationSteps.twiki
+++ b/docs/src/site/twiki/InstallationSteps.twiki
@@ -8,10 +8,10 @@ git clone https://git-wip-us.apache.org/repos/asf/incubator-falcon.git falcon
cd falcon
-export MAVEN_OPTS="-Xmx1024m -XX:MaxPermSize=256m" && mvn clean install
+export MAVEN_OPTS="-Xmx1024m -XX:MaxPermSize=256m" && mvn clean install [For hadoop 1]
+export MAVEN_OPTS="-Xmx1024m -XX:MaxPermSize=256m" && mvn clean install -Dhadoop.profile=2 [For hadoop 2]
[optionally -Dhadoop.version=<<hadoop.version>> can be appended to build for a specific version of hadoop]
-[Falcon has currently not been tested with secure Hadoop / Hadoop 2.0]
</verbatim>
@@ -20,7 +20,8 @@ Once the build successfully completes, artifacts can be packaged for deployment.
*Embedded Mode*
<verbatim>
-mvn clean assembly:assembly -DskipTests -DskipCheck=true
+mvn clean assembly:assembly -DskipTests -DskipCheck=true [For hadoop 1]
+mvn clean assembly:assembly -DskipTests -DskipCheck=true -Dhadoop.profile=2 [For hadoop 2]
</verbatim>
@@ -56,7 +57,8 @@ Tar is structured as follows
<verbatim>
-mvn clean assembly:assembly -DskipTests -DskipCheck=true -P distributed
+mvn clean assembly:assembly -DskipTests -DskipCheck=true -P distributed -Dhadoop.profile=1 [For hadoop 1]
+mvn clean assembly:assembly -DskipTests -DskipCheck=true -P distributed -Dhadoop.profile=2 [For hadoop 2]
</verbatim>
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/518dc895/feed/src/test/java/org/apache/falcon/converter/OozieFeedMapperTest.java
----------------------------------------------------------------------
diff --git a/feed/src/test/java/org/apache/falcon/converter/OozieFeedMapperTest.java b/feed/src/test/java/org/apache/falcon/converter/OozieFeedMapperTest.java
index bde6589..4fb6a58 100644
--- a/feed/src/test/java/org/apache/falcon/converter/OozieFeedMapperTest.java
+++ b/feed/src/test/java/org/apache/falcon/converter/OozieFeedMapperTest.java
@@ -19,17 +19,13 @@ package org.apache.falcon.converter;
import static org.testng.Assert.assertEquals;
-import java.io.IOException;
-import java.io.StringWriter;
import java.util.Collection;
import java.util.List;
import javax.xml.bind.JAXBContext;
import javax.xml.bind.JAXBElement;
-import javax.xml.bind.JAXBException;
import javax.xml.bind.Unmarshaller;
-import org.apache.commons.io.IOUtils;
import org.apache.falcon.FalconException;
import org.apache.falcon.cluster.util.EmbeddedCluster;
import org.apache.falcon.entity.ClusterHelper;
@@ -96,7 +92,8 @@ public class OozieFeedMapperTest {
ClusterHelper.getInterface(cluster, Interfacetype.WRITE).setEndpoint(writeEndpoint);
FileSystem fs = new Path(writeEndpoint).getFileSystem(new Configuration());
fs.create(new Path(ClusterHelper.getLocation(cluster, "working"), "libext/FEED/retention/ext.jar")).close();
- fs.create(new Path(ClusterHelper.getLocation(cluster, "working"), "libext/FEED/replication/ext.jar")).close();
+ fs.create(
+ new Path(ClusterHelper.getLocation(cluster, "working"), "libext/FEED/replication/ext.jar")).close();
}
return entity;
}
@@ -173,8 +170,8 @@ public class OozieFeedMapperTest {
private void assertLibExtensions(COORDINATORAPP coord, String lifecycle) throws Exception {
String wfPath = coord.getAction().getWorkflow().getAppPath().replace("${nameNode}", "");
- JAXBContext WORKFLOW_JAXB_CONTEXT = JAXBContext.newInstance(WORKFLOWAPP.class);
- WORKFLOWAPP wf = ((JAXBElement<WORKFLOWAPP>) WORKFLOW_JAXB_CONTEXT.createUnmarshaller().unmarshal(
+ JAXBContext jaxbContext = JAXBContext.newInstance(WORKFLOWAPP.class);
+ WORKFLOWAPP wf = ((JAXBElement<WORKFLOWAPP>) jaxbContext.createUnmarshaller().unmarshal(
trgMiniDFS.getFileSystem().open(new Path(wfPath, "workflow.xml")))).getValue();
List<Object> actions = wf.getDecisionOrForkOrJoin();
for (Object obj : actions) {
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/518dc895/oozie/src/main/java/org/apache/falcon/converter/AbstractOozieEntityMapper.java
----------------------------------------------------------------------
diff --git a/oozie/src/main/java/org/apache/falcon/converter/AbstractOozieEntityMapper.java b/oozie/src/main/java/org/apache/falcon/converter/AbstractOozieEntityMapper.java
index 4a55bb6..5ee55c2 100644
--- a/oozie/src/main/java/org/apache/falcon/converter/AbstractOozieEntityMapper.java
+++ b/oozie/src/main/java/org/apache/falcon/converter/AbstractOozieEntityMapper.java
@@ -48,10 +48,7 @@ import org.apache.log4j.Logger;
import org.apache.oozie.client.OozieClient;
import javax.xml.bind.*;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.io.StringWriter;
+import java.io.*;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -146,7 +143,13 @@ public abstract class AbstractOozieEntityMapper<T extends Entity> {
}
private void addExtensionJars(FileSystem fs, Path path, WORKFLOWAPP wf) throws IOException {
- FileStatus[] libs = fs.listStatus(path);
+ FileStatus[] libs = null;
+ try {
+ libs = fs.listStatus(path);
+ } catch(FileNotFoundException ignore) {
+ //Ok if the libext is not configured
+ }
+
if (libs == null) {
return;
}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/518dc895/process/src/test/java/org/apache/falcon/converter/OozieProcessMapperTest.java
----------------------------------------------------------------------
diff --git a/process/src/test/java/org/apache/falcon/converter/OozieProcessMapperTest.java b/process/src/test/java/org/apache/falcon/converter/OozieProcessMapperTest.java
index 6ed0d9c..9d13d79 100644
--- a/process/src/test/java/org/apache/falcon/converter/OozieProcessMapperTest.java
+++ b/process/src/test/java/org/apache/falcon/converter/OozieProcessMapperTest.java
@@ -206,8 +206,8 @@ public class OozieProcessMapperTest extends AbstractTestBase {
private void assertLibExtensions(COORDINATORAPP coord) throws Exception {
String wfPath = coord.getAction().getWorkflow().getAppPath().replace("${nameNode}", "");
- JAXBContext WORKFLOW_JAXB_CONTEXT = JAXBContext.newInstance(WORKFLOWAPP.class);
- WORKFLOWAPP wf = ((JAXBElement<WORKFLOWAPP>) WORKFLOW_JAXB_CONTEXT.createUnmarshaller().unmarshal(
+ JAXBContext jaxbContext = JAXBContext.newInstance(WORKFLOWAPP.class);
+ WORKFLOWAPP wf = ((JAXBElement<WORKFLOWAPP>) jaxbContext.createUnmarshaller().unmarshal(
fs.open(new Path(wfPath, "workflow.xml")))).getValue();
List<Object> actions = wf.getDecisionOrForkOrJoin();
for (Object obj : actions) {
@@ -236,24 +236,22 @@ public class OozieProcessMapperTest extends AbstractTestBase {
OozieProcessMapper mapper = new OozieProcessMapper(process);
Path bundlePath = new Path("/", EntityUtil.getStagingPath(process));
mapper.map(cluster, bundlePath);
-
- FileSystem fs = new Path(hdfsUrl).getFileSystem(new Configuration());
assertTrue(fs.exists(bundlePath));
- BUNDLEAPP bundle = getBundle(fs, bundlePath);
+ BUNDLEAPP bundle = getBundle(bundlePath);
assertEquals(EntityUtil.getWorkflowName(process).toString(), bundle.getName());
assertEquals(1, bundle.getCoordinator().size());
assertEquals(EntityUtil.getWorkflowName(Tag.DEFAULT, process).toString(),
bundle.getCoordinator().get(0).getName());
String coordPath = bundle.getCoordinator().get(0).getAppPath().replace("${nameNode}", "");
- COORDINATORAPP coord = getCoordinator(fs, new Path(coordPath));
+ COORDINATORAPP coord = getCoordinator(new Path(coordPath));
testDefCoordMap(process, coord);
assertEquals(coord.getControls().getThrottle(), throttle);
assertEquals(coord.getControls().getTimeout(), timeout);
String wfPath = coord.getAction().getWorkflow().getAppPath().replace("${nameNode}", "");
- return getParentWorkflow(fs, new Path(wfPath));
+ return getParentWorkflow(new Path(wfPath));
}
public void testParentWorkflow(Process process, WORKFLOWAPP parentWorkflow) {
@@ -267,8 +265,8 @@ public class OozieProcessMapperTest extends AbstractTestBase {
Assert.assertEquals("failed-post-processing", ((ACTION) decisionOrForkOrJoin.get(6)).getName());
}
- private COORDINATORAPP getCoordinator(FileSystem fs, Path path) throws Exception {
- String bundleStr = readFile(fs, path);
+ private COORDINATORAPP getCoordinator(Path path) throws Exception {
+ String bundleStr = readFile(path);
Unmarshaller unmarshaller = JAXBContext.newInstance(COORDINATORAPP.class).createUnmarshaller();
SchemaFactory schemaFactory = SchemaFactory.newInstance("http://www.w3.org/2001/XMLSchema");
@@ -279,8 +277,8 @@ public class OozieProcessMapperTest extends AbstractTestBase {
return jaxbBundle.getValue();
}
- private WORKFLOWAPP getParentWorkflow(FileSystem fs, Path path) throws Exception {
- String workflow = readFile(fs, new Path(path, "workflow.xml"));
+ private WORKFLOWAPP getParentWorkflow(Path path) throws Exception {
+ String workflow = readFile(new Path(path, "workflow.xml"));
Unmarshaller unmarshaller = JAXBContext.newInstance(WORKFLOWAPP.class).createUnmarshaller();
SchemaFactory schemaFactory = SchemaFactory.newInstance("http://www.w3.org/2001/XMLSchema");
@@ -291,8 +289,8 @@ public class OozieProcessMapperTest extends AbstractTestBase {
return jaxbWorkflow.getValue();
}
- private BUNDLEAPP getBundle(FileSystem fs, Path path) throws Exception {
- String bundleStr = readFile(fs, new Path(path, "bundle.xml"));
+ private BUNDLEAPP getBundle(Path path) throws Exception {
+ String bundleStr = readFile(new Path(path, "bundle.xml"));
Unmarshaller unmarshaller = JAXBContext.newInstance(BUNDLEAPP.class).createUnmarshaller();
SchemaFactory schemaFactory = SchemaFactory.newInstance("http://www.w3.org/2001/XMLSchema");
@@ -303,7 +301,7 @@ public class OozieProcessMapperTest extends AbstractTestBase {
return jaxbBundle.getValue();
}
- private String readFile(FileSystem fs, Path path) throws Exception {
+ private String readFile(Path path) throws Exception {
BufferedReader reader = new BufferedReader(new InputStreamReader(fs.open(path)));
String line;
StringBuilder contents = new StringBuilder();
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/518dc895/retention/src/test/java/org/apache/falcon/retention/FeedEvictorTest.java
----------------------------------------------------------------------
diff --git a/retention/src/test/java/org/apache/falcon/retention/FeedEvictorTest.java b/retention/src/test/java/org/apache/falcon/retention/FeedEvictorTest.java
index fc55e11..5377c48 100644
--- a/retention/src/test/java/org/apache/falcon/retention/FeedEvictorTest.java
+++ b/retention/src/test/java/org/apache/falcon/retention/FeedEvictorTest.java
@@ -52,10 +52,12 @@ public class FeedEvictorTest {
private EmbeddedCluster cluster;
private final InMemoryWriter stream = new InMemoryWriter(System.out);
private final Map<String, String> map = new HashMap<String, String>();
+ private String hdfsUrl;
@BeforeClass
public void start() throws Exception {
cluster = EmbeddedCluster.newCluster("test");
+ hdfsUrl = cluster.getConf().get("fs.default.name");
FeedEvictor.OUT.set(stream);
}
@@ -101,7 +103,7 @@ public class FeedEvictorTest {
Pair<List<String>, List<String>> pair;
pair = createTestData("feed1", "yyyy-MM-dd/'more'/yyyy", 10, TimeUnit.DAYS, "/data");
String dataPath = "/data/YYYY/feed1/mmHH/dd/MM/?{YEAR}-?{MONTH}-?{DAY}/more/?{YEAR}";
- String logFile = "/falcon/staging/feed/instancePaths-2012-01-01-01-00.csv";
+ String logFile = hdfsUrl + "/falcon/staging/feed/instancePaths-2012-01-01-01-00.csv";
FeedEvictor.main(new String[]{
"-feedBasePath", cluster.getConf().get("fs.default.name") + dataPath,
@@ -183,7 +185,7 @@ public class FeedEvictorTest {
Pair<List<String>, List<String>> pair;
pair = createTestData("feed2", "yyyyMMddHH/'more'/yyyy", 5, TimeUnit.HOURS, "/data");
String dataPath = "/data/YYYY/feed2/mmHH/dd/MM/?{YEAR}?{MONTH}?{DAY}?{HOUR}/more/?{YEAR}";
- String logFile = "/falcon/staging/feed/instancePaths-2012-01-01-02-00.csv";
+ String logFile = hdfsUrl + "/falcon/staging/feed/instancePaths-2012-01-01-02-00.csv";
FeedEvictor.main(new String[]{
"-feedBasePath", cluster.getConf().get("fs.default.name") + dataPath,
"-retentionType", "instance", "-retentionLimit", "hours(5)", "-timeZone", "UTC", "-frequency",
@@ -214,12 +216,12 @@ public class FeedEvictorTest {
"-feedBasePath",
cluster.getConf().get("fs.default.name") + "/data/YYYY/feed3/dd/MM/?{MONTH}/more/?{HOUR}",
"-retentionType", "instance", "-retentionLimit", "months(5)", "-timeZone",
- "UTC", "-frequency", "hourly", "-logFile", "/falcon/staging/feed/2012-01-01-04-00", });
+ "UTC", "-frequency", "hourly", "-logFile", hdfsUrl + "/falcon/staging/feed/2012-01-01-04-00", });
Assert.assertEquals("instances=NULL", stream.getBuffer());
stream.clear();
String dataPath = "/data/YYYY/feed4/dd/MM/02/more/hello";
- String logFile = "/falcon/staging/feed/instancePaths-2012-01-01-02-00.csv";
+ String logFile = hdfsUrl + "/falcon/staging/feed/instancePaths-2012-01-01-02-00.csv";
FeedEvictor.main(new String[] {"-feedBasePath",
cluster.getConf().get("fs.default.name") + dataPath, "-retentionType", "instance",
"-retentionLimit", "hours(5)", "-timeZone", "UTC", "-frequency", "hourly", "-logFile", logFile, });
@@ -252,12 +254,12 @@ public class FeedEvictorTest {
+ getFeedBasePath("/meta") + "#"
+ getFeedBasePath("/tmp"),
"-retentionType", "instance", "-retentionLimit", "months(5)", "-timeZone",
- "UTC", "-frequency", "hourly", "-logFile", "/falcon/staging/feed/2012-01-01-04-00", });
+ "UTC", "-frequency", "hourly", "-logFile", hdfsUrl + "/falcon/staging/feed/2012-01-01-04-00", });
Assert.assertEquals("instances=NULL", stream.getBuffer());
stream.clear();
String dataPath = "/data/YYYY/feed4/dd/MM/02/more/hello";
- String logFile = "/falcon/staging/feed/instancePaths-2012-01-01-02-00.csv";
+ String logFile = hdfsUrl + "/falcon/staging/feed/instancePaths-2012-01-01-02-00.csv";
FeedEvictor.main(new String[]{"-feedBasePath",
cluster.getConf().get("fs.default.name") + dataPath,
"-retentionType", "instance", "-retentionLimit",
@@ -293,7 +295,7 @@ public class FeedEvictorTest {
+ "#"
+ cluster.getConf().get("fs.default.name")
+ "/meta/YYYY/feed1/mmHH/dd/MM/?{YEAR}-?{MONTH}-?{DAY}/more/?{YEAR}";
- String logFile = "/falcon/staging/feed/instancePaths-2012-01-01-01-00.csv";
+ String logFile = hdfsUrl + "/falcon/staging/feed/instancePaths-2012-01-01-01-00.csv";
FeedEvictor.main(new String[] {"-feedBasePath", dataPath, "-retentionType", "instance",
"-retentionLimit", "days(10)", "-timeZone", "UTC", "-frequency", "daily", "-logFile", logFile, });