You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@falcon.apache.org by ra...@apache.org on 2014/12/10 22:24:52 UTC
incubator-falcon git commit: FALCON-908 Remove jars that are not
needed. Contributed by Paul Isaychuk
Repository: incubator-falcon
Updated Branches:
refs/heads/master 73c9b9f54 -> 4133c9633
FALCON-908 Remove jars that are not needed. Contributed by Paul Isaychuk
Project: http://git-wip-us.apache.org/repos/asf/incubator-falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-falcon/commit/4133c963
Tree: http://git-wip-us.apache.org/repos/asf/incubator-falcon/tree/4133c963
Diff: http://git-wip-us.apache.org/repos/asf/incubator-falcon/diff/4133c963
Branch: refs/heads/master
Commit: 4133c9633baf709a234c493cb973ac30d3bcb14c
Parents: 73c9b9f
Author: Raghav Kumar Gautam <ra...@apache.org>
Authored: Wed Dec 10 13:24:45 2014 -0800
Committer: Raghav Kumar Gautam <ra...@apache.org>
Committed: Wed Dec 10 13:24:45 2014 -0800
----------------------------------------------------------------------
falcon-regression/CHANGES.txt | 2 +
.../falcon/regression/core/bundle/Bundle.java | 11 ++--
.../helpers/entity/AbstractEntityHelper.java | 32 +++++-----
.../falcon/regression/ProcessLibPathTest.java | 10 +--
.../resources/ELbundle/workflow/workflow.xml | 61 -------------------
.../src/test/resources/IvoryClient/IvoryCLI.jar | Bin 4386324 -> 0 bytes
.../src/test/resources/ivory-oozie-lib-0.1.jar | Bin 8530 -> 0 bytes
.../oozie/lib/oozie-examples-3.1.5.jar | Bin 15826 -> 0 bytes
.../src/test/resources/oozie/workflow.xml | 4 +-
.../lib/oozie-examples-3.1.5.jar | Bin 15826 -> 0 bytes
10 files changed, 28 insertions(+), 92 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4133c963/falcon-regression/CHANGES.txt
----------------------------------------------------------------------
diff --git a/falcon-regression/CHANGES.txt b/falcon-regression/CHANGES.txt
index 8ae1c7a..f7b9781 100644
--- a/falcon-regression/CHANGES.txt
+++ b/falcon-regression/CHANGES.txt
@@ -36,6 +36,8 @@ Trunk (Unreleased)
via Samarth Gupta)
IMPROVEMENTS
+ FALCON-908 Remove jars that are not needed (Paul Isaychuk via Raghav Kumar Gautam)
+
FALCON-939 Fixing few typos and removing unused stuff (Paul Isaychuk via Ruslan Ostafiychuk)
FALCON-930 Delete old project name (ivory) from code and xmls (Ruslan Ostafiychuk via Raghav
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4133c963/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/bundle/Bundle.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/bundle/Bundle.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/bundle/Bundle.java
index 4c913f5..91abe89 100644
--- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/bundle/Bundle.java
+++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/bundle/Bundle.java
@@ -572,10 +572,11 @@ public class Bundle {
}
- public void verifyDependencyListing(ColoHelper coloHelper) {
+ public void verifyDependencyListing(ColoHelper coloHelper)
+ throws InterruptedException, IOException, AuthenticationException, URISyntaxException {
//display dependencies of process:
String dependencies = coloHelper.getProcessHelper().getDependencies(
- Util.readEntityName(getProcessData()));
+ Util.readEntityName(getProcessData())).getEntityList().toString();
//verify presence
for (String cluster : clusters) {
@@ -585,15 +586,13 @@ public class Bundle {
Assert.assertTrue(dependencies.contains("(feed) " + Util.readEntityName(feed)));
for (String cluster : clusters) {
Assert.assertTrue(coloHelper.getFeedHelper().getDependencies(
- Util.readEntityName(feed))
+ Util.readEntityName(feed)).getEntityList().toString()
.contains("(cluster) " + Util.readEntityName(cluster)));
}
Assert.assertFalse(coloHelper.getFeedHelper().getDependencies(
- Util.readEntityName(feed))
+ Util.readEntityName(feed)).getEntityList().toString()
.contains("(process)" + Util.readEntityName(getProcessData())));
}
-
-
}
public void addProcessInput(String feed, String feedName) {
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4133c963/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/entity/AbstractEntityHelper.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/entity/AbstractEntityHelper.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/entity/AbstractEntityHelper.java
index 797554b..b365a4c 100644
--- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/entity/AbstractEntityHelper.java
+++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/entity/AbstractEntityHelper.java
@@ -29,7 +29,6 @@ import org.apache.falcon.regression.core.util.ExecUtil;
import org.apache.falcon.regression.core.util.FileUtil;
import org.apache.falcon.regression.core.util.HCatUtil;
import org.apache.falcon.regression.core.util.InstanceUtil;
-import org.apache.falcon.regression.core.util.OSUtil;
import org.apache.falcon.regression.core.util.OozieUtil;
import org.apache.falcon.regression.core.util.Util;
import org.apache.falcon.regression.core.util.Util.URLS;
@@ -56,10 +55,6 @@ public abstract class AbstractEntityHelper {
private static final Logger LOGGER = Logger.getLogger(AbstractEntityHelper.class);
- protected static final String CLIENT_LOCATION = OSUtil.RESOURCES
- + OSUtil.getPath("IvoryClient", "IvoryCLI.jar");
- protected static final String BASE_COMMAND = "java -jar " + CLIENT_LOCATION;
-
private static boolean setAuthenticate() {
String value = Config.getProperty("isAuthenticationSet");
value = (null == value) ? "true" : value;
@@ -495,17 +490,6 @@ public abstract class AbstractEntityHelper {
.createAndSendRequestProcessInstance(url, params, allColo, null);
}
- public String list() {
- return ExecUtil.executeCommandGetOutput(
- BASE_COMMAND + " entity -list -url " + this.hostname + " -type " + getEntityType());
- }
-
- public String getDependencies(String entityName) {
- return ExecUtil.executeCommandGetOutput(
- BASE_COMMAND + " entity -dependency -url " + this.hostname + " -type "
- + getEntityType() + " -name " + entityName);
- }
-
public List<String> getArchiveInfo() throws IOException, JSchException {
return Util.getStoreInfo(this, "/archive/" + getEntityType().toUpperCase());
}
@@ -597,4 +581,20 @@ public abstract class AbstractEntityHelper {
}
return (InstancesResult) InstanceUtil.sendRequestProcessInstance(url, user);
}
+
+ /**
+ * Get list of all dependencies of a given entity.
+ * @param entityName entity name
+ * @return response
+ * @throws URISyntaxException
+ * @throws AuthenticationException
+ * @throws InterruptedException
+ * @throws IOException
+ */
+ public ServiceResponse getDependencies(String entityName)
+ throws URISyntaxException, AuthenticationException, InterruptedException, IOException {
+ String url = createUrl(this.hostname + URLS.DEPENDENCIES.getValue(), getEntityType(),
+ entityName + colo);
+ return Util.sendRequest(url, "get", null, null);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4133c963/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessLibPathTest.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessLibPathTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessLibPathTest.java
index 6b1dec3..f335f04 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessLibPathTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessLibPathTest.java
@@ -60,9 +60,6 @@ public class ProcessLibPathTest extends BaseTestClass {
@BeforeClass(alwaysRun = true)
public void createTestData() throws Exception {
LOGGER.info("in @BeforeClass");
-
- //common lib for both test cases
- HadoopUtil.uploadDir(clusterFS, testLibDir, OSUtil.RESOURCES_OOZIE + "lib");
Bundle b = BundleUtil.readELBundle();
b.generateUniqueBundle();
b = new Bundle(b, cluster);
@@ -98,7 +95,7 @@ public class ProcessLibPathTest extends BaseTestClass {
}
/**
- * Test which test a process with no lib folder in workflow location.
+ * Tests a process with no lib folder in workflow location.
*
* @throws Exception
*/
@@ -106,7 +103,6 @@ public class ProcessLibPathTest extends BaseTestClass {
public void setDifferentLibPathWithNoLibFolderInWorkflowfLocaltion() throws Exception {
String workflowDir = testLibDir + "/aggregatorLib1/";
HadoopUtil.uploadDir(clusterFS, workflowDir, OSUtil.RESOURCES_OOZIE);
- HadoopUtil.deleteDirIfExists(workflowDir + "/lib", clusterFS);
bundles[0].setProcessWorkflow(workflowDir);
LOGGER.info("processData: " + Util.prettyPrintXml(process));
bundles[0].submitFeedsScheduleProcess(prism);
@@ -125,8 +121,8 @@ public class ProcessLibPathTest extends BaseTestClass {
String workflowDir = testLibDir + "/aggregatorLib2/";
HadoopUtil.uploadDir(clusterFS, workflowDir, OSUtil.RESOURCES_OOZIE);
HadoopUtil.recreateDir(clusterFS, workflowDir + "/lib");
- HadoopUtil.copyDataToFolder(clusterFS, workflowDir + "/lib",
- OSUtil.RESOURCES + "ivory-oozie-lib-0.1.jar");
+ HadoopUtil.copyDataToFolder(clusterFS, workflowDir + "/lib/invalid.jar",
+ OSUtil.RESOURCES + "feed-s4Replication.xml");
bundles[0].setProcessWorkflow(workflowDir);
LOGGER.info("processData: " + Util.prettyPrintXml(process));
bundles[0].submitFeedsScheduleProcess(prism);
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4133c963/falcon-regression/merlin/src/test/resources/ELbundle/workflow/workflow.xml
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/ELbundle/workflow/workflow.xml b/falcon-regression/merlin/src/test/resources/ELbundle/workflow/workflow.xml
deleted file mode 100644
index 4a80109..0000000
--- a/falcon-regression/merlin/src/test/resources/ELbundle/workflow/workflow.xml
+++ /dev/null
@@ -1,61 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<workflow-app xmlns="uri:oozie:workflow:0.2" name="aggregator-wf">
- <start to="aggregator"/>
- <action name="aggregator">
- <map-reduce>
- <job-tracker>${jobTracker}</job-tracker>
- <name-node>${nameNode}</name-node>
- <prepare>
- <delete path="${outputData}"/>
- </prepare>
- <configuration>
- <property>
- <name>mapred.job.queue.name</name>
- <value>${queueName}</value>
- </property>
- <property>
- <name>mapred.mapper.class</name>
- <value>org.apache.oozie.example.SampleMapper</value>
- </property>
- <property>
- <name>mapred.reducer.class</name>
- <value>org.apache.oozie.example.SampleReducer</value>
- </property>
- <property>
- <name>mapred.map.tasks</name>
- <value>1</value>
- </property>
- <property>
- <name>mapred.input.dir</name>
- <value>${inputData}</value>
- </property>
- <property>
- <name>mapred.output.dir</name>
- <value>${outputData}</value>
- </property>
- </configuration>
- </map-reduce>
- <ok to="end"/>
- <error to="fail"/>
- </action>
- <kill name="fail">
- <message>Map/Reduce failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
- </kill>
- <end name="end"/>
-</workflow-app>
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4133c963/falcon-regression/merlin/src/test/resources/IvoryClient/IvoryCLI.jar
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/IvoryClient/IvoryCLI.jar b/falcon-regression/merlin/src/test/resources/IvoryClient/IvoryCLI.jar
deleted file mode 100644
index 319bf5a..0000000
Binary files a/falcon-regression/merlin/src/test/resources/IvoryClient/IvoryCLI.jar and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4133c963/falcon-regression/merlin/src/test/resources/ivory-oozie-lib-0.1.jar
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/ivory-oozie-lib-0.1.jar b/falcon-regression/merlin/src/test/resources/ivory-oozie-lib-0.1.jar
deleted file mode 100755
index 992feab..0000000
Binary files a/falcon-regression/merlin/src/test/resources/ivory-oozie-lib-0.1.jar and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4133c963/falcon-regression/merlin/src/test/resources/oozie/lib/oozie-examples-3.1.5.jar
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/oozie/lib/oozie-examples-3.1.5.jar b/falcon-regression/merlin/src/test/resources/oozie/lib/oozie-examples-3.1.5.jar
deleted file mode 100755
index 380c6f3..0000000
Binary files a/falcon-regression/merlin/src/test/resources/oozie/lib/oozie-examples-3.1.5.jar and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4133c963/falcon-regression/merlin/src/test/resources/oozie/workflow.xml
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/oozie/workflow.xml b/falcon-regression/merlin/src/test/resources/oozie/workflow.xml
index 4a80109..8c9cdaf 100644
--- a/falcon-regression/merlin/src/test/resources/oozie/workflow.xml
+++ b/falcon-regression/merlin/src/test/resources/oozie/workflow.xml
@@ -31,11 +31,11 @@
</property>
<property>
<name>mapred.mapper.class</name>
- <value>org.apache.oozie.example.SampleMapper</value>
+ <value>org.apache.hadoop.mapred.lib.IdentityMapper</value>
</property>
<property>
<name>mapred.reducer.class</name>
- <value>org.apache.oozie.example.SampleReducer</value>
+ <value>org.apache.hadoop.mapred.lib.IdentityReducer</value>
</property>
<property>
<name>mapred.map.tasks</name>
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4133c963/falcon-regression/merlin/src/test/resources/workflows/aggregatorNoOutput/lib/oozie-examples-3.1.5.jar
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/resources/workflows/aggregatorNoOutput/lib/oozie-examples-3.1.5.jar b/falcon-regression/merlin/src/test/resources/workflows/aggregatorNoOutput/lib/oozie-examples-3.1.5.jar
deleted file mode 100644
index 380c6f3..0000000
Binary files a/falcon-regression/merlin/src/test/resources/workflows/aggregatorNoOutput/lib/oozie-examples-3.1.5.jar and /dev/null differ