You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@falcon.apache.org by sr...@apache.org on 2013/11/12 12:05:20 UTC
[03/12] FALCON-85 Hive (HCatalog) integration. Contributed by
Venkatesh Seetharam FALCON-163 Merge FALCON-85 branch into main line.
Contributed by Venkatesh Seetharam
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/17f901a6/webapp/src/test/java/org/apache/falcon/process/TableStorageProcessIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/falcon/process/TableStorageProcessIT.java b/webapp/src/test/java/org/apache/falcon/process/TableStorageProcessIT.java
new file mode 100644
index 0000000..2d539c2
--- /dev/null
+++ b/webapp/src/test/java/org/apache/falcon/process/TableStorageProcessIT.java
@@ -0,0 +1,211 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.falcon.process;
+
+import org.apache.falcon.entity.ClusterHelper;
+import org.apache.falcon.entity.v0.cluster.Cluster;
+import org.apache.falcon.entity.v0.cluster.Interfacetype;
+import org.apache.falcon.resource.APIResult;
+import org.apache.falcon.resource.InstancesResult;
+import org.apache.falcon.resource.TestContext;
+import org.apache.falcon.util.FSUtils;
+import org.apache.falcon.util.HiveTestUtils;
+import org.apache.falcon.util.OozieTestUtils;
+import org.apache.falcon.util.StartupProperties;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hcatalog.api.HCatPartition;
+import org.apache.oozie.client.OozieClient;
+import org.apache.oozie.client.WorkflowJob;
+import org.testng.Assert;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+import javax.ws.rs.core.MediaType;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Integration tests for Processing Engines, Pig & Hive with both FS and table storage.
+ *
+ * This test is disabled as it heavily depends on oozie sharelibs for
+ * pig and hcatalog being made available on HDFS. captured in FALCON-139.
+ */
+@Test (enabled = false)
+public class TableStorageProcessIT {
+
+ private static final String DATABASE_NAME = "falcon_db";
+ private static final String IN_TABLE_NAME = "input_table";
+ private static final String OUT_TABLE_NAME = "output_table";
+ private static final String PARTITION_VALUE = "2012-04-21-00"; // ${YEAR}-${MONTH}-${DAY}-${HOUR}
+ private static final String CLUSTER_TEMPLATE = "/table/primary-cluster.xml";
+
+ private final TestContext context = new TestContext();
+ private Map<String, String> overlay;
+ private String metastoreUrl;
+
+ @BeforeClass
+ public void prepare() throws Exception {
+ TestContext.prepare(CLUSTER_TEMPLATE);
+
+ overlay = context.getUniqueOverlay();
+ String filePath = context.overlayParametersOverTemplate(CLUSTER_TEMPLATE, overlay);
+ context.setCluster(filePath);
+
+ final Cluster cluster = context.getCluster().getCluster();
+ final String storageUrl = ClusterHelper.getStorageUrl(cluster);
+ metastoreUrl = ClusterHelper.getInterface(cluster, Interfacetype.REGISTRY).getEndpoint();
+
+ copyDataAndScriptsToHDFS(storageUrl);
+ copyLibsToHDFS(cluster, storageUrl);
+
+ setupHiveMetastore(storageUrl);
+ scheduleFeeds();
+ }
+
+ private void copyDataAndScriptsToHDFS(String storageUrl) throws IOException {
+ // copyTestDataToHDFS
+ FSUtils.copyResourceToHDFS(
+ "/apps/data/data.txt", "data.txt", storageUrl + "/falcon/test/input/" + PARTITION_VALUE);
+
+ // copyPigScriptToHDFS
+ FSUtils.copyResourceToHDFS(
+ "/apps/pig/table-id.pig", "table-id.pig", storageUrl + "/falcon/test/apps/pig");
+
+ // copyHiveScriptToHDFS
+ FSUtils.copyResourceToHDFS(
+ "/apps/hive/script.hql", "script.hql", storageUrl + "/falcon/test/apps/hive");
+ }
+
+ private void copyLibsToHDFS(Cluster cluster, String storageUrl) throws IOException {
+ // set up kahadb to be sent as part of workflows
+ StartupProperties.get().setProperty("libext.paths", "./target/libext");
+ String libext = ClusterHelper.getLocation(cluster, "working") + "/libext";
+ FSUtils.copyOozieShareLibsToHDFS("./target/libext", storageUrl + libext);
+ }
+
+ private void setupHiveMetastore(String storageUrl) throws Exception {
+ HiveTestUtils.createDatabase(metastoreUrl, DATABASE_NAME);
+ final List<String> partitionKeys = Arrays.asList("ds");
+ HiveTestUtils.createTable(metastoreUrl, DATABASE_NAME, IN_TABLE_NAME, partitionKeys);
+ final String sourcePath = storageUrl + "/falcon/test/input/" + PARTITION_VALUE;
+ HiveTestUtils.loadData(metastoreUrl, DATABASE_NAME, IN_TABLE_NAME, sourcePath, PARTITION_VALUE);
+
+ HiveTestUtils.createTable(metastoreUrl, DATABASE_NAME, OUT_TABLE_NAME, partitionKeys);
+ }
+
+ private void scheduleFeeds() throws Exception {
+ overlay.put("cluster", "primary-cluster");
+
+ String filePath = context.overlayParametersOverTemplate(CLUSTER_TEMPLATE, overlay);
+ Assert.assertEquals(0, TestContext.executeWithURL("entity -submit -type cluster -file " + filePath));
+
+ filePath = context.overlayParametersOverTemplate("/table/table-feed-input.xml", overlay);
+ Assert.assertEquals(0,
+ TestContext.executeWithURL("entity -submitAndSchedule -type feed -file " + filePath));
+
+ filePath = context.overlayParametersOverTemplate("/table/table-feed-output.xml", overlay);
+ Assert.assertEquals(0,
+ TestContext.executeWithURL("entity -submitAndSchedule -type feed -file " + filePath));
+ }
+
+ @AfterClass
+ public void tearDown() throws Exception {
+ HiveTestUtils.dropDatabase(metastoreUrl, DATABASE_NAME);
+
+ cleanupFS(context.getCluster().getCluster());
+
+ TestContext.executeWithURL("entity -delete -type feed -name output-table");
+ TestContext.executeWithURL("entity -delete -type feed -name input-table");
+ TestContext.executeWithURL("entity -delete -type cluster -name primary-cluster");
+ }
+
+ private void cleanupFS(Cluster cluster) throws IOException {
+ FileSystem fs = FileSystem.get(ClusterHelper.getConfiguration(cluster));
+ fs.delete(new Path("/falcon/test/input/" + PARTITION_VALUE), true);
+ fs.delete(new Path("/apps/data"), true);
+ fs.delete(new Path("/apps/pig"), true);
+ fs.delete(new Path("/apps/hive"), true);
+ }
+
+ @AfterMethod
+ private void cleanHiveMetastore() throws Exception {
+ // clean up the output table for next test
+ HiveTestUtils.dropTable(metastoreUrl, DATABASE_NAME, OUT_TABLE_NAME);
+ final List<String> partitionKeys = Arrays.asList("ds");
+ HiveTestUtils.createTable(metastoreUrl, DATABASE_NAME, OUT_TABLE_NAME, partitionKeys);
+ }
+
+ @Test (enabled = false)
+ public void testSubmitAndSchedulePigProcessForTableStorage() throws Exception {
+ final String pigProcessName = "pig-tables-" + context.getProcessName();
+ overlay.put("processName", pigProcessName);
+
+ String filePath = context.overlayParametersOverTemplate("/table/pig-process-tables.xml", overlay);
+ Assert.assertEquals(0,
+ TestContext.executeWithURL("entity -submitAndSchedule -type process -file " + filePath));
+
+ WorkflowJob jobInfo = OozieTestUtils.getWorkflowJob(context.getCluster().getCluster(),
+ OozieClient.FILTER_NAME + "=FALCON_PROCESS_DEFAULT_" + pigProcessName);
+ Assert.assertEquals(WorkflowJob.Status.SUCCEEDED, jobInfo.getStatus());
+
+ HCatPartition partition = HiveTestUtils.getPartition(
+ metastoreUrl, DATABASE_NAME, OUT_TABLE_NAME, "ds", PARTITION_VALUE);
+ Assert.assertTrue(partition != null);
+
+ InstancesResult response = context.getService().path("api/instance/running/process/" + pigProcessName)
+ .header("Remote-User", "guest")
+ .accept(MediaType.APPLICATION_JSON)
+ .get(InstancesResult.class);
+ Assert.assertEquals(APIResult.Status.SUCCEEDED, response.getStatus());
+
+ TestContext.executeWithURL("entity -delete -type process -name " + pigProcessName);
+ }
+
+
+ @Test (enabled = false)
+ public void testSubmitAndScheduleHiveProcess() throws Exception {
+ final String hiveProcessName = "hive-tables-" + context.getProcessName();
+ overlay.put("processName", hiveProcessName);
+
+ String filePath = context.overlayParametersOverTemplate("/table/hive-process-template.xml", overlay);
+ Assert.assertEquals(0,
+ TestContext.executeWithURL("entity -submitAndSchedule -type process -file " + filePath));
+
+ WorkflowJob jobInfo = OozieTestUtils.getWorkflowJob(context.getCluster().getCluster(),
+ OozieClient.FILTER_NAME + "=FALCON_PROCESS_DEFAULT_" + hiveProcessName);
+ Assert.assertEquals(WorkflowJob.Status.SUCCEEDED, jobInfo.getStatus());
+
+ HCatPartition partition = HiveTestUtils.getPartition(
+ metastoreUrl, DATABASE_NAME, OUT_TABLE_NAME, "ds", PARTITION_VALUE);
+ Assert.assertTrue(partition != null);
+
+ InstancesResult response = context.getService().path("api/instance/running/process/" + hiveProcessName)
+ .header("Remote-User", "guest")
+ .accept(MediaType.APPLICATION_JSON)
+ .get(InstancesResult.class);
+ Assert.assertEquals(APIResult.Status.SUCCEEDED, response.getStatus());
+
+ TestContext.executeWithURL("entity -delete -type process -name " + hiveProcessName);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/17f901a6/webapp/src/test/java/org/apache/falcon/resource/ClusterEntityValidationIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/falcon/resource/ClusterEntityValidationIT.java b/webapp/src/test/java/org/apache/falcon/resource/ClusterEntityValidationIT.java
deleted file mode 100644
index 8f2b6e4..0000000
--- a/webapp/src/test/java/org/apache/falcon/resource/ClusterEntityValidationIT.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.falcon.resource;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.InputStream;
-import java.util.Map;
-
-import org.apache.falcon.entity.ClusterHelper;
-import org.apache.falcon.entity.v0.EntityType;
-import org.apache.falcon.entity.v0.cluster.Cluster;
-import org.apache.falcon.entity.v0.cluster.Interface;
-import org.apache.falcon.entity.v0.cluster.Interfacetype;
-import org.testng.Assert;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.DataProvider;
-import org.testng.annotations.Test;
-
-import com.sun.jersey.api.client.ClientResponse;
-
-/**
- * Tests cluster entity validation to verify if each of the specified
- * interface endpoints are valid.
- */
-public class ClusterEntityValidationIT {
- private final TestContext context = new TestContext();
- private Map<String, String> overlay;
-
-
- @BeforeClass
- public void setup() throws Exception {
- TestContext.prepare();
- }
-
- /**
- * Positive test.
- *
- * @throws Exception
- */
- @Test
- public void testClusterEntityWithValidInterfaces() throws Exception {
- overlay = context.getUniqueOverlay();
- overlay.put("colo", "default");
- ClientResponse response = context.submitToFalcon(TestContext.CLUSTER_TEMPLATE, overlay, EntityType.CLUSTER);
- context.assertSuccessful(response);
- }
-
-
- @DataProvider(name = "interfaceToInvalidURLs")
- public Object[][] createInterfaceToInvalidURLData() {
- return new Object[][] {
- // TODO FileSystem validates invalid hftp url, does NOT fail
- // {Interfacetype.READONLY, "hftp://localhost:41119"},
- {Interfacetype.READONLY, ""},
- {Interfacetype.READONLY, "localhost:41119"},
- {Interfacetype.WRITE, "write-interface:9999"},
- {Interfacetype.WRITE, "hdfs://write-interface:9999"},
- {Interfacetype.EXECUTE, "execute-interface:9999"},
- {Interfacetype.WORKFLOW, "workflow-interface:9999/oozie/"},
- {Interfacetype.WORKFLOW, "http://workflow-interface:9999/oozie/"},
- {Interfacetype.MESSAGING, "messaging-interface:9999"},
- {Interfacetype.MESSAGING, "tcp://messaging-interface:9999"},
- };
- }
-
- @Test (dataProvider = "interfaceToInvalidURLs")
- public void testClusterEntityWithInvalidInterfaces(Interfacetype interfacetype, String endpoint)
- throws Exception {
- overlay = context.getUniqueOverlay();
- String filePath = context.overlayParametersOverTemplate(TestContext.CLUSTER_TEMPLATE, overlay);
- InputStream stream = new FileInputStream(filePath);
- Cluster cluster = (Cluster) EntityType.CLUSTER.getUnmarshaller().unmarshal(stream);
- Assert.assertNotNull(cluster);
- cluster.setColo("default"); // validations will be ignored if not default & tests fail
-
- Interface anInterface = ClusterHelper.getInterface(cluster, interfacetype);
- anInterface.setEndpoint(endpoint);
-
- File tmpFile = context.getTempFile();
- EntityType.CLUSTER.getMarshaller().marshal(cluster, tmpFile);
- ClientResponse response = context.submitFileToFalcon(EntityType.CLUSTER, tmpFile.getAbsolutePath());
- context.assertFailure(response);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/17f901a6/webapp/src/test/java/org/apache/falcon/resource/PigProcessIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/falcon/resource/PigProcessIT.java b/webapp/src/test/java/org/apache/falcon/resource/PigProcessIT.java
deleted file mode 100644
index 1cb5825..0000000
--- a/webapp/src/test/java/org/apache/falcon/resource/PigProcessIT.java
+++ /dev/null
@@ -1,119 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.falcon.resource;
-
-import org.apache.falcon.entity.ClusterHelper;
-import org.testng.Assert;
-import org.testng.annotations.AfterClass;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Test;
-
-import java.io.IOException;
-import java.util.Map;
-
-/**
- * Integration tests for Pig Processing Engine.
- */
-@Test
-public class PigProcessIT {
-
- private static final String SHARE_LIB_HDFS_PATH = "/user/oozie/share/lib";
-
- private final TestContext context = new TestContext();
- private Map<String, String> overlay;
-
-
- @BeforeClass
- public void prepare() throws Exception {
- TestContext.prepare();
-
- overlay = context.getUniqueOverlay();
-
- String filePath = context.overlayParametersOverTemplate(TestContext.CLUSTER_TEMPLATE, overlay);
- context.setCluster(filePath);
-
- String storageUrl = ClusterHelper.getStorageUrl(context.getCluster().getCluster());
- System.out.println("nn = " + storageUrl);
- TestContext.copyOozieShareLibsToHDFS("./target/sharelib", storageUrl + SHARE_LIB_HDFS_PATH);
-
- // copyPigScriptToHDFS
- TestContext.copyResourceToHDFS(
- "/apps/pig/id.pig", "id.pig", storageUrl + "/falcon/test/apps/pig");
- // copyTestDataToHDFS
- TestContext.copyResourceToHDFS(
- "/apps/pig/data.txt", "data.txt", storageUrl + "/falcon/test/input/2012/04/21/00");
- }
-
- @AfterClass
- public void tearDown() throws IOException {
- TestContext.deleteOozieShareLibsFromHDFS(SHARE_LIB_HDFS_PATH);
- }
-
- @Test
- public void testSubmitAndSchedulePigProcess() throws Exception {
-
- Thread.sleep(5000);
-
- String filePath = context.overlayParametersOverTemplate(TestContext.CLUSTER_TEMPLATE, overlay);
- Assert.assertEquals(0, TestContext.executeWithURL("entity -submit -type cluster -file " + filePath));
- // context.setCluster(filePath);
-
- filePath = context.overlayParametersOverTemplate(TestContext.FEED_TEMPLATE1, overlay);
- Assert.assertEquals(0,
- TestContext.executeWithURL("entity -submitAndSchedule -type feed -file " + filePath));
-
- filePath = context.overlayParametersOverTemplate(TestContext.FEED_TEMPLATE2, overlay);
- Assert.assertEquals(0,
- TestContext.executeWithURL("entity -submitAndSchedule -type feed -file " + filePath));
-
- filePath = context.overlayParametersOverTemplate(TestContext.FEED_TEMPLATE1, overlay);
- Assert.assertEquals(0,
- TestContext.executeWithURL("entity -submit -type feed -file " + filePath));
-
- filePath = context.overlayParametersOverTemplate(TestContext.FEED_TEMPLATE2, overlay);
- Assert.assertEquals(0,
- TestContext.executeWithURL("entity -submit -type feed -file " + filePath));
-
- final String pigProcessName = "pig-" + context.getProcessName();
- overlay.put("processName", pigProcessName);
-
- filePath = context.overlayParametersOverTemplate(TestContext.PIG_PROCESS_TEMPLATE, overlay);
- Assert.assertEquals(0,
- TestContext.executeWithURL("entity -submitAndSchedule -type process -file " + filePath));
-
-/*
- WorkflowJob jobInfo = context.getWorkflowJob(
- OozieClient.FILTER_NAME + "=FALCON_PROCESS_DEFAULT_" + pigProcessName);
- Assert.assertEquals(WorkflowJob.Status.SUCCEEDED, jobInfo.getStatus());
-
- InstancesResult response = context.service.path("api/instance/running/process/" + pigProcessName)
- .header("Remote-User", "guest").accept(MediaType.APPLICATION_JSON).get(InstancesResult.class);
- org.testng.Assert.assertEquals(APIResult.Status.SUCCEEDED, response.getStatus());
- org.testng.Assert.assertNotNull(response.getInstances());
- org.testng.Assert.assertEquals(1, response.getInstances().length);
-
- // verify LogMover
- Path oozieLogPath = context.getOozieLogPath(jobInfo);
- System.out.println("oozieLogPath = " + oozieLogPath);
- Assert.assertTrue(context.getCluster().getFileSystem().exists(oozieLogPath));
-*/
-
- TestContext.executeWithURL("entity -delete -type process -name " + pigProcessName);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/17f901a6/webapp/src/test/java/org/apache/falcon/resource/TestContext.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/falcon/resource/TestContext.java b/webapp/src/test/java/org/apache/falcon/resource/TestContext.java
index af6f22c..f246948 100644
--- a/webapp/src/test/java/org/apache/falcon/resource/TestContext.java
+++ b/webapp/src/test/java/org/apache/falcon/resource/TestContext.java
@@ -23,7 +23,6 @@ import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.api.client.config.ClientConfig;
import com.sun.jersey.api.client.config.DefaultClientConfig;
-import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.RandomStringUtils;
import org.apache.falcon.FalconException;
import org.apache.falcon.cli.FalconCLI;
@@ -36,7 +35,6 @@ import org.apache.falcon.entity.v0.cluster.Cluster;
import org.apache.falcon.entity.v0.feed.Feed;
import org.apache.falcon.util.StartupProperties;
import org.apache.falcon.workflow.engine.OozieClientFactory;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
@@ -58,7 +56,6 @@ import javax.xml.bind.Marshaller;
import javax.xml.bind.Unmarshaller;
import java.io.BufferedReader;
import java.io.File;
-import java.io.FileFilter;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FileReader;
@@ -395,51 +392,6 @@ public class TestContext {
return false;
}
- /*
- public WorkflowJob getWorkflowJob(String filter) throws Exception {
- OozieClient ozClient = OozieClientFactory.get(cluster.getCluster());
-
- List<WorkflowJob> jobs;
- while (true) {
- jobs = ozClient.getJobsInfo(filter);
- System.out.println("jobs = " + jobs);
- if (jobs.size() > 0) {
- break;
- } else {
- Thread.sleep(1000);
- }
- }
-
- WorkflowJob jobInfo = jobs.get(0);
- while (true) {
- if (!(jobInfo.getStatus() == WorkflowJob.Status.RUNNING
- || jobInfo.getStatus() == WorkflowJob.Status.PREP)) {
- break;
- } else {
- Thread.sleep(1000);
- jobInfo = ozClient.getJobInfo(jobInfo.getId());
- System.out.println("jobInfo = " + jobInfo);
- }
- }
-
- return jobInfo;
- }
-
- public Path getOozieLogPath(WorkflowJob jobInfo) throws Exception {
- Path stagingPath = new Path(ClusterHelper.getLocation(cluster.getCluster(), "staging"),
- EntityUtil.getStagingPath(cluster.getCluster()) + "/../logs");
- final Path logPath = new Path(ClusterHelper.getStorageUrl(cluster.getCluster()), stagingPath);
- LogMover.main(new String[]{"-workflowEngineUrl",
- ClusterHelper.getOozieUrl(cluster.getCluster()),
- "-subflowId", jobInfo.getId(), "-runId", "1",
- "-logDir", logPath.toString() + "/job-2012-04-21-00-00",
- "-status", "SUCCEEDED", "-entityType", "process",
- "-userWorkflowEngine", "pig",});
-
- return new Path(logPath, "job-2012-04-21-00-00/001/oozie.log");
- }
- */
-
public Map<String, String> getUniqueOverlay() throws FalconException {
Map<String, String> overlay = new HashMap<String, String>();
long time = System.currentTimeMillis();
@@ -458,13 +410,16 @@ public class TestContext {
}
public static void prepare() throws Exception {
+ prepare(TestContext.CLUSTER_TEMPLATE);
+ }
+
+ public static void prepare(String clusterTemplate) throws Exception {
Map<String, String> overlay = new HashMap<String, String>();
overlay.put("cluster", RandomStringUtils.randomAlphabetic(5));
overlay.put("colo", "gs");
TestContext context = new TestContext();
- String file = context.
- overlayParametersOverTemplate(TestContext.CLUSTER_TEMPLATE, overlay);
+ String file = context.overlayParametersOverTemplate(clusterTemplate, overlay);
EmbeddedCluster cluster = StandAloneCluster.newCluster(file);
cleanupStore();
@@ -477,7 +432,8 @@ public class TestContext {
fs.delete(wfParent, true);
Path wfPath = new Path(wfParent, "workflow");
fs.mkdirs(wfPath);
- fs.copyFromLocalFile(false, true, new Path(TestContext.class.getResource("/fs-workflow.xml").getPath()),
+ fs.copyFromLocalFile(false, true,
+ new Path(TestContext.class.getResource("/fs-workflow.xml").getPath()),
new Path(wfPath, "workflow.xml"));
fs.mkdirs(new Path(wfParent, "input/2012/04/20/00"));
Path outPath = new Path(wfParent, "output");
@@ -493,73 +449,6 @@ public class TestContext {
}
}
- public static void copyOozieShareLibsToHDFS(String shareLibLocalPath, String shareLibHdfsPath)
- throws IOException {
- File shareLibDir = new File(shareLibLocalPath);
- if (!shareLibDir.exists()) {
- throw new IllegalArgumentException("Sharelibs dir must exist for tests to run.");
- }
-
- File[] jarFiles = shareLibDir.listFiles(new FileFilter() {
- @Override
- public boolean accept(File file) {
- return file.isFile() && file.getName().endsWith(".jar");
- }
- });
-
- for (File jarFile : jarFiles) {
- copyFileToHDFS(jarFile, shareLibHdfsPath);
- }
- }
-
- public static void copyFileToHDFS(File jarFile, String shareLibHdfsPath) throws IOException {
- System.out.println("Copying jarFile = " + jarFile);
- Path shareLibPath = new Path(shareLibHdfsPath);
- FileSystem fs = shareLibPath.getFileSystem(new Configuration());
- if (!fs.exists(shareLibPath)) {
- fs.mkdirs(shareLibPath);
- }
-
- OutputStream os = null;
- InputStream is = null;
- try {
- os = fs.create(new Path(shareLibPath, jarFile.getName()));
- is = new FileInputStream(jarFile);
- IOUtils.copy(is, os);
- } finally {
- IOUtils.closeQuietly(is);
- IOUtils.closeQuietly(os);
- }
- }
-
- public static void copyResourceToHDFS(String localResource, String resourceName, String hdfsPath)
- throws IOException {
- Path appPath = new Path(hdfsPath);
- FileSystem fs = appPath.getFileSystem(new Configuration());
- if (!fs.exists(appPath)) {
- fs.mkdirs(appPath);
- }
-
- OutputStream os = null;
- InputStream is = null;
- try {
- os = fs.create(new Path(appPath, resourceName));
- is = TestContext.class.getResourceAsStream(localResource);
- IOUtils.copy(is, os);
- } finally {
- IOUtils.closeQuietly(is);
- IOUtils.closeQuietly(os);
- }
- }
-
- public static void deleteOozieShareLibsFromHDFS(String shareLibHdfsPath) throws IOException {
- Path shareLibPath = new Path(shareLibHdfsPath);
- FileSystem fs = shareLibPath.getFileSystem(new Configuration());
- if (fs.exists(shareLibPath)) {
- fs.delete(shareLibPath, true);
- }
- }
-
public static int executeWithURL(String command) throws Exception {
return new FalconCLI().run((command + " -url " + TestContext.BASE_URL).split("\\s+"));
}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/17f901a6/webapp/src/test/java/org/apache/falcon/util/FSUtils.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/falcon/util/FSUtils.java b/webapp/src/test/java/org/apache/falcon/util/FSUtils.java
new file mode 100644
index 0000000..2db659d
--- /dev/null
+++ b/webapp/src/test/java/org/apache/falcon/util/FSUtils.java
@@ -0,0 +1,101 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.falcon.util;
+
+
+import org.apache.commons.io.IOUtils;
+import org.apache.falcon.resource.TestContext;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+import java.io.File;
+import java.io.FileFilter;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+/**
+ * File System Utility class for integration-tests.
+ */
+public final class FSUtils {
+
+ private FSUtils() {
+ }
+
+ public static void copyOozieShareLibsToHDFS(String shareLibLocalPath, String shareLibHdfsPath)
+ throws IOException {
+ File shareLibDir = new File(shareLibLocalPath);
+ if (!shareLibDir.exists()) {
+ throw new IllegalArgumentException("Sharelibs dir must exist for tests to run.");
+ }
+
+ File[] jarFiles = shareLibDir.listFiles(new FileFilter() {
+ @Override
+ public boolean accept(File file) {
+ return file.isFile() && file.getName().endsWith(".jar");
+ }
+ });
+
+ for (File jarFile : jarFiles) {
+ copyFileToHDFS(jarFile, shareLibHdfsPath);
+ }
+ }
+
+ public static void copyFileToHDFS(File jarFile, String shareLibHdfsPath) throws IOException {
+ System.out.println("Copying jarFile = " + jarFile);
+ Path shareLibPath = new Path(shareLibHdfsPath);
+ FileSystem fs = shareLibPath.getFileSystem(new Configuration());
+ if (!fs.exists(shareLibPath)) {
+ fs.mkdirs(shareLibPath);
+ }
+
+ OutputStream os = null;
+ InputStream is = null;
+ try {
+ os = fs.create(new Path(shareLibPath, jarFile.getName()));
+ is = new FileInputStream(jarFile);
+ IOUtils.copy(is, os);
+ } finally {
+ IOUtils.closeQuietly(is);
+ IOUtils.closeQuietly(os);
+ }
+ }
+
+ public static void copyResourceToHDFS(String localResource, String resourceName, String hdfsPath)
+ throws IOException {
+ Path appPath = new Path(hdfsPath);
+ FileSystem fs = appPath.getFileSystem(new Configuration());
+ if (!fs.exists(appPath)) {
+ fs.mkdirs(appPath);
+ }
+
+ OutputStream os = null;
+ InputStream is = null;
+ try {
+ os = fs.create(new Path(appPath, resourceName));
+ is = TestContext.class.getResourceAsStream(localResource);
+ IOUtils.copy(is, os);
+ } finally {
+ IOUtils.closeQuietly(is);
+ IOUtils.closeQuietly(os);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/17f901a6/webapp/src/test/java/org/apache/falcon/util/HiveTestUtils.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/falcon/util/HiveTestUtils.java b/webapp/src/test/java/org/apache/falcon/util/HiveTestUtils.java
new file mode 100644
index 0000000..5087e20
--- /dev/null
+++ b/webapp/src/test/java/org/apache/falcon/util/HiveTestUtils.java
@@ -0,0 +1,187 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.falcon.util;
+
+import org.apache.falcon.catalog.HiveCatalogService;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.Driver;
+import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hcatalog.api.HCatClient;
+import org.apache.hcatalog.api.HCatCreateDBDesc;
+import org.apache.hcatalog.api.HCatCreateTableDesc;
+import org.apache.hcatalog.api.HCatPartition;
+import org.apache.hcatalog.data.schema.HCatFieldSchema;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Hive Utility class for integration-tests.
+ */
+public final class HiveTestUtils {
+
+ private HiveTestUtils() {
+ }
+
+ public static void createDatabase(String metaStoreUrl,
+ String databaseName) throws Exception {
+ HCatClient client = HiveCatalogService.get(metaStoreUrl);
+ HCatCreateDBDesc dbDesc = HCatCreateDBDesc.create(databaseName)
+ .ifNotExists(true).build();
+ client.createDatabase(dbDesc);
+ }
+
+ public static void dropDatabase(String metaStoreUrl, String databaseName) throws Exception {
+ HCatClient client = HiveCatalogService.get(metaStoreUrl);
+ client.dropDatabase(databaseName, true, HCatClient.DropDBMode.CASCADE);
+ }
+
+ public static void createTable(String metaStoreUrl, String databaseName,
+ String tableName) throws Exception {
+ HCatClient client = HiveCatalogService.get(metaStoreUrl);
+ ArrayList<HCatFieldSchema> cols = new ArrayList<HCatFieldSchema>();
+ cols.add(new HCatFieldSchema("id", HCatFieldSchema.Type.INT, "id comment"));
+ cols.add(new HCatFieldSchema("value", HCatFieldSchema.Type.STRING, "value comment"));
+
+ HCatCreateTableDesc tableDesc = HCatCreateTableDesc
+ .create(databaseName, tableName, cols)
+ .ifNotExists(true)
+ .comments("falcon integration test")
+ .build();
+ client.createTable(tableDesc);
+ }
+
+ public static void createTable(String metaStoreUrl, String databaseName, String tableName,
+ List<String> partitionKeys) throws Exception {
+ HCatClient client = HiveCatalogService.get(metaStoreUrl);
+ ArrayList<HCatFieldSchema> cols = new ArrayList<HCatFieldSchema>();
+ cols.add(new HCatFieldSchema("id", HCatFieldSchema.Type.INT, "id comment"));
+ cols.add(new HCatFieldSchema("value", HCatFieldSchema.Type.STRING, "value comment"));
+
+ List<HCatFieldSchema> partitionSchema = new ArrayList<HCatFieldSchema>();
+ for (String partitionKey : partitionKeys) {
+ partitionSchema.add(new HCatFieldSchema(partitionKey, HCatFieldSchema.Type.STRING, ""));
+ }
+
+ HCatCreateTableDesc tableDesc = HCatCreateTableDesc
+ .create(databaseName, tableName, cols)
+ .ifNotExists(true)
+ .comments("falcon integration test")
+ .partCols(new ArrayList<HCatFieldSchema>(partitionSchema))
+ .build();
+ client.createTable(tableDesc);
+ }
+
+ public static void createExternalTable(String metaStoreUrl, String databaseName, String tableName,
+ List<String> partitionKeys, String externalLocation) throws Exception {
+ ArrayList<HCatFieldSchema> cols = new ArrayList<HCatFieldSchema>();
+ cols.add(new HCatFieldSchema("id", HCatFieldSchema.Type.INT, "id comment"));
+ cols.add(new HCatFieldSchema("value", HCatFieldSchema.Type.STRING, "value comment"));
+
+ List<HCatFieldSchema> partitionSchema = new ArrayList<HCatFieldSchema>();
+ for (String partitionKey : partitionKeys) {
+ partitionSchema.add(new HCatFieldSchema(partitionKey, HCatFieldSchema.Type.STRING, ""));
+ }
+
+ HCatCreateTableDesc tableDesc = HCatCreateTableDesc
+ .create(databaseName, tableName, cols)
+ .fileFormat("rcfile")
+ .ifNotExists(true)
+ .comments("falcon integration test")
+ .partCols(new ArrayList<HCatFieldSchema>(partitionSchema))
+ .isTableExternal(true)
+ .location(externalLocation)
+ .build();
+
+ HCatClient client = HiveCatalogService.get(metaStoreUrl);
+ client.createTable(tableDesc);
+ }
+
+ public static void alterTable(String metaStoreUrl, String databaseName,
+ String tableName) throws Exception {
+ StringBuilder alterTableDdl = new StringBuilder();
+ alterTableDdl
+ .append(" alter table ")
+ .append(tableName)
+ .append(" set fileformat ")
+ .append(" inputformat 'org.apache.hadoop.mapred.TextInputFormat' ")
+ .append(" outputformat 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'");
+
+ startSessionState(metaStoreUrl);
+ execHiveDDL("use " + databaseName);
+ execHiveDDL(alterTableDdl.toString());
+ }
+
+ public static void loadData(String metaStoreUrl, String databaseName, String tableName,
+ String path, String partition) throws Exception {
+ StringBuilder ddl = new StringBuilder();
+ ddl.append(" load data inpath ")
+ .append(" '").append(path).append("' ")
+ .append(" into table ")
+ .append(tableName)
+ .append(" partition ").append(" (ds='").append(partition).append("') ");
+
+ startSessionState(metaStoreUrl);
+ execHiveDDL("use " + databaseName);
+ execHiveDDL(ddl.toString());
+ }
+
+ public static void dropTable(String metaStoreUrl, String databaseName,
+ String tableName) throws Exception {
+ HCatClient client = HiveCatalogService.get(metaStoreUrl);
+ client.dropTable(databaseName, tableName, true);
+ }
+
+ public static void startSessionState(String metaStoreUrl) {
+ HiveConf hcatConf = new HiveConf();
+ hcatConf.setVar(HiveConf.ConfVars.METASTOREURIS, metaStoreUrl);
+ hcatConf.set("hive.metastore.local", "false");
+ hcatConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false");
+ hcatConf.set("hive.root.logger", "DEBUG,console");
+ SessionState.start(hcatConf);
+ }
+
+ public static void execHiveDDL(String ddl) throws Exception {
+ System.out.println("Executing ddl = " + ddl);
+
+ Driver hiveDriver = new Driver();
+ CommandProcessorResponse response = hiveDriver.run(ddl);
+
+ System.out.println("response = " + response);
+ System.out.println("response.getResponseCode() = " + response.getResponseCode());
+ System.out.println("response.getErrorMessage() = " + response.getErrorMessage());
+ System.out.println("response.getSQLState() = " + response.getSQLState());
+
+ if (response.getResponseCode() > 0) {
+ throw new Exception(response.getErrorMessage());
+ }
+ }
+
+ public static HCatPartition getPartition(String metastoreUrl, String databaseName,
+ String tableName, String partitionKey,
+ String partitionValue) throws Exception {
+ Map<String, String> partitionSpec = new HashMap<String, String>();
+ partitionSpec.put(partitionKey, partitionValue);
+
+ return HiveCatalogService.get(metastoreUrl).getPartition(databaseName, tableName, partitionSpec);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/17f901a6/webapp/src/test/java/org/apache/falcon/util/OozieTestUtils.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/falcon/util/OozieTestUtils.java b/webapp/src/test/java/org/apache/falcon/util/OozieTestUtils.java
new file mode 100644
index 0000000..690fb6b
--- /dev/null
+++ b/webapp/src/test/java/org/apache/falcon/util/OozieTestUtils.java
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.falcon.util;
+
+import org.apache.falcon.entity.ClusterHelper;
+import org.apache.falcon.entity.EntityUtil;
+import org.apache.falcon.entity.v0.cluster.Cluster;
+import org.apache.falcon.logging.LogMover;
+import org.apache.falcon.workflow.engine.OozieClientFactory;
+import org.apache.hadoop.fs.Path;
+import org.apache.oozie.client.OozieClient;
+import org.apache.oozie.client.WorkflowJob;
+
+import java.util.List;
+
+/**
+ * Oozie Utility class for integration-tests.
+ */
+public final class OozieTestUtils {
+
+ private OozieTestUtils() {
+ }
+
+ public static WorkflowJob getWorkflowJob(Cluster cluster, String filter) throws Exception {
+ OozieClient ozClient = OozieClientFactory.get(cluster);
+
+ List<WorkflowJob> jobs;
+ while (true) {
+ jobs = ozClient.getJobsInfo(filter);
+ System.out.println("jobs = " + jobs);
+ if (jobs.size() > 0) {
+ break;
+ } else {
+ Thread.sleep(1000);
+ }
+ }
+
+ WorkflowJob jobInfo = jobs.get(0);
+ while (true) {
+ if (!(jobInfo.getStatus() == WorkflowJob.Status.RUNNING
+ || jobInfo.getStatus() == WorkflowJob.Status.PREP)) {
+ break;
+ } else {
+ Thread.sleep(1000);
+ jobInfo = ozClient.getJobInfo(jobInfo.getId());
+ System.out.println("jobInfo = " + jobInfo);
+ }
+ }
+
+ return jobInfo;
+ }
+
+ public static Path getOozieLogPath(Cluster cluster, WorkflowJob jobInfo) throws Exception {
+
+ Path stagingPath = new Path(ClusterHelper.getLocation(cluster, "staging"),
+ EntityUtil.getStagingPath(cluster) + "/../logs");
+ final Path logPath = new Path(ClusterHelper.getStorageUrl(cluster), stagingPath);
+ LogMover.main(new String[] {
+ "-workflowEngineUrl", ClusterHelper.getOozieUrl(cluster),
+ "-subflowId", jobInfo.getId(), "-runId", "1",
+ "-logDir", logPath.toString() + "/job-2012-04-21-00-00",
+ "-status", "SUCCEEDED", "-entityType", "process",
+ "-userWorkflowEngine", "pig",
+ });
+
+ return new Path(logPath, "job-2012-04-21-00-00/001/oozie.log");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/17f901a6/webapp/src/test/java/org/apache/falcon/validation/ClusterEntityValidationIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/falcon/validation/ClusterEntityValidationIT.java b/webapp/src/test/java/org/apache/falcon/validation/ClusterEntityValidationIT.java
new file mode 100644
index 0000000..9299b5b
--- /dev/null
+++ b/webapp/src/test/java/org/apache/falcon/validation/ClusterEntityValidationIT.java
@@ -0,0 +1,104 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.falcon.validation;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.InputStream;
+import java.util.Map;
+
+import org.apache.falcon.entity.ClusterHelper;
+import org.apache.falcon.entity.v0.EntityType;
+import org.apache.falcon.entity.v0.cluster.Cluster;
+import org.apache.falcon.entity.v0.cluster.Interface;
+import org.apache.falcon.entity.v0.cluster.Interfacetype;
+import org.apache.falcon.resource.TestContext;
+import org.testng.Assert;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+import com.sun.jersey.api.client.ClientResponse;
+
+/**
+ * Tests cluster entity validation to verify if each of the specified
+ * interface endpoints are valid.
+ */
+public class ClusterEntityValidationIT {
+ private final TestContext context = new TestContext();
+ private Map<String, String> overlay;
+
+
+ @BeforeClass
+ public void setup() throws Exception {
+ TestContext.prepare();
+ }
+
+ /**
+ * Positive test.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testClusterEntityWithValidInterfaces() throws Exception {
+ overlay = context.getUniqueOverlay();
+ overlay.put("colo", "default");
+ ClientResponse response = context.submitToFalcon(TestContext.CLUSTER_TEMPLATE, overlay, EntityType.CLUSTER);
+ context.assertSuccessful(response);
+ }
+
+
+ @DataProvider(name = "interfaceToInvalidURLs")
+ public Object[][] createInterfaceToInvalidURLData() {
+ return new Object[][] {
+ // TODO FileSystem validates invalid hftp url, does NOT fail
+ // {Interfacetype.READONLY, "hftp://localhost:41119"},
+ {Interfacetype.READONLY, ""},
+ {Interfacetype.READONLY, "localhost:41119"},
+ {Interfacetype.WRITE, "write-interface:9999"},
+ {Interfacetype.WRITE, "hdfs://write-interface:9999"},
+ {Interfacetype.EXECUTE, "execute-interface:9999"},
+ {Interfacetype.WORKFLOW, "workflow-interface:9999/oozie/"},
+ {Interfacetype.WORKFLOW, "http://workflow-interface:9999/oozie/"},
+ {Interfacetype.MESSAGING, "messaging-interface:9999"},
+ {Interfacetype.MESSAGING, "tcp://messaging-interface:9999"},
+ {Interfacetype.REGISTRY, "catalog-interface:9999"},
+ {Interfacetype.REGISTRY, "http://catalog-interface:9999"},
+ };
+ }
+
+ @Test (dataProvider = "interfaceToInvalidURLs")
+ public void testClusterEntityWithInvalidInterfaces(Interfacetype interfacetype, String endpoint)
+ throws Exception {
+ overlay = context.getUniqueOverlay();
+ String filePath = context.overlayParametersOverTemplate(TestContext.CLUSTER_TEMPLATE, overlay);
+ InputStream stream = new FileInputStream(filePath);
+ Cluster cluster = (Cluster) EntityType.CLUSTER.getUnmarshaller().unmarshal(stream);
+ Assert.assertNotNull(cluster);
+ cluster.setColo("default"); // validations will be ignored if not default & tests fail
+
+ Interface anInterface = ClusterHelper.getInterface(cluster, interfacetype);
+ anInterface.setEndpoint(endpoint);
+
+ File tmpFile = context.getTempFile();
+ EntityType.CLUSTER.getMarshaller().marshal(cluster, tmpFile);
+ ClientResponse response = context.submitFileToFalcon(EntityType.CLUSTER, tmpFile.getAbsolutePath());
+ context.assertFailure(response);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/17f901a6/webapp/src/test/java/org/apache/falcon/validation/FeedEntityValidationIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/falcon/validation/FeedEntityValidationIT.java b/webapp/src/test/java/org/apache/falcon/validation/FeedEntityValidationIT.java
new file mode 100644
index 0000000..db24b9a
--- /dev/null
+++ b/webapp/src/test/java/org/apache/falcon/validation/FeedEntityValidationIT.java
@@ -0,0 +1,145 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.falcon.validation;
+
+import com.sun.jersey.api.client.ClientResponse;
+import org.apache.falcon.FalconException;
+import org.apache.falcon.entity.parser.EntityParserFactory;
+import org.apache.falcon.entity.parser.FeedEntityParser;
+import org.apache.falcon.entity.v0.EntityType;
+import org.apache.falcon.entity.v0.Frequency;
+import org.apache.falcon.entity.v0.feed.Feed;
+import org.apache.falcon.entity.v0.feed.LateArrival;
+import org.apache.falcon.resource.TestContext;
+import org.apache.falcon.util.HiveTestUtils;
+import org.testng.Assert;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+import javax.xml.bind.Marshaller;
+import java.io.FileInputStream;
+import java.io.InputStream;
+import java.io.StringWriter;
+import java.util.Map;
+
+/**
+ * Tests feed entity validation to verify if the table specified is valid.
+ */
+public class FeedEntityValidationIT {
+
+ private static final String METASTORE_URL = "thrift://localhost:49083";
+ private static final String DATABASE_NAME = "falcondb";
+ private static final String TABLE_NAME = "clicks";
+ private static final String TABLE_URI =
+ "catalog:" + DATABASE_NAME + ":" + TABLE_NAME + "#ds=${YEAR}-${MONTH}-${DAY}-${HOUR}";
+
+ private final TestContext context = new TestContext();
+
+ @BeforeClass
+ public void setup() throws Exception {
+ TestContext.prepare();
+
+ HiveTestUtils.createDatabase(METASTORE_URL, DATABASE_NAME);
+ HiveTestUtils.createTable(METASTORE_URL, DATABASE_NAME, TABLE_NAME);
+ }
+
+ @AfterClass
+ public void tearDown() throws Exception {
+ HiveTestUtils.dropTable(METASTORE_URL, DATABASE_NAME, TABLE_NAME);
+ HiveTestUtils.dropDatabase(METASTORE_URL, DATABASE_NAME);
+ }
+
+ /**
+ * Positive test.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testFeedEntityWithValidTable() throws Exception {
+ Map<String, String> overlay = context.getUniqueOverlay();
+ overlay.put("colo", "default");
+
+ ClientResponse response = context.submitToFalcon(TestContext.CLUSTER_TEMPLATE, overlay, EntityType.CLUSTER);
+ context.assertSuccessful(response);
+
+ // submission will parse and validate the feed with table
+ overlay.put("tableUri", TABLE_URI);
+ response = context.submitToFalcon("/hive-table-feed.xml", overlay, EntityType.FEED);
+ context.assertSuccessful(response);
+ }
+
+ /**
+ * Late data handling test.
+ *
+ * @throws Exception
+ */
+ @Test (expectedExceptions = FalconException.class)
+ public void testFeedEntityWithValidTableAndLateArrival() throws Exception {
+ Map<String, String> overlay = context.getUniqueOverlay();
+ overlay.put("colo", "default"); // validations will be ignored if not default & tests fail
+ overlay.put("tableUri", TABLE_URI);
+
+ String filePath = context.overlayParametersOverTemplate("/hive-table-feed.xml", overlay);
+ InputStream stream = new FileInputStream(filePath);
+ FeedEntityParser parser = (FeedEntityParser) EntityParserFactory.getParser(EntityType.FEED);
+ Feed feed = parser.parse(stream);
+ Assert.assertNotNull(feed);
+
+ final LateArrival lateArrival = new LateArrival();
+ lateArrival.setCutOff(new Frequency("4", Frequency.TimeUnit.hours));
+ feed.setLateArrival(lateArrival);
+
+ StringWriter stringWriter = new StringWriter();
+ Marshaller marshaller = EntityType.FEED.getMarshaller();
+ marshaller.marshal(feed, stringWriter);
+ System.out.println(stringWriter.toString());
+ parser.parseAndValidate(stringWriter.toString());
+ }
+
+ @DataProvider(name = "invalidTableUris")
+ public Object[][] createInvalidTableUriData() {
+ return new Object[][] {
+ // does not match with group input's frequency
+ {"catalog:" + DATABASE_NAME + ":" + TABLE_NAME + "#ds=ds=${YEAR}-${MONTH}-${DAY}", ""},
+ {"catalog:" + DATABASE_NAME + ":" + TABLE_NAME + "#ds=ds=${YEAR}-${MONTH}-${DAY}", ""},
+ {"badscheme:" + DATABASE_NAME + ":" + TABLE_NAME + "#ds=ds=${YEAR}-${MONTH}-${DAY}", ""},
+ {"catalog:" + DATABASE_NAME + ":" + "badtable" + "#ds=ds=${YEAR}-${MONTH}-${DAY}", ""},
+ {"catalog:" + "baddb" + ":" + TABLE_NAME + "#ds=ds=${YEAR}-${MONTH}-${DAY}", ""},
+ {"catalog:" + "baddb" + ":" + "badtable" + "#ds=ds=${YEAR}-${MONTH}-${DAY}", ""},
+ };
+ }
+
+ @Test (dataProvider = "invalidTableUris")
+ public void testFeedEntityWithInvalidTableUri(String tableUri, @SuppressWarnings("unused") String ignore)
+ throws Exception {
+
+ Map<String, String> overlay = context.getUniqueOverlay();
+ overlay.put("colo", "default");
+
+ ClientResponse response = context.submitToFalcon(TestContext.CLUSTER_TEMPLATE, overlay, EntityType.CLUSTER);
+ context.assertSuccessful(response);
+
+ // submission will parse and validate the feed with table
+ overlay.put("tableUri", tableUri);
+ response = context.submitToFalcon("/hive-table-feed.xml", overlay, EntityType.FEED);
+ context.assertFailure(response);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/17f901a6/webapp/src/test/resources/apps/data/data.txt
----------------------------------------------------------------------
diff --git a/webapp/src/test/resources/apps/data/data.txt b/webapp/src/test/resources/apps/data/data.txt
new file mode 100644
index 0000000..fc1c491
--- /dev/null
+++ b/webapp/src/test/resources/apps/data/data.txt
@@ -0,0 +1,1000 @@
+0^A238val_238
+1^A86val_86
+2^A311val_311
+3^A27val_27
+4^A165val_165
+5^A409val_409
+6^A255val_255
+7^A278val_278
+8^A98val_98
+9^A484val_484
+10^A265val_265
+11^A193val_193
+12^A401val_401
+13^A150val_150
+14^A273val_273
+15^A224val_224
+17^A369val_369
+18^A66val_66
+19^A128val_128
+20^A213val_213
+21^A146val_146
+22^A406val_406
+23^A429val_429
+24^A374val_374
+25^A152val_152
+26^A469val_469
+27^A145val_145
+28^A495val_495
+29^A37val_37
+30^A327val_327
+31^A281val_281
+32^A277val_277
+33^A209val_209
+34^A15val_15
+35^A82val_82
+36^A403val_403
+37^A166val_166
+38^A417val_417
+39^A430val_430
+40^A252val_252
+41^A292val_292
+42^A219val_219
+43^A287val_287
+44^A153val_153
+45^A193val_193
+46^A338val_338
+47^A446val_446
+48^A459val_459
+49^A394val_394
+50^A237val_237
+51^A482val_482
+52^A174val_174
+53^A413val_413
+54^A494val_494
+55^A207val_207
+56^A199val_199
+57^A466val_466
+58^A208val_208
+59^A174val_174
+60^A399val_399
+61^A396val_396
+62^A247val_247
+63^A417val_417
+65^A489val_489
+66^A162val_162
+67^A377val_377
+68^A397val_397
+69^A309val_309
+70^A365val_365
+71^A266val_266
+72^A439val_439
+73^A342val_342
+74^A367val_367
+75^A325val_325
+76^A167val_167
+77^A195val_195
+78^A475val_475
+79^A17val_17
+80^A113val_113
+81^A155val_155
+82^A203val_203
+83^A339val_339
+84^A0val_0
+85^A455val_455
+86^A128val_128
+87^A311val_311
+88^A316val_316
+89^A57val_57
+90^A302val_302
+91^A205val_205
+92^A149val_149
+93^A438val_438
+94^A345val_345
+95^A129val_129
+96^A170val_170
+97^A20val_20
+98^A489val_489
+99^A157val_157
+100^A378val_378
+101^A221val_221
+102^A92val_92
+103^A111val_111
+104^A47val_47
+105^A72val_72
+106^A4val_4
+107^A280val_280
+108^A35val_35
+109^A427val_427
+110^A277val_277
+111^A208val_208
+112^A356val_356
+113^A399val_399
+114^A169val_169
+115^A382val_382
+116^A498val_498
+117^A125val_125
+118^A386val_386
+119^A437val_437
+120^A469val_469
+121^A192val_192
+122^A286val_286
+123^A187val_187
+125^A176val_176
+126^A54val_54
+127^A459val_459
+128^A51val_51
+129^A138val_138
+130^A103val_103
+131^A239val_239
+132^A213val_213
+133^A216val_216
+134^A430val_430
+135^A278val_278
+136^A176val_176
+137^A289val_289
+138^A221val_221
+139^A65val_65
+140^A318val_318
+141^A332val_332
+142^A311val_311
+143^A275val_275
+144^A137val_137
+145^A241val_241
+146^A83val_83
+147^A333val_333
+148^A180val_180
+149^A284val_284
+150^A12val_12
+151^A230val_230
+152^A181val_181
+153^A67val_67
+154^A260val_260
+155^A404val_404
+156^A384val_384
+157^A489val_489
+158^A353val_353
+159^A373val_373
+160^A272val_272
+161^A138val_138
+162^A217val_217
+163^A84val_84
+164^A348val_348
+165^A466val_466
+166^A58val_58
+167^A8val_8
+168^A411val_411
+170^A230val_230
+171^A208val_208
+172^A348val_348
+173^A24val_24
+174^A463val_463
+175^A431val_431
+176^A179val_179
+177^A172val_172
+178^A42val_42
+179^A129val_129
+180^A158val_158
+181^A119val_119
+182^A496val_496
+183^A0val_0
+184^A322val_322
+185^A197val_197
+186^A468val_468
+187^A393val_393
+188^A454val_454
+189^A100val_100
+190^A298val_298
+191^A199val_199
+192^A191val_191
+193^A418val_418
+194^A96val_96
+195^A26val_26
+196^A165val_165
+197^A327val_327
+198^A230val_230
+199^A205val_205
+200^A120val_120
+201^A131val_131
+202^A51val_51
+203^A404val_404
+204^A43val_43
+205^A436val_436
+206^A156val_156
+207^A469val_469
+208^A468val_468
+209^A308val_308
+210^A95val_95
+211^A196val_196
+212^A288val_288
+213^A481val_481
+214^A457val_457
+215^A98val_98
+216^A282val_282
+217^A197val_197
+218^A187val_187
+219^A318val_318
+220^A318val_318
+221^A409val_409
+222^A470val_470
+223^A137val_137
+224^A369val_369
+225^A316val_316
+226^A169val_169
+227^A413val_413
+228^A85val_85
+229^A77val_77
+230^A0val_0
+231^A490val_490
+232^A87val_87
+233^A364val_364
+234^A179val_179
+235^A118val_118
+236^A134val_134
+237^A395val_395
+238^A282val_282
+239^A138val_138
+240^A238val_238
+242^A419val_419
+243^A15val_15
+244^A118val_118
+245^A72val_72
+246^A90val_90
+247^A307val_307
+248^A19val_19
+249^A435val_435
+250^A10val_10
+251^A277val_277
+252^A273val_273
+253^A306val_306
+254^A224val_224
+255^A309val_309
+256^A389val_389
+257^A327val_327
+258^A242val_242
+259^A369val_369
+260^A392val_392
+261^A272val_272
+262^A331val_331
+263^A401val_401
+264^A242val_242
+265^A452val_452
+266^A177val_177
+267^A226val_226
+268^A5val_5
+269^A497val_497
+270^A402val_402
+272^A396val_396
+273^A317val_317
+274^A395val_395
+275^A58val_58
+276^A35val_35
+277^A336val_336
+278^A95val_95
+279^A11val_11
+280^A168val_168
+281^A34val_34
+282^A229val_229
+283^A233val_233
+284^A143val_143
+285^A472val_472
+286^A322val_322
+287^A498val_498
+288^A160val_160
+289^A195val_195
+290^A42val_42
+291^A321val_321
+292^A430val_430
+293^A119val_119
+294^A489val_489
+295^A458val_458
+296^A78val_78
+297^A76val_76
+298^A41val_41
+299^A223val_223
+300^A492val_492
+301^A149val_149
+302^A449val_449
+303^A218val_218
+304^A228val_228
+305^A138val_138
+306^A453val_453
+307^A30val_30
+308^A209val_209
+309^A64val_64
+310^A468val_468
+311^A76val_76
+312^A74val_74
+313^A342val_342
+314^A69val_69
+316^A230val_230
+317^A33val_33
+318^A368val_368
+319^A103val_103
+320^A296val_296
+321^A113val_113
+322^A216val_216
+323^A367val_367
+324^A344val_344
+325^A167val_167
+326^A274val_274
+327^A219val_219
+328^A239val_239
+329^A485val_485
+330^A116val_116
+331^A223val_223
+332^A256val_256
+333^A263val_263
+334^A70val_70
+335^A487val_487
+336^A480val_480
+337^A401val_401
+338^A288val_288
+339^A191val_191
+340^A5val_5
+341^A244val_244
+342^A438val_438
+343^A128val_128
+344^A467val_467
+345^A432val_432
+346^A202val_202
+347^A316val_316
+348^A229val_229
+349^A469val_469
+350^A463val_463
+351^A280val_280
+352^A2val_2
+353^A35val_35
+354^A283val_283
+355^A331val_331
+356^A235val_235
+357^A80val_80
+358^A44val_44
+359^A193val_193
+360^A321val_321
+361^A335val_335
+362^A104val_104
+363^A466val_466
+364^A366val_366
+365^A175val_175
+366^A403val_403
+367^A483val_483
+369^A53val_53
+370^A105val_105
+371^A257val_257
+372^A406val_406
+373^A409val_409
+374^A190val_190
+375^A406val_406
+376^A401val_401
+377^A114val_114
+378^A258val_258
+379^A90val_90
+380^A203val_203
+381^A262val_262
+382^A348val_348
+383^A424val_424
+384^A12val_12
+385^A396val_396
+386^A201val_201
+387^A217val_217
+388^A164val_164
+389^A431val_431
+390^A454val_454
+391^A478val_478
+392^A298val_298
+393^A125val_125
+394^A431val_431
+395^A164val_164
+396^A424val_424
+397^A187val_187
+398^A382val_382
+399^A5val_5
+400^A70val_70
+401^A397val_397
+402^A480val_480
+403^A291val_291
+404^A24val_24
+405^A351val_351
+407^A255val_255
+408^A104val_104
+409^A70val_70
+410^A163val_163
+411^A438val_438
+412^A119val_119
+413^A414val_414
+414^A200val_200
+415^A491val_491
+416^A237val_237
+417^A439val_439
+418^A360val_360
+419^A248val_248
+420^A479val_479
+421^A305val_305
+422^A417val_417
+423^A199val_199
+424^A444val_444
+425^A120val_120
+426^A429val_429
+427^A169val_169
+428^A443val_443
+429^A323val_323
+430^A325val_325
+431^A277val_277
+432^A230val_230
+433^A478val_478
+434^A178val_178
+435^A468val_468
+436^A310val_310
+437^A317val_317
+438^A333val_333
+439^A493val_493
+440^A460val_460
+441^A207val_207
+442^A249val_249
+443^A265val_265
+444^A480val_480
+445^A83val_83
+447^A136val_136
+448^A353val_353
+449^A172val_172
+450^A214val_214
+451^A462val_462
+452^A233val_233
+453^A406val_406
+454^A133val_133
+455^A175val_175
+456^A189val_189
+457^A454val_454
+458^A375val_375
+459^A401val_401
+460^A421val_421
+461^A407val_407
+462^A384val_384
+463^A256val_256
+464^A26val_26
+465^A134val_134
+466^A67val_67
+467^A384val_384
+468^A379val_379
+469^A18val_18
+470^A462val_462
+471^A492val_492
+472^A100val_100
+473^A298val_298
+474^A9val_9
+475^A341val_341
+476^A498val_498
+477^A146val_146
+478^A458val_458
+479^A362val_362
+480^A186val_186
+481^A285val_285
+483^A348val_348
+484^A167val_167
+485^A18val_18
+486^A273val_273
+487^A183val_183
+488^A281val_281
+489^A344val_344
+490^A97val_97
+491^A469val_469
+492^A315val_315
+493^A84val_84
+494^A28val_28
+495^A37val_37
+496^A448val_448
+497^A152val_152
+498^A348val_348
+499^A307val_307
+500^A194val_194
+501^A414val_414
+502^A477val_477
+503^A222val_222
+504^A126val_126
+505^A90val_90
+506^A169val_169
+507^A403val_403
+508^A400val_400
+509^A200val_200
+511^A97val_97
+512^A238val_238
+513^A86val_86
+514^A311val_311
+515^A27val_27
+516^A165val_165
+517^A409val_409
+518^A255val_255
+519^A278val_278
+520^A98val_98
+521^A484val_484
+522^A265val_265
+523^A193val_193
+524^A401val_401
+525^A150val_150
+526^A273val_273
+527^A224val_224
+528^A369val_369
+529^A66val_66
+530^A128val_128
+531^A213val_213
+532^A146val_146
+533^A406val_406
+535^A429val_429
+536^A374val_374
+537^A152val_152
+538^A469val_469
+539^A145val_145
+540^A495val_495
+541^A37val_37
+542^A327val_327
+543^A281val_281
+544^A277val_277
+545^A209val_209
+546^A15val_15
+547^A82val_82
+548^A403val_403
+549^A166val_166
+550^A417val_417
+551^A430val_430
+552^A252val_252
+553^A292val_292
+554^A219val_219
+555^A287val_287
+556^A153val_153
+557^A193val_193
+559^A338val_338
+560^A446val_446
+561^A459val_459
+562^A394val_394
+563^A237val_237
+564^A482val_482
+565^A174val_174
+566^A413val_413
+567^A494val_494
+568^A207val_207
+569^A199val_199
+570^A466val_466
+571^A208val_208
+572^A174val_174
+573^A399val_399
+574^A396val_396
+575^A247val_247
+576^A417val_417
+577^A489val_489
+578^A162val_162
+579^A377val_377
+580^A397val_397
+581^A309val_309
+582^A365val_365
+583^A266val_266
+584^A439val_439
+585^A342val_342
+586^A367val_367
+587^A325val_325
+588^A167val_167
+589^A195val_195
+590^A475val_475
+591^A17val_17
+593^A113val_113
+594^A155val_155
+595^A203val_203
+596^A339val_339
+597^A0val_0
+598^A455val_455
+599^A128val_128
+600^A311val_311
+601^A316val_316
+602^A57val_57
+603^A302val_302
+604^A205val_205
+605^A149val_149
+606^A438val_438
+607^A345val_345
+608^A129val_129
+609^A170val_170
+610^A20val_20
+611^A489val_489
+612^A157val_157
+613^A378val_378
+614^A221val_221
+615^A92val_92
+616^A111val_111
+617^A47val_47
+618^A72val_72
+619^A4val_4
+620^A280val_280
+621^A35val_35
+622^A427val_427
+623^A277val_277
+624^A208val_208
+625^A356val_356
+626^A399val_399
+627^A169val_169
+628^A382val_382
+629^A498val_498
+630^A125val_125
+631^A386val_386
+632^A437val_437
+633^A469val_469
+634^A192val_192
+635^A286val_286
+636^A187val_187
+637^A176val_176
+638^A54val_54
+639^A459val_459
+640^A51val_51
+641^A138val_138
+642^A103val_103
+643^A239val_239
+644^A213val_213
+645^A216val_216
+646^A430val_430
+647^A278val_278
+648^A176val_176
+649^A289val_289
+650^A221val_221
+651^A65val_65
+652^A318val_318
+653^A332val_332
+654^A311val_311
+655^A275val_275
+656^A137val_137
+657^A241val_241
+658^A83val_83
+659^A333val_333
+660^A180val_180
+661^A284val_284
+662^A12val_12
+663^A230val_230
+664^A181val_181
+665^A67val_67
+666^A260val_260
+667^A404val_404
+668^A384val_384
+669^A489val_489
+670^A353val_353
+671^A373val_373
+672^A272val_272
+673^A138val_138
+674^A217val_217
+675^A84val_84
+676^A348val_348
+677^A466val_466
+678^A58val_58
+679^A8val_8
+680^A411val_411
+681^A230val_230
+682^A208val_208
+683^A348val_348
+684^A24val_24
+685^A463val_463
+686^A431val_431
+687^A179val_179
+688^A172val_172
+689^A42val_42
+690^A129val_129
+691^A158val_158
+692^A119val_119
+693^A496val_496
+694^A0val_0
+695^A322val_322
+696^A197val_197
+697^A468val_468
+698^A393val_393
+699^A454val_454
+700^A100val_100
+701^A298val_298
+702^A199val_199
+703^A191val_191
+704^A418val_418
+705^A96val_96
+706^A26val_26
+707^A165val_165
+708^A327val_327
+709^A230val_230
+710^A205val_205
+711^A120val_120
+712^A131val_131
+713^A51val_51
+714^A404val_404
+715^A43val_43
+716^A436val_436
+717^A156val_156
+718^A469val_469
+719^A468val_468
+720^A308val_308
+721^A95val_95
+722^A196val_196
+723^A288val_288
+724^A481val_481
+725^A457val_457
+726^A98val_98
+727^A282val_282
+728^A197val_197
+729^A187val_187
+730^A318val_318
+731^A318val_318
+732^A409val_409
+733^A470val_470
+734^A137val_137
+735^A369val_369
+736^A316val_316
+737^A169val_169
+738^A413val_413
+739^A85val_85
+740^A77val_77
+741^A0val_0
+742^A490val_490
+743^A87val_87
+744^A364val_364
+745^A179val_179
+746^A118val_118
+747^A134val_134
+748^A395val_395
+749^A282val_282
+750^A138val_138
+751^A238val_238
+752^A419val_419
+753^A15val_15
+754^A118val_118
+755^A72val_72
+756^A90val_90
+757^A307val_307
+758^A19val_19
+759^A435val_435
+760^A10val_10
+761^A277val_277
+762^A273val_273
+763^A306val_306
+764^A224val_224
+765^A309val_309
+766^A389val_389
+767^A327val_327
+768^A242val_242
+769^A369val_369
+770^A392val_392
+771^A272val_272
+772^A331val_331
+773^A401val_401
+774^A242val_242
+775^A452val_452
+776^A177val_177
+777^A226val_226
+778^A5val_5
+779^A497val_497
+780^A402val_402
+781^A396val_396
+782^A317val_317
+783^A395val_395
+784^A58val_58
+785^A35val_35
+786^A336val_336
+787^A95val_95
+788^A11val_11
+789^A168val_168
+790^A34val_34
+791^A229val_229
+792^A233val_233
+793^A143val_143
+794^A472val_472
+795^A322val_322
+796^A498val_498
+797^A160val_160
+798^A195val_195
+799^A42val_42
+800^A321val_321
+801^A430val_430
+802^A119val_119
+803^A489val_489
+804^A458val_458
+805^A78val_78
+806^A76val_76
+807^A41val_41
+808^A223val_223
+809^A492val_492
+810^A149val_149
+811^A449val_449
+812^A218val_218
+813^A228val_228
+814^A138val_138
+815^A453val_453
+816^A30val_30
+817^A209val_209
+818^A64val_64
+819^A468val_468
+820^A76val_76
+821^A74val_74
+822^A342val_342
+823^A69val_69
+824^A230val_230
+825^A33val_33
+826^A368val_368
+827^A103val_103
+828^A296val_296
+829^A113val_113
+830^A216val_216
+831^A367val_367
+832^A344val_344
+833^A167val_167
+834^A274val_274
+835^A219val_219
+836^A239val_239
+837^A485val_485
+838^A116val_116
+839^A223val_223
+840^A256val_256
+841^A263val_263
+842^A70val_70
+843^A487val_487
+844^A480val_480
+845^A401val_401
+846^A288val_288
+847^A191val_191
+848^A5val_5
+849^A244val_244
+850^A438val_438
+851^A128val_128
+852^A467val_467
+853^A432val_432
+854^A202val_202
+855^A316val_316
+856^A229val_229
+857^A469val_469
+858^A463val_463
+859^A280val_280
+860^A2val_2
+861^A35val_35
+862^A283val_283
+863^A331val_331
+864^A235val_235
+865^A80val_80
+866^A44val_44
+867^A193val_193
+868^A321val_321
+869^A335val_335
+870^A104val_104
+871^A466val_466
+872^A366val_366
+873^A175val_175
+874^A403val_403
+875^A483val_483
+876^A53val_53
+877^A105val_105
+878^A257val_257
+879^A406val_406
+880^A409val_409
+881^A190val_190
+882^A406val_406
+883^A401val_401
+884^A114val_114
+885^A258val_258
+886^A90val_90
+887^A203val_203
+888^A262val_262
+889^A348val_348
+890^A424val_424
+891^A12val_12
+892^A396val_396
+893^A201val_201
+894^A217val_217
+895^A164val_164
+896^A431val_431
+897^A454val_454
+898^A478val_478
+899^A298val_298
+900^A125val_125
+901^A431val_431
+902^A164val_164
+903^A424val_424
+904^A187val_187
+905^A382val_382
+906^A5val_5
+907^A70val_70
+908^A397val_397
+909^A480val_480
+910^A291val_291
+911^A24val_24
+912^A351val_351
+913^A255val_255
+914^A104val_104
+915^A70val_70
+916^A163val_163
+917^A438val_438
+918^A119val_119
+919^A414val_414
+920^A200val_200
+921^A491val_491
+922^A237val_237
+923^A439val_439
+924^A360val_360
+925^A248val_248
+926^A479val_479
+927^A305val_305
+928^A417val_417
+929^A199val_199
+930^A444val_444
+931^A120val_120
+932^A429val_429
+933^A169val_169
+934^A443val_443
+935^A323val_323
+936^A325val_325
+937^A277val_277
+938^A230val_230
+939^A478val_478
+940^A178val_178
+941^A468val_468
+942^A310val_310
+943^A317val_317
+944^A333val_333
+945^A493val_493
+946^A460val_460
+947^A207val_207
+948^A249val_249
+949^A265val_265
+950^A480val_480
+951^A83val_83
+952^A136val_136
+953^A353val_353
+954^A172val_172
+955^A214val_214
+956^A462val_462
+957^A233val_233
+958^A406val_406
+959^A133val_133
+960^A175val_175
+961^A189val_189
+962^A454val_454
+963^A375val_375
+964^A401val_401
+965^A421val_421
+966^A407val_407
+967^A384val_384
+968^A256val_256
+969^A26val_26
+970^A134val_134
+971^A67val_67
+972^A384val_384
+973^A379val_379
+974^A18val_18
+975^A462val_462
+976^A492val_492
+977^A100val_100
+978^A298val_298
+979^A9val_9
+980^A341val_341
+981^A498val_498
+982^A146val_146
+983^A458val_458
+984^A362val_362
+985^A186val_186
+986^A285val_285
+987^A348val_348
+988^A167val_167
+989^A18val_18
+990^A273val_273
+991^A183val_183
+992^A281val_281
+993^A344val_344
+994^A97val_97
+995^A469val_469
+996^A315val_315
+997^A84val_84
+998^A28val_28
+999^A37val_37
+1000^A448val_448
+1001^A152val_152
+1002^A348val_348
+1003^A307val_307
+1004^A194val_194
+1005^A414val_414
+1006^A477val_477
+1007^A222val_222
+1008^A126val_126
+1009^A90val_90
+1010^A169val_169
+1011^A403val_403
+1012^A400val_400
+1013^A200val_200
+1014^A97val_97
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/17f901a6/webapp/src/test/resources/apps/hive/script.hql
----------------------------------------------------------------------
diff --git a/webapp/src/test/resources/apps/hive/script.hql b/webapp/src/test/resources/apps/hive/script.hql
new file mode 100644
index 0000000..6a81635
--- /dev/null
+++ b/webapp/src/test/resources/apps/hive/script.hql
@@ -0,0 +1,19 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+INSERT OVERWRITE TABLE ${falcon_output_database}.${falcon_output_table} PARTITION ${falcon_input_filter} SELECT id, value FROM ${falcon_input_database}.${falcon_input_table} WHERE ${falcon_input_filter};