You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by rv...@apache.org on 2014/02/27 20:46:38 UTC
[2/2] git commit: BIGTOP-1188. Smoke tests for HttpFs
BIGTOP-1188. Smoke tests for HttpFs
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/69f7c942
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/69f7c942
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/69f7c942
Branch: refs/heads/master
Commit: 69f7c9426c4a80dde7c9b1509120cd8600c65976
Parents: d4504ae
Author: Adam Kawa <ka...@gmail.com>
Authored: Tue Jan 21 09:37:55 2014 +0000
Committer: vagrant <vagrant@ubuntu-server-12042-x64-vbox4210.(none)>
Committed: Thu Feb 27 19:26:09 2014 +0000
----------------------------------------------------------------------
bigtop-tests/test-artifacts/httpfs/pom.xml | 35 +++++
.../bigtop/itest/httpfs/TestHttpFs.groovy | 153 +++++++++++++++++++
.../main/resources/text-files/helloworld.txt | 1 +
bigtop-tests/test-artifacts/pom.xml | 1 +
.../test-execution/smokes/httpfs/pom.xml | 97 ++++++++++++
bigtop-tests/test-execution/smokes/pom.xml | 9 +-
6 files changed, 295 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/69f7c942/bigtop-tests/test-artifacts/httpfs/pom.xml
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/httpfs/pom.xml b/bigtop-tests/test-artifacts/httpfs/pom.xml
new file mode 100644
index 0000000..aafa8f3
--- /dev/null
+++ b/bigtop-tests/test-artifacts/httpfs/pom.xml
@@ -0,0 +1,35 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.bigtop.itest</groupId>
+ <artifactId>bigtop-smokes</artifactId>
+ <version>0.8.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+
+ <groupId>org.apache.bigtop.itest</groupId>
+ <artifactId>httpfs-smoke</artifactId>
+ <version>0.8.0-SNAPSHOT</version>
+ <name>httpfssmokes</name>
+
+</project>
http://git-wip-us.apache.org/repos/asf/bigtop/blob/69f7c942/bigtop-tests/test-artifacts/httpfs/src/main/groovy/org/apache/bigtop/itest/httpfs/TestHttpFs.groovy
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/httpfs/src/main/groovy/org/apache/bigtop/itest/httpfs/TestHttpFs.groovy b/bigtop-tests/test-artifacts/httpfs/src/main/groovy/org/apache/bigtop/itest/httpfs/TestHttpFs.groovy
new file mode 100644
index 0000000..0aa9a45
--- /dev/null
+++ b/bigtop-tests/test-artifacts/httpfs/src/main/groovy/org/apache/bigtop/itest/httpfs/TestHttpFs.groovy
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.bigtop.itest.hadoop.hdfs
+
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.apache.bigtop.itest.shell.Shell;
+import org.apache.bigtop.itest.TestUtils
+import org.apache.commons.logging.Log
+import org.apache.commons.logging.LogFactory
+
+public class TestHttpFs {
+ private static Log LOG = LogFactory.getLog(TestHttpFs.class)
+
+ private static final String USERNAME = System.getProperty("user.name");
+ private static final String HTTPFS_PROXY = System.getenv('HTTPFS_PROXY');
+ static {
+ assertNotNull("HTTPFS_PROXY has to be set to run this test",
+ HTTPFS_PROXY);
+ }
+
+ private static final String HTTPFS_PREFIX = "http://$HTTPFS_PROXY/webhdfs/v1";
+ private static final String HTTPFS_SUCCESS = "{\"boolean\":true}";
+
+ private static final String DATA_DIR = System.getProperty("data.dir", "text-files");
+
+ private static String testHttpFsFolder = "/tmp/httpfssmoke-" + (new Date().getTime());
+ private static String testHttpFsFolderRenamed = "$testHttpFsFolder-renamed";
+
+ private static Shell shHDFS = new Shell("/bin/bash", USERNAME);
+ private static Shell sh = new Shell("/bin/bash");
+
+
+ @BeforeClass
+ public static void setUp() {
+ }
+
+ @AfterClass
+ public static void tearDown() {
+ // clean up of existing folders
+ shHDFS.exec("hadoop fs -test -e $testHttpFsFolder");
+ if (shHDFS.getRet() == 0) {
+ shHDFS.exec("hadoop fs -rmr -skipTrash $testHttpFsFolder");
+ assertTrue("Deletion of previous testHttpFsFolder from HDFS failed",
+ shHDFS.getRet() == 0);
+ }
+ shHDFS.exec("hadoop fs -test -e $testHttpFsFolderRenamed");
+ if (shHDFS.getRet() == 0) {
+ shHDFS.exec("hadoop fs -rmr -skipTrash $testHttpFsFolderRenamed");
+ assertTrue("Deletion of previous testHttpFsFolderRenamed from HDFS failed",
+ shHDFS.getRet() == 0);
+ }
+ }
+
+ public void assertValueExists(List<String> values, String expected) {
+ boolean exists = false;
+ for (String value: values) {
+ if (expected.startsWith(value)) {
+ exists = true;
+ }
+ }
+ assertTrue(expected + " NOT found!", exists == true);
+ }
+
+ private void createDir(String dirname) {
+ sh.exec("curl -i -X PUT '$HTTPFS_PREFIX$dirname?user.name=$USERNAME&op=MKDIRS'");
+ }
+
+ @Test
+ public void testCreateDir() {
+ createDir(testHttpFsFolder)
+ assertTrue("curl command to create a dir failed", sh.getRet() == 0);
+ assertValueExists(sh.getOut(), HTTPFS_SUCCESS);
+ }
+
+ @Test
+ public void testRenameDir() {
+ createDir(testHttpFsFolder);
+ assertTrue("curl command to create a dir failed", sh.getRet() == 0);
+ assertValueExists(sh.getOut(), HTTPFS_SUCCESS);
+ sh.exec("curl -i -X PUT '$HTTPFS_PREFIX$testHttpFsFolder?user.name=$USERNAME&op=RENAME&destination=$testHttpFsFolderRenamed'");
+ assertTrue("curl command to rename a dir failed", shHDFS.getRet() == 0);
+ assertValueExists(sh.getOut(), HTTPFS_SUCCESS);
+ }
+
+ @Test
+ public void testDeleteDir() {
+ createDir(testHttpFsFolder);
+ assertTrue("curl command to create a dir failed", sh.getRet() == 0);
+ assertValueExists(sh.getOut(), HTTPFS_SUCCESS);
+ sh.exec("curl -i -X DELETE '$HTTPFS_PREFIX$testHttpFsFolder?user.name=$USERNAME&op=DELETE'");
+ assertTrue("curl command to delete a dir failed", shHDFS.getRet() == 0);
+ assertValueExists(sh.getOut(), HTTPFS_SUCCESS);
+ }
+
+ @Test
+ public void testStatusDir() {
+ createDir(testHttpFsFolder);
+ assertTrue("curl command to create a dir failed", sh.getRet() == 0);
+ assertValueExists(sh.getOut(), HTTPFS_SUCCESS);
+ sh.exec("curl -i '$HTTPFS_PREFIX$testHttpFsFolder?user.name=$USERNAME&op=GETFILESTATUS'");
+ assertTrue("curl command to create a dir failed", sh.getRet() == 0);
+ assertValueExists(sh.getOut(), HTTPFS_SUCCESS);
+ assertValueExists(sh.getOut(), "DIRECTORY");
+ }
+
+ @Test
+ public void testCreateFile() {
+ String filename = "helloworld.txt";
+ String filenameContent = 'Hello World!';
+
+ createDir(testHttpFsFolder);
+ assertTrue("curl command to create a dir failed", sh.getRet() == 0);
+ sh.exec("curl -i -X PUT '$HTTPFS_PREFIX$testHttpFsFolder/$filename?user.name=$USERNAME&op=CREATE'");
+ assertTrue("curl command to create a file failed", shHDFS.getRet() == 0);
+ String datanodeLocation = null;
+ sh.getOut().each {
+ if (it.startsWith("Location:")) {
+ datanodeLocation = it.split(' ')[1];
+ return true;
+ }
+ }
+ LOG.debug("Datanode location: $datanodeLocation");
+ assertValueExists(sh.getOut(), HTTPFS_SUCCESS);
+ sh.exec("curl -i -T $DATA_DIR/$filename '$datanodeLocation' --header 'Content-Type:application/octet-stream'");
+ assertTrue("curl command to create a file failed", shHDFS.getRet() == 0);
+ assertValueExists(sh.getOut(), HTTPFS_SUCCESS);
+ sh.exec("curl -i -L '$HTTPFS_PREFIX$testHttpFsFolder/$filename?user.name=$USERNAME&op=OPEN'");
+ assertTrue("curl command to create a file failed", shHDFS.getRet() == 0);
+ assertValueExists(sh.getOut(), HTTPFS_SUCCESS);
+ assertValueExists(sh.getOut(), filenameContent);
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/bigtop/blob/69f7c942/bigtop-tests/test-artifacts/httpfs/src/main/resources/text-files/helloworld.txt
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/httpfs/src/main/resources/text-files/helloworld.txt b/bigtop-tests/test-artifacts/httpfs/src/main/resources/text-files/helloworld.txt
new file mode 100644
index 0000000..980a0d5
--- /dev/null
+++ b/bigtop-tests/test-artifacts/httpfs/src/main/resources/text-files/helloworld.txt
@@ -0,0 +1 @@
+Hello World!
http://git-wip-us.apache.org/repos/asf/bigtop/blob/69f7c942/bigtop-tests/test-artifacts/pom.xml
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/pom.xml b/bigtop-tests/test-artifacts/pom.xml
index e3963cb..a7911e8 100644
--- a/bigtop-tests/test-artifacts/pom.xml
+++ b/bigtop-tests/test-artifacts/pom.xml
@@ -35,6 +35,7 @@
<modules>
<module>hadoop</module>
<module>package</module>
+ <module>httpfs</module>
<module>oozie</module>
<module>hive</module>
<module>hbase</module>
http://git-wip-us.apache.org/repos/asf/bigtop/blob/69f7c942/bigtop-tests/test-execution/smokes/httpfs/pom.xml
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-execution/smokes/httpfs/pom.xml b/bigtop-tests/test-execution/smokes/httpfs/pom.xml
new file mode 100644
index 0000000..74696e2
--- /dev/null
+++ b/bigtop-tests/test-execution/smokes/httpfs/pom.xml
@@ -0,0 +1,97 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+ <parent>
+ <groupId>org.apache.bigtop.itest</groupId>
+ <artifactId>smoke-tests</artifactId>
+ <version>0.8.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+
+ <modelVersion>4.0.0</modelVersion>
+ <groupId>org.apache.bigtop.itest</groupId>
+ <artifactId>httpfs-smoke-execution</artifactId>
+ <version>0.8.0-SNAPSHOT</version>
+ <name>Httpfs smoke test execution</name>
+
+ <properties>
+ <org.apache.maven-dependency-plugin.groupId>org.apache.bigtop.itest</org.apache.maven-dependency-plugin.groupId>
+ <org.apache.maven-dependency-plugin.artifactId>httpfs-smoke</org.apache.maven-dependency-plugin.artifactId>
+ <org.apache.maven-dependency-plugin.version>${httpfs-smoke.version}</org.apache.maven-dependency-plugin.version>
+ <org.apache.maven-dependency-plugin.type>jar</org.apache.maven-dependency-plugin.type>
+
+ <HADOOP_HOME>${env.HADOOP_HOME}</HADOOP_HOME>
+ <HADOOP_CONF_DIR>${env.HADOOP_CONF_DIR}</HADOOP_CONF_DIR>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.bigtop.itest</groupId>
+ <artifactId>httpfs-smoke</artifactId>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-enforcer-plugin</artifactId>
+ <version>1.0</version>
+ <executions>
+ <execution>
+ <id>enforce-property</id>
+ <goals>
+ <goal>enforce</goal>
+ </goals>
+ <configuration>
+ <rules>
+ <requireProperty>
+ <property>HADOOP_HOME</property>
+ <message>HADOOP_HOME env. variable has to be set</message>
+ </requireProperty>
+ <requireProperty>
+ <property>HADOOP_CONF_DIR</property>
+ <message>HADOOP_CONF_DIR env. variable has to be set</message>
+ </requireProperty>
+ </rules>
+ <fail>true</fail>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-failsafe-plugin</artifactId>
+ <version>2.11</version>
+ <configuration>
+ <additionalClasspathElements>
+ <additionalClasspathElement>${HADOOP_CONF_DIR}</additionalClasspathElement>
+ </additionalClasspathElements>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
http://git-wip-us.apache.org/repos/asf/bigtop/blob/69f7c942/bigtop-tests/test-execution/smokes/pom.xml
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-execution/smokes/pom.xml b/bigtop-tests/test-execution/smokes/pom.xml
index 7ccc8bd..1da31b2 100644
--- a/bigtop-tests/test-execution/smokes/pom.xml
+++ b/bigtop-tests/test-execution/smokes/pom.xml
@@ -37,6 +37,7 @@
<itest-conf.version>${project.version}</itest-conf.version>
<itest-common.version>${project.version}</itest-common.version>
<hadoop-smoke.version>${project.version}</hadoop-smoke.version>
+ <httpfs-smoke.version>${project.version}</httpfs-smoke.version>
<flume-smoke.version>${project.version}</flume-smoke.version>
<hbase-smoke.version>${project.version}</hbase-smoke.version>
<hive-smoke.version>${project.version}</hive-smoke.version>
@@ -64,8 +65,8 @@
<module>giraph</module>
<module>hue</module>
<module>crunch</module>
-
<!--
+ <module>httpfs</module>
<module>flume</module>
<module>sqoop</module>
-->
@@ -154,6 +155,12 @@
<version>${phoenix-smoke.version}</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.bigtop.itest</groupId>
+ <artifactId>httpfs-smoke</artifactId>
+ <version>${httpfs-smoke.version}</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
</dependencyManagement>