You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by co...@apache.org on 2015/01/22 00:52:19 UTC
bigtop git commit: BIGTOP-1334. Add DFS tests to TestCLI
Repository: bigtop
Updated Branches:
refs/heads/master 3bc16d0d0 -> 2d5fd34fd
BIGTOP-1334. Add DFS tests to TestCLI
Signed-off-by: Konstantin Boudnik <co...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/2d5fd34f
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/2d5fd34f
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/2d5fd34f
Branch: refs/heads/master
Commit: 2d5fd34fddbf1d40a6aeb8aa5d39e2c6c630cff5
Parents: 3bc16d0
Author: Dasha Boudnik <da...@wandisco.com>
Authored: Tue Jan 20 20:03:17 2015 -0800
Committer: Konstantin Boudnik <co...@apache.org>
Committed: Wed Jan 21 15:52:10 2015 -0800
----------------------------------------------------------------------
.../bigtop/itest/hadoop/hcfs/TestDFSCLI.java | 109 ++++++++
.../main/resources/clitest_data/testDFSConf.xml | 252 +++++++++++++++++++
2 files changed, 361 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/2d5fd34f/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/hcfs/TestDFSCLI.java
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/hcfs/TestDFSCLI.java b/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/hcfs/TestDFSCLI.java
new file mode 100644
index 0000000..586c0e5
--- /dev/null
+++ b/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/hcfs/TestDFSCLI.java
@@ -0,0 +1,109 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.bigtop.itest.hadoop.hcfs;
+
+import org.apache.bigtop.itest.shell.Shell;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.cli.TestHDFSCLI;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+
+import java.io.File;
+
+public class TestDFSCLI extends TestHDFSCLI {
+ public static final String TEST_DIR_ABSOLUTE = "/tmp/testcli_" + Long.valueOf(System.currentTimeMillis());
+ public static String NAMENODE_TESTDIR_HACK;
+ private String supergroup;
+ private static Shell shHDFS = new Shell("/bin/bash");
+
+ @Before
+ @Override
+ public void setUp() throws Exception {
+ readTestConfigFile();
+
+ // Configuration of real Hadoop cluster
+ conf = new HdfsConfiguration();
+ supergroup = System.getProperty("hcfs.root.groupname",
+ conf.get(DFSConfigKeys.DFS_PERMISSIONS_SUPERUSERGROUP_KEY));
+ namenode = conf.get(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY);
+ username = System.getProperty("user.name");
+
+ conf.setBoolean(CommonConfigurationKeys.HADOOP_SECURITY_AUTHORIZATION, true);
+ // Many of the tests expect a replication value of 1 in the output
+ conf.setInt("dfs.replication", 1);
+
+ clitestDataDir = new File(TEST_CACHE_DATA_DIR).toURI().toString().replace(' ', '+');
+
+ String[] createTestcliDirCmds = {
+ "hadoop fs -mkdir -p " + TEST_DIR_ABSOLUTE,
+ "hadoop fs -chmod 777 " + TEST_DIR_ABSOLUTE
+ };
+ shHDFS.exec(createTestcliDirCmds);
+
+ // Check assumptions which would make some cases fail if not met
+ Assert.assertEquals("Creation of testcli dir should succeed and return 0"
+ + " (but it failed with the following error message: "
+ + StringUtils.join(shHDFS.getErr().toArray(), "\\n") + ")",
+ 0, shHDFS.getRet());
+ // We can't just use conf.setInt(fs.trash.interval", 0) because if trash is
+ // enabled on the server, client configuration value is ignored.
+ Assert.assertEquals("HDFS trash should be disabled via fs.trash.interval",
+ 0, conf.getInt("fs.trash.interval", 0));
+ Assert.assertEquals("This test needs to be run under root user of hcfs",
+ System.getProperty("hcfs.root.username", "hdfs"), username);
+ }
+
+ @After
+ @Override
+ public void tearDown() throws Exception {
+ super.tearDown();
+
+ String removeTestcliDirCmd = "hadoop fs -rm -r " + TEST_DIR_ABSOLUTE;
+ shHDFS.exec(removeTestcliDirCmd);
+ }
+
+ @Override
+ protected String getTestFile() {
+ return "testDFSConf.xml";
+ }
+
+ /**
+ * Expand commands from the test config file.
+ * This method is used in displayResults() and compareTestOutput() only,
+ * so it doesn't have any effect on the test execution itself.
+ *
+ * @param cmd executed command
+ * @return String expanded command
+ */
+ @Override
+ protected String expandCommand(String cmd) {
+ String expCmd = super.expandCommand(cmd);
+ // note: super.expandCommand() expands CLITEST_DATA and USERNAME
+ expCmd = expCmd.replaceAll("NAMENODETEST_DIR_ABSOLUTE", NAMENODE_TESTDIR_HACK);
+ expCmd = expCmd.replaceAll("TEST_DIR_ABSOLUTE", TEST_DIR_ABSOLUTE);
+ expCmd = expCmd.replaceAll("supergroup", supergroup);
+ expCmd = expCmd.replaceAll("NAMENODE", namenode);
+ expCmd = expCmd.replaceAll("USER_NAME", System.getProperty("user.name"));
+ return expCmd;
+ }
+}
http://git-wip-us.apache.org/repos/asf/bigtop/blob/2d5fd34f/bigtop-tests/test-artifacts/hadoop/src/main/resources/clitest_data/testDFSConf.xml
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/hadoop/src/main/resources/clitest_data/testDFSConf.xml b/bigtop-tests/test-artifacts/hadoop/src/main/resources/clitest_data/testDFSConf.xml
new file mode 100644
index 0000000..2f601f8
--- /dev/null
+++ b/bigtop-tests/test-artifacts/hadoop/src/main/resources/clitest_data/testDFSConf.xml
@@ -0,0 +1,252 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="testConf.xsl"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<configuration>
+ <!-- Normal mode is test. To run just the commands and dump the output
+ to the log, set it to nocompare -->
+ <mode>test</mode>
+
+<tests>
+
+ <test>
+ <description>allowSnapshot</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir -p /user/USERNAME/dir1</command>
+ <dfs-admin-command>-fs NAMENODE -allowSnapshot /user/USERNAME/dir1</dfs-admin-command>
+ </test-commands>
+ <cleanup-commands>
+ <dfs-admin-command>-fs NAMENODE -disallowSnapshot /user/USERNAME/dir1</dfs-admin-command>
+ <command>-fs NAMENODE -rm -r /user/USERNAME</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>Allowing snaphot on /user/USERNAME/dir1 succeeded</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+
+ <test>
+ <description>disallowSnapshot</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir -p /user/USERNAME/dir1</command>
+ <dfs-admin-command>-fs NAMENODE -allowSnapshot /user/USERNAME/dir1</dfs-admin-command>
+ <dfs-admin-command>-fs NAMENODE -disallowSnapshot /user/USERNAME/dir1</dfs-admin-command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -r /user/USERNAME</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>Disallowing snaphot on /user/USERNAME/dir1 succeeded</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+
+ <test>
+ <description>createSnapshot</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir -p /user/USERNAME/dir1</command>
+ <dfs-admin-command>-fs NAMENODE -allowSnapshot /user/USERNAME/dir1</dfs-admin-command>
+ <command>-fs NAMENODE -createSnapshot /user/USERNAME/dir1 snapshot1</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -deleteSnapshot /user/USERNAME/dir1 snapshot1</command>
+ <dfs-admin-command>-fs NAMENODE -disallowSnapshot /user/USERNAME/dir1</dfs-admin-command>
+ <command>-fs NAMENODE -rm -r /user/USERNAME</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>Created snapshot /user/USERNAME/dir1/.snapshot/snapshot1</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+
+ <test>
+ <description>renameSnapshot</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir -p /user/USERNAME/dir1</command>
+ <dfs-admin-command>-fs NAMENODE -allowSnapshot /user/USERNAME/dir1</dfs-admin-command>
+ <command>-fs NAMENODE -createSnapshot /user/USERNAME/dir1 snapshot1</command>
+ <command>-fs NAMENODE -renameSnapshot /user/USERNAME/dir1 snapshot1 snapshot2</command>
+ <command>-fs NAMENODE -ls /user/USERNAME/dir1/.snapshot</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -deleteSnapshot /user/USERNAME/dir1 snapshot2</command>
+ <dfs-admin-command>-fs NAMENODE -disallowSnapshot /user/USERNAME/dir1</dfs-admin-command>
+ <command>-fs NAMENODE -rm -r /user/USERNAME</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>Found 1 items</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^drwxr-xr-x( )*-( )*USERNAME( )*supergroup( )*0( )*[0-9]{4,}-[0-9]{2,}-[0-9]{2,} [0-9]{2,}:[0-9]{2,}( )*/user/USERNAME/dir1/.snapshot/snapshot2</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+
+ <test>
+ <description>deleteSnapshot</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir -p /user/USERNAME/dir1</command>
+ <dfs-admin-command>-fs NAMENODE -allowSnapshot /user/USERNAME/dir1</dfs-admin-command>
+ <command>-fs NAMENODE -createSnapshot /user/USERNAME/dir1 snapshot1</command>
+ <command>-fs NAMENODE -deleteSnapshot /user/USERNAME/dir1 snapshot1</command>
+ <command>-fs NAMENODE -ls /user/USERNAME/dir1/.snapshot</command>
+ </test-commands>
+ <cleanup-commands>
+ <dfs-admin-command>-fs NAMENODE -disallowSnapshot /user/USERNAME/dir1</dfs-admin-command>
+ <command>-fs NAMENODE -rm -r /user/USERNAME</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>ExactComparator</type>
+ <expected-output></expected-output>
+ </comparator>
+ </comparators>
+ </test>
+
+ <!-- DFSadmin tests -->
+ <test>
+ <description>refreshUserToGroupsMappings</description>
+ <test-commands>
+ <dfs-admin-command>-fs NAMENODE -refreshUserToGroupsMappings</dfs-admin-command>
+ </test-commands>
+ <cleanup-commands>
+ <command></command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output></expected-output>
+ </comparator>
+ </comparators>
+ </test>
+
+ <test>
+ <description>refreshSuperUserGroupsConfiguration</description>
+ <test-commands>
+ <dfs-admin-command>-fs NAMENODE -refreshSuperUserGroupsConfiguration</dfs-admin-command>
+ </test-commands>
+ <cleanup-commands>
+ <command></command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output></expected-output>
+ </comparator>
+ </comparators>
+ </test>
+
+ <test>
+ <description>setQuota</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir -p /user/USERNAME/dir1</command>
+ <dfs-admin-command>-fs NAMENODE -setQuota 3 /user/USERNAME/dir1</dfs-admin-command>
+ <command>-fs NAMENODE -count -q /user/USERNAME/dir1</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -r /user/USERNAME</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>( )*3.*/user/USERNAME/dir1</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+
+ <test>
+ <description>clrQuota</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir -p /user/USERNAME/dir1</command>
+ <dfs-admin-command>-fs NAMENODE -setQuota 3 /user/USERNAME/dir1</dfs-admin-command>
+ <dfs-admin-command>-fs NAMENODE -clrQuota /user/USERNAME/dir1</dfs-admin-command>
+ <command>-fs NAMENODE -count -q /user/USERNAME/dir1</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -r /user/USERNAME</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>( )*none.*/user/USERNAME/dir1</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+
+ <test>
+ <description>setSpaceQuota</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir -p /user/USERNAME/dir1</command>
+ <dfs-admin-command>-fs NAMENODE -setSpaceQuota 1G /user/USERNAME/dir1</dfs-admin-command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -r /user/USERNAME/dir1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>ExactComparator</type>
+ <expected-output></expected-output>
+ </comparator>
+ </comparators>
+ </test>
+
+ <test>
+ <description>setBalancerBandwidth</description>
+ <test-commands>
+ <dfs-admin-command>-fs NAMENODE -setBalancerBandwidth 104857600</dfs-admin-command>
+ </test-commands>
+ <cleanup-commands>
+ <command></command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output></expected-output>
+ </comparator>
+ </comparators>
+ </test>
+
+ <test>
+ <description>finalizeUpgrade</description>
+ <test-commands>
+ <dfs-admin-command>-fs NAMENODE -finalizeUpgrade</dfs-admin-command>
+ </test-commands>
+ <cleanup-commands>
+ <command></command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output></expected-output>
+ </comparator>
+ </comparators>
+ </test>
+
+</tests>
+
+</configuration>