You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2013/09/27 13:12:45 UTC
svn commit: r1526848 [7/8] - in /hadoop/common/branches/branch-2:
hadoop-project/ hadoop-tools/ hadoop-tools/hadoop-openstack/
hadoop-tools/hadoop-openstack/src/ hadoop-tools/hadoop-openstack/src/main/
hadoop-tools/hadoop-openstack/src/main/java/ hadoo...
Added: hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemDirectories.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemDirectories.java?rev=1526848&view=auto
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemDirectories.java (added)
+++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemDirectories.java Fri Sep 27 11:12:42 2013
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.swift.snative.SwiftFileStatus;
+import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
+import org.junit.Test;
+
+import java.io.FileNotFoundException;
+
+/**
+ * Test swift-specific directory logic.
+ * This class is HDFS-1 compatible; its designed to be subclases by something
+ * with HDFS2 extensions
+ */
+public class TestSwiftFileSystemDirectories extends SwiftFileSystemBaseTest {
+
+ /**
+ * Asserts that a zero byte file has a status of file and not
+ * directory or symlink
+ *
+ * @throws Exception on failures
+ */
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testZeroByteFilesAreDirectories() throws Exception {
+ Path src = path("/test/testZeroByteFilesAreFiles");
+ //create a zero byte file
+ SwiftTestUtils.touch(fs, src);
+ SwiftTestUtils.assertIsDirectory(fs, src);
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testNoStatusForMissingDirectories() throws Throwable {
+ Path missing = path("/test/testNoStatusForMissingDirectories");
+ assertPathDoesNotExist("leftover?", missing);
+ try {
+ FileStatus[] statuses = fs.listStatus(missing);
+ //not expected
+ fail("Expected a FileNotFoundException, got the status " + statuses);
+ } catch (FileNotFoundException expected) {
+ //expected
+ }
+ }
+
+ /**
+ * test that a dir off root has a listStatus() call that
+ * works as expected. and that when a child is added. it changes
+ *
+ * @throws Exception on failures
+ */
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testDirectoriesOffRootHaveMatchingFileStatus() throws Exception {
+ Path test = path("/test");
+ fs.delete(test, true);
+ mkdirs(test);
+ assertExists("created test directory", test);
+ FileStatus[] statuses = fs.listStatus(test);
+ String statusString = statusToString(test.toString(), statuses);
+ assertEquals("Wrong number of elements in file status " + statusString, 0,
+ statuses.length);
+
+ Path src = path("/test/file");
+
+ //create a zero byte file
+ SwiftTestUtils.touch(fs, src);
+ //stat it
+ statuses = fs.listStatus(test);
+ statusString = statusToString(test.toString(), statuses);
+ assertEquals("Wrong number of elements in file status " + statusString, 1,
+ statuses.length);
+ SwiftFileStatus stat = (SwiftFileStatus) statuses[0];
+ assertTrue("isDir(): Not a directory: " + stat, stat.isDir());
+ extraStatusAssertions(stat);
+ }
+
+ /**
+ * test that a dir two levels down has a listStatus() call that
+ * works as expected.
+ *
+ * @throws Exception on failures
+ */
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testDirectoriesLowerDownHaveMatchingFileStatus() throws Exception {
+ Path test = path("/test/testDirectoriesLowerDownHaveMatchingFileStatus");
+ fs.delete(test, true);
+ mkdirs(test);
+ assertExists("created test sub directory", test);
+ FileStatus[] statuses = fs.listStatus(test);
+ String statusString = statusToString(test.toString(), statuses);
+ assertEquals("Wrong number of elements in file status " + statusString,0,
+ statuses.length);
+ }
+
+ private String statusToString(String pathname,
+ FileStatus[] statuses) {
+ assertNotNull(statuses);
+ return SwiftTestUtils.dumpStats(pathname,statuses);
+ }
+
+ /**
+ * method for subclasses to add extra assertions
+ * @param stat status to look at
+ */
+ protected void extraStatusAssertions(SwiftFileStatus stat) {
+
+ }
+
+ /**
+ * Asserts that a zero byte file has a status of file and not
+ * directory or symlink
+ *
+ * @throws Exception on failures
+ */
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testMultiByteFilesAreFiles() throws Exception {
+ Path src = path("/test/testMultiByteFilesAreFiles");
+ SwiftTestUtils.writeTextFile(fs, src, "testMultiByteFilesAreFiles", false);
+ assertIsFile(src);
+ FileStatus status = fs.getFileStatus(src);
+ assertFalse(status.isDir());
+ }
+
+}
Added: hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemExtendedContract.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemExtendedContract.java?rev=1526848&view=auto
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemExtendedContract.java (added)
+++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemExtendedContract.java Fri Sep 27 11:12:42 2013
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.swift.http.RestClientBindings;
+import org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem;
+import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
+import org.apache.hadoop.io.IOUtils;
+import org.junit.Test;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.URI;
+import java.util.Locale;
+
+public class TestSwiftFileSystemExtendedContract extends SwiftFileSystemBaseTest {
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testOpenNonExistingFile() throws IOException {
+ final Path p = new Path("/test/testOpenNonExistingFile");
+ //open it as a file, should get FileNotFoundException
+ try {
+ final FSDataInputStream in = fs.open(p);
+ in.close();
+ fail("didn't expect to get here");
+ } catch (FileNotFoundException fnfe) {
+ LOG.debug("Expected: " + fnfe, fnfe);
+ }
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testFilesystemHasURI() throws Throwable {
+ assertNotNull(fs.getUri());
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testCreateFile() throws Exception {
+ final Path f = new Path("/test/testCreateFile");
+ final FSDataOutputStream fsDataOutputStream = fs.create(f);
+ fsDataOutputStream.close();
+ assertExists("created file", f);
+ }
+
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testWriteReadFile() throws Exception {
+ final Path f = new Path("/test/test");
+ final FSDataOutputStream fsDataOutputStream = fs.create(f);
+ final String message = "Test string";
+ fsDataOutputStream.write(message.getBytes());
+ fsDataOutputStream.close();
+ assertExists("created file", f);
+ FSDataInputStream open = null;
+ try {
+ open = fs.open(f);
+ final byte[] bytes = new byte[512];
+ final int read = open.read(bytes);
+ final byte[] buffer = new byte[read];
+ System.arraycopy(bytes, 0, buffer, 0, read);
+ assertEquals(message, new String(buffer));
+ } finally {
+ fs.delete(f, false);
+ IOUtils.closeStream(open);
+ }
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testConfDefinesFilesystem() throws Throwable {
+ Configuration conf = new Configuration();
+ SwiftTestUtils.getServiceURI(conf);
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testConfIsValid() throws Throwable {
+ Configuration conf = new Configuration();
+ URI fsURI = SwiftTestUtils.getServiceURI(conf);
+ RestClientBindings.bind(fsURI, conf);
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testGetSchemeImplemented() throws Throwable {
+ String scheme = fs.getScheme();
+ assertEquals(SwiftNativeFileSystem.SWIFT,scheme);
+ }
+
+ /**
+ * Assert that a filesystem is case sensitive.
+ * This is done by creating a mixed-case filename and asserting that
+ * its lower case version is not there.
+ *
+ * @throws Exception failures
+ */
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testFilesystemIsCaseSensitive() throws Exception {
+ String mixedCaseFilename = "/test/UPPER.TXT";
+ Path upper = path(mixedCaseFilename);
+ Path lower = path(mixedCaseFilename.toLowerCase(Locale.ENGLISH));
+ assertFalse("File exists" + upper, fs.exists(upper));
+ assertFalse("File exists" + lower, fs.exists(lower));
+ FSDataOutputStream out = fs.create(upper);
+ out.writeUTF("UPPER");
+ out.close();
+ FileStatus upperStatus = fs.getFileStatus(upper);
+ assertExists("Original upper case file" + upper, upper);
+ //verify the lower-case version of the filename doesn't exist
+ assertPathDoesNotExist("lower case file", lower);
+ //now overwrite the lower case version of the filename with a
+ //new version.
+ out = fs.create(lower);
+ out.writeUTF("l");
+ out.close();
+ assertExists("lower case file", lower);
+ //verifEy the length of the upper file hasn't changed
+ assertExists("Original upper case file " + upper, upper);
+ FileStatus newStatus = fs.getFileStatus(upper);
+ assertEquals("Expected status:" + upperStatus
+ + " actual status " + newStatus,
+ upperStatus.getLen(),
+ newStatus.getLen());
+ }
+
+}
Added: hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemLsOperations.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemLsOperations.java?rev=1526848&view=auto
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemLsOperations.java (added)
+++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemLsOperations.java Fri Sep 27 11:12:42 2013
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.assertListStatusFinds;
+import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.cleanup;
+import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.dumpStats;
+import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.touch;
+
+/**
+ * Test the FileSystem#listStatus() operations
+ */
+public class TestSwiftFileSystemLsOperations extends SwiftFileSystemBaseTest {
+
+ private Path[] testDirs;
+
+ /**
+ * Setup creates dirs under test/hadoop
+ *
+ * @throws Exception
+ */
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ //delete the test directory
+ Path test = path("/test");
+ fs.delete(test, true);
+ mkdirs(test);
+ }
+
+ /**
+ * Create subdirectories and files under test/ for those tests
+ * that want them. Doing so adds overhead to setup and teardown,
+ * so should only be done for those tests that need them.
+ * @throws IOException on an IO problem
+ */
+ private void createTestSubdirs() throws IOException {
+ testDirs = new Path[]{
+ path("/test/hadoop/a"),
+ path("/test/hadoop/b"),
+ path("/test/hadoop/c/1"),
+ };
+
+ assertPathDoesNotExist("test directory setup", testDirs[0]);
+ for (Path path : testDirs) {
+ mkdirs(path);
+ }
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testListLevelTest() throws Exception {
+ createTestSubdirs();
+ FileStatus[] paths = fs.listStatus(path("/test"));
+ assertEquals(dumpStats("/test", paths), 1, paths.length);
+ assertEquals(path("/test/hadoop"), paths[0].getPath());
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testListLevelTestHadoop() throws Exception {
+ createTestSubdirs();
+ FileStatus[] paths;
+ paths = fs.listStatus(path("/test/hadoop"));
+ String stats = dumpStats("/test/hadoop", paths);
+ assertEquals("Paths.length wrong in " + stats, 3, paths.length);
+ assertEquals("Path element[0] wrong: " + stats, path("/test/hadoop/a"),
+ paths[0].getPath());
+ assertEquals("Path element[1] wrong: " + stats, path("/test/hadoop/b"),
+ paths[1].getPath());
+ assertEquals("Path element[2] wrong: " + stats, path("/test/hadoop/c"),
+ paths[2].getPath());
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testListStatusEmptyDirectory() throws Exception {
+ createTestSubdirs();
+ FileStatus[] paths;
+ paths = fs.listStatus(path("/test/hadoop/a"));
+ assertEquals(dumpStats("/test/hadoop/a", paths), 0,
+ paths.length);
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testListStatusFile() throws Exception {
+ describe("Create a single file under /test;" +
+ " assert that listStatus(/test) finds it");
+ Path file = path("/test/filename");
+ createFile(file);
+ FileStatus[] pathStats = fs.listStatus(file);
+ assertEquals(dumpStats("/test/", pathStats),
+ 1,
+ pathStats.length);
+ //and assert that the len of that ls'd path is the same as the original
+ FileStatus lsStat = pathStats[0];
+ assertEquals("Wrong file len in listing of " + lsStat,
+ data.length, lsStat.getLen());
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testListEmptyRoot() throws Throwable {
+ describe("Empty the root dir and verify that an LS / returns {}");
+ cleanup("testListEmptyRoot", fs, "/test");
+ cleanup("testListEmptyRoot", fs, "/user");
+ FileStatus[] fileStatuses = fs.listStatus(path("/"));
+ assertEquals("Non-empty root" + dumpStats("/", fileStatuses),
+ 0,
+ fileStatuses.length);
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testListNonEmptyRoot() throws Throwable {
+ Path test = path("/test");
+ touch(fs, test);
+ FileStatus[] fileStatuses = fs.listStatus(path("/"));
+ String stats = dumpStats("/", fileStatuses);
+ assertEquals("Wrong #of root children" + stats, 1, fileStatuses.length);
+ FileStatus status = fileStatuses[0];
+ assertEquals("Wrong path value" + stats,test, status.getPath());
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testListStatusRootDir() throws Throwable {
+ Path dir = path("/");
+ Path child = path("/test");
+ touch(fs, child);
+ assertListStatusFinds(fs, dir, child);
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testListStatusFiltered() throws Throwable {
+ Path dir = path("/");
+ Path child = path("/test");
+ touch(fs, child);
+ FileStatus[] stats = fs.listStatus(dir, new AcceptAllFilter());
+ boolean found = false;
+ StringBuilder builder = new StringBuilder();
+ for (FileStatus stat : stats) {
+ builder.append(stat.toString()).append('\n');
+ if (stat.getPath().equals(child)) {
+ found = true;
+ }
+ }
+ assertTrue("Path " + child
+ + " not found in directory " + dir + ":" + builder,
+ found);
+ }
+
+}
Added: hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemPartitionedUploads.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemPartitionedUploads.java?rev=1526848&view=auto
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemPartitionedUploads.java (added)
+++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemPartitionedUploads.java Fri Sep 27 11:12:42 2013
@@ -0,0 +1,442 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.swift;
+
+import org.apache.commons.httpclient.Header;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.swift.http.SwiftProtocolConstants;
+import org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem;
+import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
+import org.apache.hadoop.fs.swift.util.SwiftUtils;
+import org.apache.hadoop.io.IOUtils;
+import org.junit.Test;
+import org.junit.internal.AssumptionViolatedException;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+
+import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.assertPathExists;
+import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.readDataset;
+
+/**
+ * Test partitioned uploads.
+ * This is done by forcing a very small partition size and verifying that it
+ * is picked up.
+ */
+public class TestSwiftFileSystemPartitionedUploads extends
+ SwiftFileSystemBaseTest {
+
+ public static final String WRONG_PARTITION_COUNT =
+ "wrong number of partitions written into ";
+ public static final int PART_SIZE = 1;
+ public static final int PART_SIZE_BYTES = PART_SIZE * 1024;
+ public static final int BLOCK_SIZE = 1024;
+ private URI uri;
+
+ @Override
+ protected Configuration createConfiguration() {
+ Configuration conf = super.createConfiguration();
+ //set the partition size to 1 KB
+ conf.setInt(SwiftProtocolConstants.SWIFT_PARTITION_SIZE, PART_SIZE);
+ return conf;
+ }
+
+ @Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
+ public void testPartitionPropertyPropagatesToConf() throws Throwable {
+ assertEquals(1,
+ getConf().getInt(SwiftProtocolConstants.SWIFT_PARTITION_SIZE,
+ 0));
+ }
+
+ @Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
+ public void testPartionPropertyPropagatesToStore() throws Throwable {
+ assertEquals(1, fs.getStore().getPartsizeKB());
+ }
+
+ /**
+ * tests functionality for big files ( > 5Gb) upload
+ */
+ @Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
+ public void testFilePartUpload() throws Throwable {
+
+ final Path path = new Path("/test/testFilePartUpload");
+
+ int len = 8192;
+ final byte[] src = SwiftTestUtils.dataset(len, 32, 144);
+ FSDataOutputStream out = fs.create(path,
+ false,
+ getBufferSize(),
+ (short) 1,
+ BLOCK_SIZE);
+
+ try {
+ int totalPartitionsToWrite = len / PART_SIZE_BYTES;
+ assertPartitionsWritten("Startup", out, 0);
+ //write 2048
+ int firstWriteLen = 2048;
+ out.write(src, 0, firstWriteLen);
+ //assert
+ long expected = getExpectedPartitionsWritten(firstWriteLen,
+ PART_SIZE_BYTES,
+ false);
+ SwiftUtils.debug(LOG, "First write: predict %d partitions written",
+ expected);
+ assertPartitionsWritten("First write completed", out, expected);
+ //write the rest
+ int remainder = len - firstWriteLen;
+ SwiftUtils.debug(LOG, "remainder: writing: %d bytes", remainder);
+
+ out.write(src, firstWriteLen, remainder);
+ expected =
+ getExpectedPartitionsWritten(len, PART_SIZE_BYTES, false);
+ assertPartitionsWritten("Remaining data", out, expected);
+ out.close();
+ expected =
+ getExpectedPartitionsWritten(len, PART_SIZE_BYTES, true);
+ assertPartitionsWritten("Stream closed", out, expected);
+
+ Header[] headers = fs.getStore().getObjectHeaders(path, true);
+ for (Header header : headers) {
+ LOG.info(header.toString());
+ }
+
+ byte[] dest = readDataset(fs, path, len);
+ LOG.info("Read dataset from " + path + ": data length =" + len);
+ //compare data
+ SwiftTestUtils.compareByteArrays(src, dest, len);
+ FileStatus status;
+
+ final Path qualifiedPath = path.makeQualified(fs);
+ status = fs.getFileStatus(qualifiedPath);
+ //now see what block location info comes back.
+ //This will vary depending on the Swift version, so the results
+ //aren't checked -merely that the test actually worked
+ BlockLocation[] locations = fs.getFileBlockLocations(status, 0, len);
+ assertNotNull("Null getFileBlockLocations()", locations);
+ assertTrue("empty array returned for getFileBlockLocations()",
+ locations.length > 0);
+
+ //last bit of test -which seems to play up on partitions, which we download
+ //to a skip
+ try {
+ validatePathLen(path, len);
+ } catch (AssertionError e) {
+ //downgrade to a skip
+ throw new AssumptionViolatedException(e, null);
+ }
+
+ } finally {
+ IOUtils.closeStream(out);
+ }
+ }
+ /**
+ * tests functionality for big files ( > 5Gb) upload
+ */
+ @Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
+ public void testFilePartUploadNoLengthCheck() throws IOException, URISyntaxException {
+
+ final Path path = new Path("/test/testFilePartUploadLengthCheck");
+
+ int len = 8192;
+ final byte[] src = SwiftTestUtils.dataset(len, 32, 144);
+ FSDataOutputStream out = fs.create(path,
+ false,
+ getBufferSize(),
+ (short) 1,
+ BLOCK_SIZE);
+
+ try {
+ int totalPartitionsToWrite = len / PART_SIZE_BYTES;
+ assertPartitionsWritten("Startup", out, 0);
+ //write 2048
+ int firstWriteLen = 2048;
+ out.write(src, 0, firstWriteLen);
+ //assert
+ long expected = getExpectedPartitionsWritten(firstWriteLen,
+ PART_SIZE_BYTES,
+ false);
+ SwiftUtils.debug(LOG, "First write: predict %d partitions written",
+ expected);
+ assertPartitionsWritten("First write completed", out, expected);
+ //write the rest
+ int remainder = len - firstWriteLen;
+ SwiftUtils.debug(LOG, "remainder: writing: %d bytes", remainder);
+
+ out.write(src, firstWriteLen, remainder);
+ expected =
+ getExpectedPartitionsWritten(len, PART_SIZE_BYTES, false);
+ assertPartitionsWritten("Remaining data", out, expected);
+ out.close();
+ expected =
+ getExpectedPartitionsWritten(len, PART_SIZE_BYTES, true);
+ assertPartitionsWritten("Stream closed", out, expected);
+
+ Header[] headers = fs.getStore().getObjectHeaders(path, true);
+ for (Header header : headers) {
+ LOG.info(header.toString());
+ }
+
+ byte[] dest = readDataset(fs, path, len);
+ LOG.info("Read dataset from " + path + ": data length =" + len);
+ //compare data
+ SwiftTestUtils.compareByteArrays(src, dest, len);
+ FileStatus status = fs.getFileStatus(path);
+
+ //now see what block location info comes back.
+ //This will vary depending on the Swift version, so the results
+ //aren't checked -merely that the test actually worked
+ BlockLocation[] locations = fs.getFileBlockLocations(status, 0, len);
+ assertNotNull("Null getFileBlockLocations()", locations);
+ assertTrue("empty array returned for getFileBlockLocations()",
+ locations.length > 0);
+ } finally {
+ IOUtils.closeStream(out);
+ }
+ }
+
+ private FileStatus validatePathLen(Path path, int len) throws IOException {
+ //verify that the length is what was written in a direct status check
+ final Path qualifiedPath = path.makeQualified(fs);
+ FileStatus[] parentDirListing = fs.listStatus(qualifiedPath.getParent());
+ StringBuilder listing = lsToString(parentDirListing);
+ String parentDirLS = listing.toString();
+ FileStatus status = fs.getFileStatus(qualifiedPath);
+ assertEquals("Length of written file " + qualifiedPath
+ + " from status check " + status
+ + " in dir " + listing,
+ len,
+ status.getLen());
+ String fileInfo = qualifiedPath + " " + status;
+ assertFalse("File claims to be a directory " + fileInfo,
+ status.isDir());
+
+ FileStatus listedFileStat = resolveChild(parentDirListing, qualifiedPath);
+ assertNotNull("Did not find " + path + " in " + parentDirLS,
+ listedFileStat);
+ //file is in the parent dir. Now validate it's stats
+ assertEquals("Wrong len for " + path + " in listing " + parentDirLS,
+ len,
+ listedFileStat.getLen());
+ listedFileStat.toString();
+ return status;
+ }
+
+ private FileStatus resolveChild(FileStatus[] parentDirListing,
+ Path childPath) {
+ FileStatus listedFileStat = null;
+ for (FileStatus stat : parentDirListing) {
+ if (stat.getPath().equals(childPath)) {
+ listedFileStat = stat;
+ }
+ }
+ return listedFileStat;
+ }
+
+ private StringBuilder lsToString(FileStatus[] parentDirListing) {
+ StringBuilder listing = new StringBuilder();
+ for (FileStatus stat : parentDirListing) {
+ listing.append(stat).append("\n");
+ }
+ return listing;
+ }
+
+ /**
+ * Calculate the #of partitions expected from the upload
+ * @param uploaded number of bytes uploaded
+ * @param partSizeBytes the partition size
+ * @param closed whether or not the stream has closed
+ * @return the expected number of partitions, for use in assertions.
+ */
+ private int getExpectedPartitionsWritten(long uploaded,
+ int partSizeBytes,
+ boolean closed) {
+ //#of partitions in total
+ int partitions = (int) (uploaded / partSizeBytes);
+ //#of bytes past the last partition
+ int remainder = (int) (uploaded % partSizeBytes);
+ if (closed) {
+ //all data is written, so if there was any remainder, it went up
+ //too
+ return partitions + ((remainder > 0) ? 1 : 0);
+ } else {
+ //not closed. All the remainder is buffered,
+ return partitions;
+ }
+ }
+
+ private int getBufferSize() {
+ return fs.getConf().getInt("io.file.buffer.size", 4096);
+ }
+
+ /**
+ * Test sticks up a very large partitioned file and verifies that
+ * it comes back unchanged.
+ * @throws Throwable
+ */
+ @Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
+ public void testManyPartitionedFile() throws Throwable {
+ final Path path = new Path("/test/testManyPartitionedFile");
+
+ int len = PART_SIZE_BYTES * 15;
+ final byte[] src = SwiftTestUtils.dataset(len, 32, 144);
+ FSDataOutputStream out = fs.create(path,
+ false,
+ getBufferSize(),
+ (short) 1,
+ BLOCK_SIZE);
+
+ out.write(src, 0, src.length);
+ int expected =
+ getExpectedPartitionsWritten(len, PART_SIZE_BYTES, true);
+ out.close();
+ assertPartitionsWritten("write completed", out, expected);
+ assertEquals("too few bytes written", len,
+ SwiftNativeFileSystem.getBytesWritten(out));
+ assertEquals("too few bytes uploaded", len,
+ SwiftNativeFileSystem.getBytesUploaded(out));
+ //now we verify that the data comes back. If it
+ //doesn't, it means that the ordering of the partitions
+ //isn't right
+ byte[] dest = readDataset(fs, path, len);
+ //compare data
+ SwiftTestUtils.compareByteArrays(src, dest, len);
+ //finally, check the data
+ FileStatus[] stats = fs.listStatus(path);
+ assertEquals("wrong entry count in "
+ + SwiftTestUtils.dumpStats(path.toString(), stats),
+ expected, stats.length);
+ }
+
+ /**
+ * Test that when a partitioned file is overwritten by a smaller one,
+ * all the old partitioned files go away
+ * @throws Throwable
+ */
+ @Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
+ public void testOverwritePartitionedFile() throws Throwable {
+ final Path path = new Path("/test/testOverwritePartitionedFile");
+
+ final int len1 = 8192;
+ final byte[] src1 = SwiftTestUtils.dataset(len1, 'A', 'Z');
+ FSDataOutputStream out = fs.create(path,
+ false,
+ getBufferSize(),
+ (short) 1,
+ 1024);
+ out.write(src1, 0, len1);
+ out.close();
+ long expected = getExpectedPartitionsWritten(len1,
+ PART_SIZE_BYTES,
+ false);
+ assertPartitionsWritten("initial upload", out, expected);
+ assertExists("Exists", path);
+ FileStatus status = fs.getFileStatus(path);
+ assertEquals("Length", len1, status.getLen());
+ //now write a shorter file with a different dataset
+ final int len2 = 4095;
+ final byte[] src2 = SwiftTestUtils.dataset(len2, 'a', 'z');
+ out = fs.create(path,
+ true,
+ getBufferSize(),
+ (short) 1,
+ 1024);
+ out.write(src2, 0, len2);
+ out.close();
+ status = fs.getFileStatus(path);
+ assertEquals("Length", len2, status.getLen());
+ byte[] dest = readDataset(fs, path, len2);
+ //compare data
+ SwiftTestUtils.compareByteArrays(src2, dest, len2);
+ }
+
+ @Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
+ public void testDeleteSmallPartitionedFile() throws Throwable {
+ final Path path = new Path("/test/testDeleteSmallPartitionedFile");
+
+ final int len1 = 1024;
+ final byte[] src1 = SwiftTestUtils.dataset(len1, 'A', 'Z');
+ SwiftTestUtils.writeDataset(fs, path, src1, len1, 1024, false);
+ assertExists("Exists", path);
+
+ Path part_0001 = new Path(path, SwiftUtils.partitionFilenameFromNumber(1));
+ Path part_0002 = new Path(path, SwiftUtils.partitionFilenameFromNumber(2));
+ String ls = SwiftTestUtils.ls(fs, path);
+ assertExists("Partition 0001 Exists in " + ls, part_0001);
+ assertPathDoesNotExist("partition 0002 found under " + ls, part_0002);
+ assertExists("Partition 0002 Exists in " + ls, part_0001);
+ fs.delete(path, false);
+ assertPathDoesNotExist("deleted file still there", path);
+ ls = SwiftTestUtils.ls(fs, path);
+ assertPathDoesNotExist("partition 0001 file still under " + ls, part_0001);
+ }
+
+ @Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
+ public void testDeletePartitionedFile() throws Throwable {
+ final Path path = new Path("/test/testDeletePartitionedFile");
+
+ SwiftTestUtils.writeDataset(fs, path, data, data.length, 1024, false);
+ assertExists("Exists", path);
+
+ Path part_0001 = new Path(path, SwiftUtils.partitionFilenameFromNumber(1));
+ Path part_0002 = new Path(path, SwiftUtils.partitionFilenameFromNumber(2));
+ String ls = SwiftTestUtils.ls(fs, path);
+ assertExists("Partition 0001 Exists in " + ls, part_0001);
+ assertExists("Partition 0002 Exists in " + ls, part_0001);
+ fs.delete(path, false);
+ assertPathDoesNotExist("deleted file still there", path);
+ ls = SwiftTestUtils.ls(fs, path);
+ assertPathDoesNotExist("partition 0001 file still under " + ls, part_0001);
+ assertPathDoesNotExist("partition 0002 file still under " + ls, part_0002);
+ }
+
+
+ @Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
+ public void testRenamePartitionedFile() throws Throwable {
+ Path src = new Path("/test/testRenamePartitionedFileSrc");
+
+ int len = data.length;
+ SwiftTestUtils.writeDataset(fs, src, data, len, 1024, false);
+ assertExists("Exists", src);
+
+ String partOneName = SwiftUtils.partitionFilenameFromNumber(1);
+ Path srcPart = new Path(src, partOneName);
+ Path dest = new Path("/test/testRenamePartitionedFileDest");
+ Path destPart = new Path(src, partOneName);
+ assertExists("Partition Exists", srcPart);
+ fs.rename(src, dest);
+ assertPathExists(fs, "dest file missing", dest);
+ FileStatus status = fs.getFileStatus(dest);
+ assertEquals("Length of renamed file is wrong", len, status.getLen());
+ byte[] destData = readDataset(fs, dest, len);
+ //compare data
+ SwiftTestUtils.compareByteArrays(data, destData, len);
+ String srcLs = SwiftTestUtils.ls(fs, src);
+ String destLs = SwiftTestUtils.ls(fs, dest);
+
+ assertPathDoesNotExist("deleted file still found in " + srcLs, src);
+
+ assertPathDoesNotExist("partition file still found in " + srcLs, srcPart);
+ }
+
+
+}
Added: hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemRead.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemRead.java?rev=1526848&view=auto
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemRead.java (added)
+++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemRead.java Fri Sep 27 11:12:42 2013
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift;
+
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.junit.Test;
+
+import java.io.EOFException;
+import java.io.IOException;
+
+import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.readBytesToString;
+import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.writeTextFile;
+
+/**
+ * Test filesystem read operations
+ */
+public class TestSwiftFileSystemRead extends SwiftFileSystemBaseTest {
+
+
+ /**
+ * Read past the end of a file: expect the operation to fail
+ * @throws IOException
+ */
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testOverRead() throws IOException {
+ final String message = "message";
+ final Path filePath = new Path("/test/file.txt");
+
+ writeTextFile(fs, filePath, message, false);
+
+ try {
+ readBytesToString(fs, filePath, 20);
+ fail("expected an exception");
+ } catch (EOFException e) {
+ //expected
+ }
+ }
+
+ /**
+ * Read and write some JSON
+ * @throws IOException
+ */
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testRWJson() throws IOException {
+ final String message = "{" +
+ " 'json': { 'i':43, 'b':true}," +
+ " 's':'string'" +
+ "}";
+ final Path filePath = new Path("/test/file.json");
+
+ writeTextFile(fs, filePath, message, false);
+ String readJson = readBytesToString(fs, filePath, message.length());
+ assertEquals(message,readJson);
+ //now find out where it is
+ FileStatus status = fs.getFileStatus(filePath);
+ BlockLocation[] locations = fs.getFileBlockLocations(status, 0, 10);
+ }
+
+ /**
+ * Read and write some XML
+ * @throws IOException
+ */
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testRWXML() throws IOException {
+ final String message = "<x>" +
+ " <json i='43' 'b'=true/>" +
+ " string" +
+ "</x>";
+ final Path filePath = new Path("/test/file.xml");
+
+ writeTextFile(fs, filePath, message, false);
+ String read = readBytesToString(fs, filePath, message.length());
+ assertEquals(message,read);
+ }
+
+}
Added: hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemRename.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemRename.java?rev=1526848&view=auto
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemRename.java (added)
+++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemRename.java Fri Sep 27 11:12:42 2013
@@ -0,0 +1,270 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift;
+
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.compareByteArrays;
+import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.dataset;
+import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.readBytesToString;
+import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.readDataset;
+import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.writeDataset;
+
+public class TestSwiftFileSystemRename extends SwiftFileSystemBaseTest {
+
+ /**
+ * Rename a file into a directory
+ *
+ * @throws Exception
+ */
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testRenameFileIntoExistingDirectory() throws Exception {
+ assumeRenameSupported();
+
+ Path src = path("/test/olddir/file");
+ createFile(src);
+ Path dst = path("/test/new/newdir");
+ fs.mkdirs(dst);
+ rename(src, dst, true, false, true);
+ Path newFile = path("/test/new/newdir/file");
+ if (!fs.exists(newFile)) {
+ String ls = ls(dst);
+ LOG.info(ls(path("/test/new")));
+ LOG.info(ls(path("/test/hadoop")));
+ fail("did not find " + newFile + " - directory: " + ls);
+ }
+ assertTrue("Destination changed",
+ fs.exists(path("/test/new/newdir/file")));
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testRenameFile() throws Exception {
+ assumeRenameSupported();
+
+ final Path old = new Path("/test/alice/file");
+ final Path newPath = new Path("/test/bob/file");
+ fs.mkdirs(newPath.getParent());
+ final FSDataOutputStream fsDataOutputStream = fs.create(old);
+ final byte[] message = "Some data".getBytes();
+ fsDataOutputStream.write(message);
+ fsDataOutputStream.close();
+
+ assertTrue(fs.exists(old));
+ rename(old, newPath, true, false, true);
+
+ final FSDataInputStream bobStream = fs.open(newPath);
+ final byte[] bytes = new byte[512];
+ final int read = bobStream.read(bytes);
+ bobStream.close();
+ final byte[] buffer = new byte[read];
+ System.arraycopy(bytes, 0, buffer, 0, read);
+ assertEquals(new String(message), new String(buffer));
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testRenameDirectory() throws Exception {
+ assumeRenameSupported();
+
+ final Path old = new Path("/test/data/logs");
+ final Path newPath = new Path("/test/var/logs");
+ fs.mkdirs(old);
+ fs.mkdirs(newPath.getParent());
+ assertTrue(fs.exists(old));
+ rename(old, newPath, true, false, true);
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testRenameTheSameDirectory() throws Exception {
+ assumeRenameSupported();
+
+ final Path old = new Path("/test/usr/data");
+ fs.mkdirs(old);
+ rename(old, old, false, true, true);
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testRenameDirectoryIntoExistingDirectory() throws Exception {
+ assumeRenameSupported();
+
+ Path src = path("/test/olddir/dir");
+ fs.mkdirs(src);
+ createFile(path("/test/olddir/dir/file1"));
+ createFile(path("/test/olddir/dir/subdir/file2"));
+
+ Path dst = path("/test/new/newdir");
+ fs.mkdirs(dst);
+ //this renames into a child
+ rename(src, dst, true, false, true);
+ assertExists("new dir", path("/test/new/newdir/dir"));
+ assertExists("Renamed nested file1", path("/test/new/newdir/dir/file1"));
+ assertPathDoesNotExist("Nested file1 should have been deleted",
+ path("/test/olddir/dir/file1"));
+ assertExists("Renamed nested subdir",
+ path("/test/new/newdir/dir/subdir/"));
+ assertExists("file under subdir",
+ path("/test/new/newdir/dir/subdir/file2"));
+
+ assertPathDoesNotExist("Nested /test/hadoop/dir/subdir/file2 still exists",
+ path("/test/olddir/dir/subdir/file2"));
+ }
+
+ /**
+ * trying to rename a directory onto itself should fail,
+ * preserving everything underneath.
+ */
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testRenameDirToSelf() throws Throwable {
+ assumeRenameSupported();
+ Path parentdir = path("/test/parentdir");
+ fs.mkdirs(parentdir);
+ Path child = new Path(parentdir, "child");
+ createFile(child);
+
+ rename(parentdir, parentdir, false, true, true);
+ //verify the child is still there
+ assertIsFile(child);
+ }
+
+ /**
+ * Assert that root directory renames are not allowed
+ *
+ * @throws Exception on failures
+ */
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testRenameRootDirForbidden() throws Exception {
+ assumeRenameSupported();
+ rename(path("/"),
+ path("/test/newRootDir"),
+ false, true, false);
+ }
+
+ /**
+ * Assert that renaming a parent directory to be a child
+ * of itself is forbidden
+ *
+ * @throws Exception on failures
+ */
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testRenameChildDirForbidden() throws Exception {
+ assumeRenameSupported();
+
+ Path parentdir = path("/test/parentdir");
+ fs.mkdirs(parentdir);
+ Path childFile = new Path(parentdir, "childfile");
+ createFile(childFile);
+ //verify one level down
+ Path childdir = new Path(parentdir, "childdir");
+ rename(parentdir, childdir, false, true, false);
+ //now another level
+ fs.mkdirs(childdir);
+ Path childchilddir = new Path(childdir, "childdir");
+ rename(parentdir, childchilddir, false, true, false);
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testRenameFileAndVerifyContents() throws IOException {
+ assumeRenameSupported();
+
+ final Path filePath = new Path("/test/home/user/documents/file.txt");
+ final Path newFilePath = new Path("/test/home/user/files/file.txt");
+ mkdirs(newFilePath.getParent());
+ int len = 1024;
+ byte[] dataset = dataset(len, 'A', 26);
+ writeDataset(fs, filePath, dataset, len, len, false);
+ rename(filePath, newFilePath, true, false, true);
+ byte[] dest = readDataset(fs, newFilePath, len);
+ compareByteArrays(dataset, dest, len);
+ String reread = readBytesToString(fs, newFilePath, 20);
+ }
+
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testMoveFileUnderParent() throws Throwable {
+ if (!renameSupported()) return;
+ Path filepath = path("test/file");
+ createFile(filepath);
+ //HDFS expects rename src, src -> true
+ rename(filepath, filepath, true, true, true);
+ //verify the file is still there
+ assertIsFile(filepath);
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testMoveDirUnderParent() throws Throwable {
+ if (!renameSupported()) {
+ return;
+ }
+ Path testdir = path("test/dir");
+ fs.mkdirs(testdir);
+ Path parent = testdir.getParent();
+ //the outcome here is ambiguous, so is not checked
+ fs.rename(testdir, parent);
+ assertExists("Source directory has been deleted ", testdir);
+ }
+
+ /**
+ * trying to rename a file onto itself should succeed (it's a no-op)
+ */
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testRenameFileToSelf() throws Throwable {
+ if (!renameSupported()) return;
+ Path filepath = path("test/file");
+ createFile(filepath);
+ //HDFS expects rename src, src -> true
+ rename(filepath, filepath, true, true, true);
+ //verify the file is still there
+ assertIsFile(filepath);
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testRenamedConsistence() throws IOException {
+ assumeRenameSupported();
+ describe("verify that overwriting a file with new data doesn't impact" +
+ " the existing content");
+
+ final Path filePath = new Path("/test/home/user/documents/file.txt");
+ final Path newFilePath = new Path("/test/home/user/files/file.txt");
+ mkdirs(newFilePath.getParent());
+ int len = 1024;
+ byte[] dataset = dataset(len, 'A', 26);
+ byte[] dataset2 = dataset(len, 'a', 26);
+ writeDataset(fs, filePath, dataset, len, len, false);
+ rename(filePath, newFilePath, true, false, true);
+ SwiftTestUtils.writeAndRead(fs, filePath, dataset2, len, len, false, true);
+ byte[] dest = readDataset(fs, newFilePath, len);
+ compareByteArrays(dataset, dest, len);
+ String reread = readBytesToString(fs, newFilePath, 20);
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testRenameMissingFile() throws Throwable {
+ assumeRenameSupported();
+ Path path = path("/test/RenameMissingFile");
+ Path path2 = path("/test/RenameMissingFileDest");
+ mkdirs(path("test"));
+ rename(path, path2, false, false, false);
+ }
+
+}
Added: hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftObjectPath.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftObjectPath.java?rev=1526848&view=auto
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftObjectPath.java (added)
+++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftObjectPath.java Fri Sep 27 11:12:42 2013
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.swift;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.swift.http.RestClientBindings;
+import org.apache.hadoop.fs.swift.http.SwiftRestClient;
+import org.apache.hadoop.fs.swift.util.SwiftObjectPath;
+import org.apache.hadoop.fs.swift.util.SwiftUtils;
+import org.junit.Test;
+
+import java.net.URI;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Unit tests for SwiftObjectPath class.
+ */
+public class TestSwiftObjectPath implements SwiftTestConstants {
+ private static final Log LOG = LogFactory.getLog(TestSwiftObjectPath.class);
+
+ /**
+ * What an endpoint looks like. This is derived from a (valid)
+ * rackspace endpoint address
+ */
+ private static final String ENDPOINT =
+ "https://storage101.region1.example.org/v1/MossoCloudFS_9fb40cc0-1234-5678-9abc-def000c9a66";
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testParsePath() throws Exception {
+ final String pathString = "/home/user/files/file1";
+ final Path path = new Path(pathString);
+ final URI uri = new URI("http://container.localhost");
+ final SwiftObjectPath expected = SwiftObjectPath.fromPath(uri, path);
+ final SwiftObjectPath actual = new SwiftObjectPath(
+ RestClientBindings.extractContainerName(uri),
+ pathString);
+
+ assertEquals(expected, actual);
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testParseUrlPath() throws Exception {
+ final String pathString = "swift://container.service1/home/user/files/file1";
+ final URI uri = new URI(pathString);
+ final Path path = new Path(pathString);
+ final SwiftObjectPath expected = SwiftObjectPath.fromPath(uri, path);
+ final SwiftObjectPath actual = new SwiftObjectPath(
+ RestClientBindings.extractContainerName(uri),
+ "/home/user/files/file1");
+
+ assertEquals(expected, actual);
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testParseAuthenticatedUrl() throws Exception {
+ final String pathString = "swift://container.service1/v2/AUTH_00345h34l93459y4/home/tom/documents/finance.docx";
+ final URI uri = new URI(pathString);
+ final Path path = new Path(pathString);
+ final SwiftObjectPath expected = SwiftObjectPath.fromPath(uri, path);
+ final SwiftObjectPath actual = new SwiftObjectPath(
+ RestClientBindings.extractContainerName(uri),
+ "/home/tom/documents/finance.docx");
+
+ assertEquals(expected, actual);
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testConvertToPath() throws Throwable {
+ String initialpath = "/dir/file1";
+ Path ipath = new Path(initialpath);
+ SwiftObjectPath objectPath = SwiftObjectPath.fromPath(new URI(initialpath),
+ ipath);
+ URI endpoint = new URI(ENDPOINT);
+ URI uri = SwiftRestClient.pathToURI(objectPath, endpoint);
+ LOG.info("Inital Hadoop Path =" + initialpath);
+ LOG.info("Merged URI=" + uri);
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testRootDirProbeEmptyPath() throws Throwable {
+ SwiftObjectPath object=new SwiftObjectPath("container","");
+ assertTrue(SwiftUtils.isRootDir(object));
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testRootDirProbeRootPath() throws Throwable {
+ SwiftObjectPath object=new SwiftObjectPath("container","/");
+ assertTrue(SwiftUtils.isRootDir(object));
+ }
+
+ private void assertParentOf(SwiftObjectPath p1, SwiftObjectPath p2) {
+ assertTrue(p1.toString() + " is not a parent of " + p2 ,p1.isEqualToOrParentOf(
+ p2));
+ }
+
+ private void assertNotParentOf(SwiftObjectPath p1, SwiftObjectPath p2) {
+ assertFalse(p1.toString() + " is a parent of " + p2, p1.isEqualToOrParentOf(
+ p2));
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testChildOfProbe() throws Throwable {
+ SwiftObjectPath parent = new SwiftObjectPath("container",
+ "/parent");
+ SwiftObjectPath parent2 = new SwiftObjectPath("container",
+ "/parent2");
+ SwiftObjectPath child = new SwiftObjectPath("container",
+ "/parent/child");
+ SwiftObjectPath sibling = new SwiftObjectPath("container",
+ "/parent/sibling");
+ SwiftObjectPath grandchild = new SwiftObjectPath("container",
+ "/parent/child/grandchild");
+ assertParentOf(parent, child);
+ assertParentOf(parent, grandchild);
+ assertParentOf(child, grandchild);
+ assertParentOf(parent, parent);
+ assertNotParentOf(child, parent);
+ assertParentOf(child, child);
+ assertNotParentOf(parent, parent2);
+ assertNotParentOf(grandchild, parent);
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testChildOfRoot() throws Throwable {
+ SwiftObjectPath root = new SwiftObjectPath("container", "/");
+ SwiftObjectPath child = new SwiftObjectPath("container", "child");
+ SwiftObjectPath grandchild = new SwiftObjectPath("container",
+ "/child/grandchild");
+ assertParentOf(root, child);
+ assertParentOf(root, grandchild);
+ assertParentOf(child, grandchild);
+ assertParentOf(root, root);
+ assertNotParentOf(child, root);
+ assertParentOf(child, child);
+ assertNotParentOf(grandchild, root);
+ }
+
+}
Added: hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/hdfs2/TestSwiftFileSystemDirectoriesHdfs2.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/hdfs2/TestSwiftFileSystemDirectoriesHdfs2.java?rev=1526848&view=auto
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/hdfs2/TestSwiftFileSystemDirectoriesHdfs2.java (added)
+++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/hdfs2/TestSwiftFileSystemDirectoriesHdfs2.java Fri Sep 27 11:12:42 2013
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.hdfs2;
+
+import org.apache.hadoop.fs.swift.TestSwiftFileSystemDirectories;
+import org.apache.hadoop.fs.swift.snative.SwiftFileStatus;
+
+/**
+ * Add some HDFS-2 only assertions to {@link TestSwiftFileSystemDirectories}
+ */
+public class TestSwiftFileSystemDirectoriesHdfs2 extends
+ TestSwiftFileSystemDirectories {
+
+
+ /**
+ * make assertions about fields that only appear in
+ * FileStatus in HDFS2
+ * @param stat status to look at
+ */
+ protected void extraStatusAssertions(SwiftFileStatus stat) {
+ //HDFS2
+ assertTrue("isDirectory(): Not a directory: " + stat, stat.isDirectory());
+ assertFalse("isFile(): declares itself a file: " + stat, stat.isFile());
+ assertFalse("isFile(): declares itself a file: " + stat, stat.isSymlink());
+ }
+
+}
Added: hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/hdfs2/TestV2LsOperations.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/hdfs2/TestV2LsOperations.java?rev=1526848&view=auto
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/hdfs2/TestV2LsOperations.java (added)
+++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/hdfs2/TestV2LsOperations.java Fri Sep 27 11:12:42 2013
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.hdfs2;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.fs.swift.SwiftFileSystemBaseTest;
+import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
+import org.junit.Test;
+
+import java.io.IOException;
+
+public class TestV2LsOperations extends SwiftFileSystemBaseTest {
+
+ private Path[] testDirs;
+
+ /**
+ * Setup creates dirs under test/hadoop
+ * @throws Exception
+ */
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ //delete the test directory
+ Path test = path("/test");
+ fs.delete(test, true);
+ mkdirs(test);
+ }
+
+ /**
+ * Create subdirectories and files under test/ for those tests
+ * that want them. Doing so adds overhead to setup and teardown,
+ * so should only be done for those tests that need them.
+ * @throws IOException on an IO problem
+ */
+ private void createTestSubdirs() throws IOException {
+ testDirs = new Path[]{
+ path("/test/hadoop/a"),
+ path("/test/hadoop/b"),
+ path("/test/hadoop/c/1"),
+ };
+ assertPathDoesNotExist("test directory setup", testDirs[0]);
+ for (Path path : testDirs) {
+ mkdirs(path);
+ }
+ }
+
+ /**
+ * To get this project to compile under Hadoop 1, this code needs to be
+ * commented out
+ *
+ *
+ * @param fs filesystem
+ * @param dir dir
+ * @param subdir subdir
+ * @param recursive recurse?
+ * @throws IOException IO problems
+ */
+ public static void assertListFilesFinds(FileSystem fs,
+ Path dir,
+ Path subdir,
+ boolean recursive) throws IOException {
+ RemoteIterator<LocatedFileStatus> iterator =
+ fs.listFiles(dir, recursive);
+ boolean found = false;
+ int entries = 0;
+ StringBuilder builder = new StringBuilder();
+ while (iterator.hasNext()) {
+ LocatedFileStatus next = iterator.next();
+ entries++;
+ builder.append(next.toString()).append('\n');
+ if (next.getPath().equals(subdir)) {
+ found = true;
+ }
+ }
+ assertTrue("Path " + subdir
+ + " not found in directory " + dir + " : "
+ + " entries=" + entries
+ + " content"
+ + builder.toString(),
+ found);
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testListFilesRootDir() throws Throwable {
+ Path dir = path("/");
+ Path child = new Path(dir, "test");
+ fs.delete(child, true);
+ SwiftTestUtils.writeTextFile(fs, child, "text", false);
+ assertListFilesFinds(fs, dir, child, false);
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testListFilesSubDir() throws Throwable {
+ createTestSubdirs();
+ Path dir = path("/test");
+ Path child = new Path(dir, "text.txt");
+ SwiftTestUtils.writeTextFile(fs, child, "text", false);
+ assertListFilesFinds(fs, dir, child, false);
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testListFilesRecursive() throws Throwable {
+ createTestSubdirs();
+ Path dir = path("/test");
+ Path child = new Path(dir, "hadoop/a/a.txt");
+ SwiftTestUtils.writeTextFile(fs, child, "text", false);
+ assertListFilesFinds(fs, dir, child, true);
+ }
+
+}
Added: hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/http/TestRestClientBindings.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/http/TestRestClientBindings.java?rev=1526848&view=auto
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/http/TestRestClientBindings.java (added)
+++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/http/TestRestClientBindings.java Fri Sep 27 11:12:42 2013
@@ -0,0 +1,198 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.http;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.swift.SwiftTestConstants;
+import org.apache.hadoop.fs.swift.exceptions.SwiftConfigurationException;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.Properties;
+
+import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.DOT_AUTH_URL;
+import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.DOT_PASSWORD;
+import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.DOT_USERNAME;
+import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.SWIFT_AUTH_PROPERTY;
+import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.SWIFT_CONTAINER_PROPERTY;
+import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.SWIFT_HTTPS_PORT_PROPERTY;
+import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.SWIFT_HTTP_PORT_PROPERTY;
+import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.SWIFT_PASSWORD_PROPERTY;
+import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.SWIFT_REGION_PROPERTY;
+import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.SWIFT_SERVICE_PROPERTY;
+import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.SWIFT_TENANT_PROPERTY;
+import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.SWIFT_USERNAME_PROPERTY;
+import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.assertPropertyEquals;
+
+public class TestRestClientBindings extends Assert
+ implements SwiftTestConstants {
+
+ private static final String SERVICE = "sname";
+ private static final String CONTAINER = "cname";
+ private static final String FS_URI = "swift://"
+ + CONTAINER + "." + SERVICE + "/";
+ private static final String AUTH_URL = "http://localhost:8080/auth";
+ private static final String USER = "user";
+ private static final String PASS = "pass";
+ private static final String TENANT = "tenant";
+ private URI filesysURI;
+ private Configuration conf;
+
+ @Before
+ public void setup() throws URISyntaxException {
+ filesysURI = new URI(FS_URI);
+ conf = new Configuration(true);
+ setInstanceVal(conf, SERVICE, DOT_AUTH_URL, AUTH_URL);
+ setInstanceVal(conf, SERVICE, DOT_USERNAME, USER);
+ setInstanceVal(conf, SERVICE, DOT_PASSWORD, PASS);
+ }
+
+ private void setInstanceVal(Configuration conf,
+ String host,
+ String key,
+ String val) {
+ String instance = RestClientBindings.buildSwiftInstancePrefix(host);
+ String confkey = instance
+ + key;
+ conf.set(confkey, val);
+ }
+
+ public void testPrefixBuilder() throws Throwable {
+ String built = RestClientBindings.buildSwiftInstancePrefix(SERVICE);
+ assertEquals("fs.swift.service." + SERVICE, built);
+ }
+
+ public void testBindAgainstConf() throws Exception {
+ Properties props = RestClientBindings.bind(filesysURI, conf);
+ assertPropertyEquals(props, SWIFT_CONTAINER_PROPERTY, CONTAINER);
+ assertPropertyEquals(props, SWIFT_SERVICE_PROPERTY, SERVICE);
+ assertPropertyEquals(props, SWIFT_AUTH_PROPERTY, AUTH_URL);
+ assertPropertyEquals(props, SWIFT_AUTH_PROPERTY, AUTH_URL);
+ assertPropertyEquals(props, SWIFT_USERNAME_PROPERTY, USER);
+ assertPropertyEquals(props, SWIFT_PASSWORD_PROPERTY, PASS);
+
+ assertPropertyEquals(props, SWIFT_TENANT_PROPERTY, null);
+ assertPropertyEquals(props, SWIFT_REGION_PROPERTY, null);
+ assertPropertyEquals(props, SWIFT_HTTP_PORT_PROPERTY, null);
+ assertPropertyEquals(props, SWIFT_HTTPS_PORT_PROPERTY, null);
+ }
+
+ public void expectBindingFailure(URI fsURI, Configuration config) {
+ try {
+ Properties binding = RestClientBindings.bind(fsURI, config);
+ //if we get here, binding didn't fail- there is something else.
+ //list the properties but not the values.
+ StringBuilder details = new StringBuilder() ;
+ for (Object key: binding.keySet()) {
+ details.append(key.toString()).append(" ");
+ }
+ fail("Expected a failure, got the binding [ "+ details+"]");
+ } catch (SwiftConfigurationException expected) {
+
+ }
+ }
+
+ public void testBindAgainstConfMissingInstance() throws Exception {
+ Configuration badConf = new Configuration();
+ expectBindingFailure(filesysURI, badConf);
+ }
+
+
+/* Hadoop 2.x+ only, as conf.unset() isn't a v1 feature
+ public void testBindAgainstConfIncompleteInstance() throws Exception {
+ String instance = RestClientBindings.buildSwiftInstancePrefix(SERVICE);
+ conf.unset(instance + DOT_PASSWORD);
+ expectBindingFailure(filesysURI, conf);
+ }
+*/
+
+ @Test(expected = SwiftConfigurationException.class)
+ public void testDottedServiceURL() throws Exception {
+ RestClientBindings.bind(new URI("swift://hadoop.apache.org/"), conf);
+ }
+
+ @Test(expected = SwiftConfigurationException.class)
+ public void testMissingServiceURL() throws Exception {
+ RestClientBindings.bind(new URI("swift:///"), conf);
+ }
+
+ /**
+ * inner test method that expects container extraction to fail
+ * -if not prints a meaningful error message.
+ *
+ * @param hostname hostname to parse
+ */
+ private static void expectExtractContainerFail(String hostname) {
+ try {
+ String container = RestClientBindings.extractContainerName(hostname);
+ fail("Expected an error -got a container of '" + container
+ + "' from " + hostname);
+ } catch (SwiftConfigurationException expected) {
+ //expected
+ }
+ }
+
+ /**
+ * inner test method that expects service extraction to fail
+ * -if not prints a meaningful error message.
+ *
+ * @param hostname hostname to parse
+ */
+ public static void expectExtractServiceFail(String hostname) {
+ try {
+ String service = RestClientBindings.extractServiceName(hostname);
+ fail("Expected an error -got a service of '" + service
+ + "' from " + hostname);
+ } catch (SwiftConfigurationException expected) {
+ //expected
+ }
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testEmptyHostname() throws Throwable {
+ expectExtractContainerFail("");
+ expectExtractServiceFail("");
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testDot() throws Throwable {
+ expectExtractContainerFail(".");
+ expectExtractServiceFail(".");
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testSimple() throws Throwable {
+ expectExtractContainerFail("simple");
+ expectExtractServiceFail("simple");
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testTrailingDot() throws Throwable {
+ expectExtractServiceFail("simple.");
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testLeadingDot() throws Throwable {
+ expectExtractServiceFail(".leading");
+ }
+
+}
Added: hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/http/TestSwiftRestClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/http/TestSwiftRestClient.java?rev=1526848&view=auto
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/http/TestSwiftRestClient.java (added)
+++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/http/TestSwiftRestClient.java Fri Sep 27 11:12:42 2013
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.http;
+
+import org.apache.commons.httpclient.Header;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.swift.SwiftTestConstants;
+import org.apache.hadoop.fs.swift.util.Duration;
+import org.apache.hadoop.fs.swift.util.DurationStats;
+import org.apache.hadoop.fs.swift.util.SwiftObjectPath;
+import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
+import org.junit.Assert;
+import org.junit.Assume;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.ByteArrayInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.URI;
+
+public class TestSwiftRestClient implements SwiftTestConstants {
+ private static final Log LOG =
+ LogFactory.getLog(TestSwiftRestClient.class);
+
+ private Configuration conf;
+ private boolean runTests;
+ private URI serviceURI;
+
+ @Before
+ public void setup() throws IOException {
+ conf = new Configuration();
+ runTests = SwiftTestUtils.hasServiceURI(conf);
+ if (runTests) {
+ serviceURI = SwiftTestUtils.getServiceURI(conf);
+ }
+ }
+
+ protected void assumeEnabled() {
+ Assume.assumeTrue(runTests);
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testCreate() throws Throwable {
+ assumeEnabled();
+ SwiftRestClient client = createClient();
+ }
+
+ private SwiftRestClient createClient() throws IOException {
+ return SwiftRestClient.getInstance(serviceURI, conf);
+ }
+
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testAuthenticate() throws Throwable {
+ assumeEnabled();
+ SwiftRestClient client = createClient();
+ client.authenticate();
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testPutAndDelete() throws Throwable {
+ assumeEnabled();
+ SwiftRestClient client = createClient();
+ client.authenticate();
+ Path path = new Path("restTestPutAndDelete");
+ SwiftObjectPath sobject = SwiftObjectPath.fromPath(serviceURI, path);
+ byte[] stuff = new byte[1];
+ stuff[0] = 'a';
+ client.upload(sobject, new ByteArrayInputStream(stuff), stuff.length);
+ //check file exists
+ Duration head = new Duration();
+ Header[] responseHeaders = client.headRequest("expect success",
+ sobject,
+ SwiftRestClient.NEWEST);
+ head.finished();
+ LOG.info("head request duration " + head);
+ for (Header header: responseHeaders) {
+ LOG.info(header.toString());
+ }
+ //delete the file
+ client.delete(sobject);
+ //check file is gone
+ try {
+ Header[] headers = client.headRequest("expect fail",
+ sobject,
+ SwiftRestClient.NEWEST);
+ Assert.fail("Expected deleted file, but object is still present: "
+ + sobject);
+ } catch (FileNotFoundException e) {
+ //expected
+ }
+ for (DurationStats stats: client.getOperationStatistics()) {
+ LOG.info(stats);
+ }
+ }
+
+}
Added: hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/scale/SwiftScaleTestBase.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/scale/SwiftScaleTestBase.java?rev=1526848&view=auto
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/scale/SwiftScaleTestBase.java (added)
+++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/scale/SwiftScaleTestBase.java Fri Sep 27 11:12:42 2013
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.scale;
+
+import org.apache.hadoop.fs.swift.SwiftFileSystemBaseTest;
+
+/**
+ * Base class for scale tests; here is where the common scale configuration
+ * keys are defined
+ */
+
+public class SwiftScaleTestBase extends SwiftFileSystemBaseTest {
+
+ public static final String SCALE_TEST = "scale.test.";
+ public static final String KEY_OPERATION_COUNT = SCALE_TEST + "operation.count";
+ public static final long DEFAULT_OPERATION_COUNT = 10;
+
+ protected long getOperationCount() {
+ return getConf().getLong(KEY_OPERATION_COUNT, DEFAULT_OPERATION_COUNT);
+ }
+}
Added: hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/scale/TestWriteManySmallFiles.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/scale/TestWriteManySmallFiles.java?rev=1526848&view=auto
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/scale/TestWriteManySmallFiles.java (added)
+++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/scale/TestWriteManySmallFiles.java Fri Sep 27 11:12:42 2013
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.scale;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.swift.util.Duration;
+import org.apache.hadoop.fs.swift.util.DurationStats;
+import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
+import org.junit.Test;
+
+public class TestWriteManySmallFiles extends SwiftScaleTestBase {
+
+ public static final Log LOG = LogFactory.getLog(TestWriteManySmallFiles.class);
+
+ @Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
+ public void testScaledWriteThenRead() throws Throwable {
+ Path dir = new Path("/test/manysmallfiles");
+ Duration rm1 = new Duration();
+ fs.delete(dir, true);
+ rm1.finished();
+ fs.mkdirs(dir);
+ Duration ls1 = new Duration();
+ fs.listStatus(dir);
+ ls1.finished();
+ long count = getOperationCount();
+ SwiftTestUtils.noteAction("Beginning Write of "+ count + " files ");
+ DurationStats writeStats = new DurationStats("write");
+ DurationStats readStats = new DurationStats("read");
+ String format = "%08d";
+ for (long l = 0; l < count; l++) {
+ String name = String.format(format, l);
+ Path p = new Path(dir, "part-" + name);
+ Duration d = new Duration();
+ SwiftTestUtils.writeTextFile(fs, p, name, false);
+ d.finished();
+ writeStats.add(d);
+ Thread.sleep(1000);
+ }
+ //at this point, the directory is full.
+ SwiftTestUtils.noteAction("Beginning ls");
+
+ Duration ls2 = new Duration();
+ FileStatus[] status2 = (FileStatus[]) fs.listStatus(dir);
+ ls2.finished();
+ assertEquals("Not enough entries in the directory", count, status2.length);
+
+ SwiftTestUtils.noteAction("Beginning read");
+
+ for (long l = 0; l < count; l++) {
+ String name = String.format(format, l);
+ Path p = new Path(dir, "part-" + name);
+ Duration d = new Duration();
+ String result = SwiftTestUtils.readBytesToString(fs, p, name.length());
+ assertEquals(name, result);
+ d.finished();
+ readStats.add(d);
+ }
+ //do a recursive delete
+ SwiftTestUtils.noteAction("Beginning delete");
+ Duration rm2 = new Duration();
+ fs.delete(dir, true);
+ rm2.finished();
+ //print the stats
+ LOG.info(String.format("'filesystem','%s'",fs.getUri()));
+ LOG.info(writeStats.toString());
+ LOG.info(readStats.toString());
+ LOG.info(String.format(
+ "'rm1',%d,'ls1',%d",
+ rm1.value(),
+ ls1.value()));
+ LOG.info(String.format(
+ "'rm2',%d,'ls2',%d",
+ rm2.value(),
+ ls2.value()));
+ }
+
+}
Added: hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/test/resources/core-site.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/test/resources/core-site.xml?rev=1526848&view=auto
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/test/resources/core-site.xml (added)
+++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/test/resources/core-site.xml Fri Sep 27 11:12:42 2013
@@ -0,0 +1,51 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<!-- Values used when running unit tests. This is mostly empty, to -->
+<!-- use of the default values, overriding the potentially -->
+<!-- user-editted core-site.xml in the conf/ directory. -->
+
+<configuration>
+
+
+ <property>
+ <name>hadoop.tmp.dir</name>
+ <value>target/build/test</value>
+ <description>A base for other temporary directories.</description>
+ <final>true</final>
+ </property>
+
+ <!-- Turn security off for tests by default -->
+ <property>
+ <name>hadoop.security.authentication</name>
+ <value>simple</value>
+ </property>
+
+ <!--
+ To run these tests.
+
+ # Create a file auth-keys.xml - DO NOT ADD TO REVISION CONTROL
+ # add the property test.fs.swift.name to point to a swift filesystem URL
+ # Add the credentials for the service you are testing against
+ -->
+ <include xmlns="http://www.w3.org/2001/XInclude"
+ href="auth-keys.xml"/>
+
+</configuration>