You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by xy...@apache.org on 2020/03/07 01:33:12 UTC
[hadoop-ozone] branch HDDS-2665-ofs updated: HDDS-2929. Implement
ofs://: temp directory mount (#610)
This is an automated email from the ASF dual-hosted git repository.
xyao pushed a commit to branch HDDS-2665-ofs
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
The following commit(s) were added to refs/heads/HDDS-2665-ofs by this push:
new c410c4d HDDS-2929. Implement ofs://: temp directory mount (#610)
c410c4d is described below
commit c410c4d329a839547bbf9491b5bf4562efcda59e
Author: Siyao Meng <50...@users.noreply.github.com>
AuthorDate: Fri Mar 6 17:33:01 2020 -0800
HDDS-2929. Implement ofs://: temp directory mount (#610)
---
.../src/main/compose/ozonesecure/docker-config | 1 +
.../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 115 +++++++++++++++++----
.../ozone/BasicRootedOzoneClientAdapterImpl.java | 6 +-
.../java/org/apache/hadoop/fs/ozone/OFSPath.java | 48 +++++++--
.../org/apache/hadoop/fs/ozone/TestOFSPath.java | 22 ++--
5 files changed, 159 insertions(+), 33 deletions(-)
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config
index b8ddc97..0570b46 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config
@@ -14,6 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+CORE-SITE.XML_fs.ofs.impl=org.apache.hadoop.fs.ozone.RootedOzoneFileSystem
CORE-SITE.XML_fs.o3fs.impl=org.apache.hadoop.fs.ozone.OzoneFileSystem
OZONE-SITE.XML_ozone.om.address=om
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
index a881f10..b0550e1 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
@@ -29,12 +29,19 @@ import org.apache.hadoop.fs.contract.ContractTestUtils;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.TestDataUtil;
import org.apache.hadoop.ozone.client.ObjectStore;
import org.apache.hadoop.ozone.client.OzoneBucket;
import org.apache.hadoop.ozone.client.OzoneKeyDetails;
import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.VolumeArgs;
+import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLIdentityType;
+import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType;
+import org.apache.hadoop.ozone.security.acl.OzoneAclConfig;
import org.apache.hadoop.test.GenericTestUtils;
import org.junit.After;
import org.junit.Assert;
@@ -45,6 +52,7 @@ import org.junit.rules.Timeout;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
@@ -53,8 +61,10 @@ import java.util.TreeSet;
import java.util.stream.Collectors;
import static org.apache.hadoop.fs.ozone.Constants.LISTING_PAGE_SIZE;
+import static org.apache.hadoop.ozone.OzoneAcl.AclScope.ACCESS;
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.BUCKET_NOT_FOUND;
/**
* Ozone file system tests that are not covered by contract tests.
@@ -65,10 +75,11 @@ public class TestRootedOzoneFileSystem {
@Rule
public Timeout globalTimeout = new Timeout(300_000);
- private static MiniOzoneCluster cluster = null;
- private static FileSystem fs;
- private static RootedOzoneFileSystem ofs;
- private static ObjectStore objectStore;
+ private OzoneConfiguration conf;
+ private MiniOzoneCluster cluster = null;
+ private FileSystem fs;
+ private RootedOzoneFileSystem ofs;
+ private ObjectStore objectStore;
private static BasicRootedOzoneClientAdapterImpl adapter;
private String volumeName;
@@ -79,7 +90,7 @@ public class TestRootedOzoneFileSystem {
@Before
public void init() throws Exception {
- OzoneConfiguration conf = new OzoneConfiguration();
+ conf = new OzoneConfiguration();
cluster = MiniOzoneCluster.newBuilder(conf)
.setNumDatanodes(3)
.build();
@@ -117,12 +128,13 @@ public class TestRootedOzoneFileSystem {
@Test
public void testOzoneFsServiceLoader() throws IOException {
- OzoneConfiguration conf = new OzoneConfiguration();
+ OzoneConfiguration confTestLoader = new OzoneConfiguration();
// Note: FileSystem#loadFileSystems won't load OFS class due to META-INF
// hence this workaround.
- conf.set("fs.ofs.impl", "org.apache.hadoop.fs.ozone.RootedOzoneFileSystem");
- Assert.assertEquals(
- FileSystem.getFileSystemClass(OzoneConsts.OZONE_OFS_URI_SCHEME, conf),
+ confTestLoader.set("fs.ofs.impl",
+ "org.apache.hadoop.fs.ozone.RootedOzoneFileSystem");
+ Assert.assertEquals(FileSystem.getFileSystemClass(
+ OzoneConsts.OZONE_OFS_URI_SCHEME, confTestLoader),
RootedOzoneFileSystem.class);
}
@@ -661,32 +673,97 @@ public class TestRootedOzoneFileSystem {
// FileSystem because we can't change LISTING_PAGE_SIZE. Use adapter instead
// numEntries > 5
- FileStatus[] fileStatusesOver = customListStatus(
- new Path("/"), false, "", 8);
+ FileStatus[] fileStatusesOver = customListStatus(new Path("/"),
+ false, "", 8);
// There are only 5 volumes
Assert.assertEquals(5, fileStatusesOver.length);
// numEntries = 5
- FileStatus[] fileStatusesExact = customListStatus(
- new Path("/"), false, "", 5);
+ FileStatus[] fileStatusesExact = customListStatus(new Path("/"),
+ false, "", 5);
Assert.assertEquals(5, fileStatusesExact.length);
// numEntries < 5
- FileStatus[] fileStatusesLimit1 = customListStatus(
- new Path("/"), false, "", 3);
+ FileStatus[] fileStatusesLimit1 = customListStatus(new Path("/"),
+ false, "", 3);
// Should only return 3 volumes even though there are more than that due to
// the specified limit
Assert.assertEquals(3, fileStatusesLimit1.length);
// Get the last entry in the list as startPath
- String nextStartPath = fileStatusesLimit1[fileStatusesLimit1.length - 1]
- .getPath().toString();
- FileStatus[] fileStatusesLimit2 = customListStatus(
- new Path("/"), false, nextStartPath, 3);
+ String nextStartPath =
+ fileStatusesLimit1[fileStatusesLimit1.length - 1].getPath().toString();
+ FileStatus[] fileStatusesLimit2 = customListStatus(new Path("/"),
+ false, nextStartPath, 3);
// Note: at the time of writing this test, OmMetadataManagerImpl#listVolumes
// excludes startVolume (startPath) from the result. Might change.
Assert.assertEquals(fileStatusesOver.length,
fileStatusesLimit1.length + fileStatusesLimit2.length);
}
+ /*
+ * OFS: Test /tmp mount behavior.
+ */
+ @Test
+ public void testTempMount() throws Exception {
+ // Prep
+ // Use ClientProtocol to pass in volume ACL, ObjectStore won't do it
+ ClientProtocol proxy = objectStore.getClientProxy();
+ // Get default acl rights for user
+ OzoneAclConfig aclConfig = conf.getObject(OzoneAclConfig.class);
+ ACLType userRights = aclConfig.getUserDefaultRights();
+ // Construct ACL for world access
+ OzoneAcl aclWorldAccess = new OzoneAcl(ACLIdentityType.WORLD, "",
+ userRights, ACCESS);
+ // Construct VolumeArgs
+ VolumeArgs volumeArgs = new VolumeArgs.Builder()
+ .setAcls(Collections.singletonList(aclWorldAccess)).build();
+ // Sanity check
+ Assert.assertNull(volumeArgs.getOwner());
+ Assert.assertNull(volumeArgs.getAdmin());
+ Assert.assertNull(volumeArgs.getQuota());
+ Assert.assertEquals(0, volumeArgs.getMetadata().size());
+ Assert.assertEquals(1, volumeArgs.getAcls().size());
+ // Create volume "tmp" with world access. allow non-admin to create buckets
+ proxy.createVolume(OFSPath.OFS_MOUNT_TMP_VOLUMENAME, volumeArgs);
+
+ OzoneVolume vol = objectStore.getVolume(OFSPath.OFS_MOUNT_TMP_VOLUMENAME);
+ Assert.assertNotNull(vol);
+
+ // Begin test
+ String hashedUsername = OFSPath.getTempMountBucketNameOfCurrentUser();
+
+ // Expect failure since temp bucket for current user is not created yet
+ try {
+ vol.getBucket(hashedUsername);
+ } catch (OMException ex) {
+ // Expect BUCKET_NOT_FOUND
+ if (!ex.getResult().equals(BUCKET_NOT_FOUND)) {
+ Assert.fail("Temp bucket for current user shouldn't have been created");
+ }
+ }
+
+ // Write under /tmp/, OFS will create the temp bucket if not exist
+ fs.mkdirs(new Path("/tmp/dir1"));
+
+ try (FSDataOutputStream stream = ofs.create(new Path("/tmp/dir1/file1"))) {
+ stream.write(1);
+ }
+
+ // Verify temp bucket creation
+ OzoneBucket bucket = vol.getBucket(hashedUsername);
+ Assert.assertNotNull(bucket);
+ // Verify dir1 creation
+ FileStatus[] fileStatuses = fs.listStatus(new Path("/tmp/"));
+ Assert.assertEquals(1, fileStatuses.length);
+ Assert.assertEquals(
+ "/tmp/dir1", fileStatuses[0].getPath().toUri().getPath());
+ // Verify file1 creation
+ FileStatus[] fileStatusesInDir1 =
+ fs.listStatus(new Path("/tmp/dir1"));
+ Assert.assertEquals(1, fileStatusesInDir1.length);
+ Assert.assertEquals("/tmp/dir1/file1",
+ fileStatusesInDir1[0].getPath().toUri().getPath());
+ }
+
}
diff --git a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
index 3baba25..64f8581 100644
--- a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
+++ b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
@@ -213,6 +213,7 @@ public class BasicRootedOzoneClientAdapterImpl
try {
bucket = proxy.getBucketDetails(volumeStr, bucketStr);
} catch (OMException ex) {
+ // Note: always create bucket if volumeStr matches "tmp" so -put works
if (createIfNotExist) {
// Note: getBucketDetails always throws BUCKET_NOT_FOUND, even if
// the volume doesn't exist.
@@ -292,7 +293,8 @@ public class BasicRootedOzoneClientAdapterImpl
OFSPath ofsPath = new OFSPath(pathStr);
String key = ofsPath.getKeyName();
try {
- OzoneBucket bucket = getBucket(ofsPath, false);
+ // Hadoop CopyCommands class always sets recursive to true
+ OzoneBucket bucket = getBucket(ofsPath, recursive);
OzoneOutputStream ozoneOutputStream = bucket.createFile(
key, 0, replicationType, replicationFactor, overWrite, recursive);
return new OzoneFSOutputStream(ozoneOutputStream.getOutputStream());
@@ -458,6 +460,8 @@ public class BasicRootedOzoneClientAdapterImpl
} catch (OMException e) {
if (e.getResult() == OMException.ResultCodes.FILE_NOT_FOUND) {
throw new FileNotFoundException(key + ": No such file or directory!");
+ } else if (e.getResult() == OMException.ResultCodes.BUCKET_NOT_FOUND) {
+ throw new FileNotFoundException(key + ": Bucket doesn't exist!");
}
throw e;
}
diff --git a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OFSPath.java b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OFSPath.java
index c9ae6a7..f7d4ded 100644
--- a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OFSPath.java
+++ b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OFSPath.java
@@ -17,13 +17,17 @@
*/
package org.apache.hadoop.fs.ozone;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import org.apache.commons.codec.digest.DigestUtils;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.http.ParseException;
import org.apache.yetus.audience.InterfaceAudience;
import org.apache.yetus.audience.InterfaceStability;
-
import java.net.URI;
import java.net.URISyntaxException;
+import java.io.IOException;
import java.util.StringTokenizer;
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
@@ -44,7 +48,7 @@ class OFSPath {
* /vol1/buc2/dir3/key4 vol1 buc2 (empty) dir3/key4
* /vol1/buc2 vol1 buc2 (empty) (empty)
* /vol1 vol1 (empty) (empty) (empty)
- * /tmp/dir3/key4 tempVolume tempBucket tmp dir3/key4
+ * /tmp/dir3/key4 tmp <username> tmp dir3/key4
*
* Note the leading '/' doesn't matter.
*/
@@ -53,6 +57,9 @@ class OFSPath {
private String mountName = "";
private String keyName = "";
private static final String OFS_MOUNT_NAME_TMP = "tmp";
+ // Hard-code the volume name to tmp for the first implementation
+ @VisibleForTesting
+ static final String OFS_MOUNT_TMP_VOLUMENAME = "tmp";
OFSPath(Path path) {
String pathStr = path.toUri().getPath();
@@ -83,10 +90,15 @@ class OFSPath {
// TODO: Compare a keyword list instead for future expansion.
if (firstToken.equals(OFS_MOUNT_NAME_TMP)) {
mountName = firstToken;
- // TODO: Retrieve volume and bucket of the mount from user protobuf.
- // Leave them hard-coded just for now. Will be addressed in HDDS-2929
- volumeName = "tempVolume";
- bucketName = "tempBucket";
+ // TODO: In the future, may retrieve volume and bucket from
+ // UserVolumeInfo on the server side. TBD.
+ volumeName = OFS_MOUNT_TMP_VOLUMENAME;
+ try {
+ bucketName = getTempMountBucketNameOfCurrentUser();
+ } catch (IOException ex) {
+ throw new ParseException(
+ "Failed to get temp bucket name for current user.");
+ }
} else if (numToken >= 2) {
// Regular volume and bucket path
volumeName = firstToken;
@@ -95,7 +107,6 @@ class OFSPath {
// Volume only
volumeName = firstToken;
}
-// } else { // TODO: Implement '/' case for ls.
}
// Compose key name
@@ -174,4 +185,27 @@ class OFSPath {
public boolean isVolume() {
return this.getBucketName().isEmpty() && !this.getVolumeName().isEmpty();
}
+
+ /**
+ * Get the bucket name of temp for given username.
+ * @param username Input user name String. Mustn't be null.
+ * @return Username MD5 hash in hex digits.
+ */
+ @VisibleForTesting
+ static String getTempMountBucketName(String username) {
+ Preconditions.checkNotNull(username);
+ // TODO: Improve this to "slugify(username)-md5(username)" for better
+ // readability?
+ return DigestUtils.md5Hex(username);
+ }
+
+ /**
+ * Get the bucket name of temp for the current user from UserGroupInformation.
+ * @return Username MD5 hash in hex digits.
+ * @throws IOException When UserGroupInformation.getCurrentUser() fails.
+ */
+ static String getTempMountBucketNameOfCurrentUser() throws IOException {
+ String username = UserGroupInformation.getCurrentUser().getUserName();
+ return getTempMountBucketName(username);
+ }
}
diff --git a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOFSPath.java b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOFSPath.java
index 05e1ce2..c46c09f 100644
--- a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOFSPath.java
+++ b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOFSPath.java
@@ -20,6 +20,8 @@ package org.apache.hadoop.fs.ozone;
import org.junit.Assert;
import org.junit.Test;
+import java.io.IOException;
+
/**
* Testing basic functions of utility class OFSPath.
*/
@@ -103,11 +105,19 @@ public class TestOFSPath {
@Test
public void testParsingMount() {
+ String bucketName;
+ try {
+ bucketName = OFSPath.getTempMountBucketNameOfCurrentUser();
+ } catch (IOException ex) {
+ Assert.fail("Failed to get the current user name, "
+ + "thus failed to get temp bucket name.");
+ bucketName = ""; // Make javac happy
+ }
// Mount only
OFSPath ofsPath = new OFSPath("/tmp/");
- // TODO: Subject to change in HDDS-2929.
- Assert.assertEquals("tempVolume", ofsPath.getVolumeName());
- Assert.assertEquals("tempBucket", ofsPath.getBucketName());
+ Assert.assertEquals(
+ OFSPath.OFS_MOUNT_TMP_VOLUMENAME, ofsPath.getVolumeName());
+ Assert.assertEquals(bucketName, ofsPath.getBucketName());
Assert.assertEquals("tmp", ofsPath.getMountName());
Assert.assertEquals("", ofsPath.getKeyName());
Assert.assertEquals("/tmp", ofsPath.getNonKeyPath());
@@ -115,9 +125,9 @@ public class TestOFSPath {
// Mount with key
ofsPath = new OFSPath("/tmp/key1");
- // TODO: Subject to change in HDDS-2929.
- Assert.assertEquals("tempVolume", ofsPath.getVolumeName());
- Assert.assertEquals("tempBucket", ofsPath.getBucketName());
+ Assert.assertEquals(
+ OFSPath.OFS_MOUNT_TMP_VOLUMENAME, ofsPath.getVolumeName());
+ Assert.assertEquals(bucketName, ofsPath.getBucketName());
Assert.assertEquals("tmp", ofsPath.getMountName());
Assert.assertEquals("key1", ofsPath.getKeyName());
Assert.assertEquals("/tmp", ofsPath.getNonKeyPath());
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org