You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@ozone.apache.org by GitBox <gi...@apache.org> on 2020/01/07 18:32:07 UTC
[GitHub] [hadoop-ozone] smengcl opened a new pull request #415: [WIP]
HDDS-2840. Implement ofs://: mkdir
smengcl opened a new pull request #415: [WIP] HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415
## What changes were proposed in this pull request?
Implement mkdir in ofs://
## What is the link to the Apache JIRA
https://issues.apache.org/jira/browse/HDDS-2840
## How was this patch tested?
This is work-in-progress. Hasn't been completely tested at the moment.
Writing new test cases and adapting some old test cases to the new FS.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] smengcl commented on a change in pull request #415:
HDDS-2840. Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
smengcl commented on a change in pull request #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#discussion_r371274576
##########
File path: hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystemWithMocks.java
##########
@@ -0,0 +1,152 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.ozone;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.OmUtils;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PowerMockIgnore;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+
+import java.net.URI;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * Ozone File system tests that are light weight and use mocks.
+ */
+@RunWith(PowerMockRunner.class)
+@PrepareForTest({ OzoneClientFactory.class, UserGroupInformation.class })
+@PowerMockIgnore("javax.management.*")
+public class TestRootedOzoneFileSystemWithMocks {
+
+ @Test
+ public void testFSUriWithHostPortOverrides() throws Exception {
+ Configuration conf = new OzoneConfiguration();
+ OzoneClient ozoneClient = mock(OzoneClient.class);
+ ObjectStore objectStore = mock(ObjectStore.class);
+ OzoneVolume volume = mock(OzoneVolume.class);
+ OzoneBucket bucket = mock(OzoneBucket.class);
+
+ when(ozoneClient.getObjectStore()).thenReturn(objectStore);
+ when(objectStore.getVolume(eq("volume1"))).thenReturn(volume);
+ when(volume.getBucket("bucket1")).thenReturn(bucket);
+
+ PowerMockito.mockStatic(OzoneClientFactory.class);
+ PowerMockito.when(OzoneClientFactory.getRpcClient(eq("local.host"),
+ eq(5899), eq(conf))).thenReturn(ozoneClient);
+
+ UserGroupInformation ugi = mock(UserGroupInformation.class);
+ PowerMockito.mockStatic(UserGroupInformation.class);
+ PowerMockito.when(UserGroupInformation.getCurrentUser()).thenReturn(ugi);
+ when(ugi.getShortUserName()).thenReturn("user1");
+
+ // Note: FileSystem#loadFileSystems doesn't load OFS class because
+ // META-INF still points to org.apache.hadoop.fs.ozone.OzoneFileSystem
+ conf.set("fs.ofs.impl", "org.apache.hadoop.fs.ozone.RootedOzoneFileSystem");
+ URI uri = new URI("ofs://local.host:5899/volume1/bucket1");
+
+ FileSystem fileSystem = FileSystem.get(uri, conf);
+ RootedOzoneFileSystem ofs = (RootedOzoneFileSystem) fileSystem;
+
+ assertEquals(ofs.getUri().getAuthority(), "local.host:5899");
+ PowerMockito.verifyStatic();
+ OzoneClientFactory.getRpcClient("local.host", 5899, conf);
+ }
+
+ @Test
+ public void testFSUriWithHostPortUnspecified() throws Exception {
+ Configuration conf = new OzoneConfiguration();
+ final int omPort = OmUtils.getOmRpcPort(conf);
+
+ OzoneClient ozoneClient = mock(OzoneClient.class);
+ ObjectStore objectStore = mock(ObjectStore.class);
+ OzoneVolume volume = mock(OzoneVolume.class);
+ OzoneBucket bucket = mock(OzoneBucket.class);
+
+ when(ozoneClient.getObjectStore()).thenReturn(objectStore);
+ when(objectStore.getVolume(eq("volume1"))).thenReturn(volume);
+ when(volume.getBucket("bucket1")).thenReturn(bucket);
+
+ PowerMockito.mockStatic(OzoneClientFactory.class);
+ PowerMockito.when(OzoneClientFactory.getRpcClient(eq("local.host"),
+ eq(omPort), eq(conf))).thenReturn(ozoneClient);
+
+ UserGroupInformation ugi = mock(UserGroupInformation.class);
+ PowerMockito.mockStatic(UserGroupInformation.class);
+ PowerMockito.when(UserGroupInformation.getCurrentUser()).thenReturn(ugi);
+ when(ugi.getShortUserName()).thenReturn("user1");
+
+ URI uri = new URI("o3fs://bucket1.volume1.local.host");
+
Review comment:
Done in 5a20f9da30a25fd40fa1a73e6b0e742f40647a94.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] xiaoyuyao commented on a change in pull request
#415: HDDS-2840. Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
xiaoyuyao commented on a change in pull request #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#discussion_r372027079
##########
File path: hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
##########
@@ -0,0 +1,694 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.ozone;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.key.KeyProvider;
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdds.client.ReplicationFactor;
+import org.apache.hadoop.hdds.client.ReplicationType;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ozone.OmUtils;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.client.OzoneKey;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
+import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
+import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenRenewer;
+
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .BUCKET_ALREADY_EXISTS;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .VOLUME_ALREADY_EXISTS;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .VOLUME_NOT_FOUND;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .BUCKET_NOT_FOUND;
+
+/**
+ * Basic Implementation of the OzoneFileSystem calls.
+ * <p>
+ * This is the minimal version which doesn't include any statistics.
+ * <p>
+ * For full featured version use OzoneClientAdapterImpl.
+ */
+public class BasicRootedOzoneClientAdapterImpl
+ implements RootedOzoneClientAdapter {
+
+ static final Logger LOG =
+ LoggerFactory.getLogger(BasicRootedOzoneClientAdapterImpl.class);
+
+ private OzoneClient ozoneClient;
+ private ClientProtocol proxy;
+ private ObjectStore objectStore;
+ private ReplicationType replicationType;
+ private ReplicationFactor replicationFactor;
+ private boolean securityEnabled;
+ private int configuredDnPort;
+
+ /**
+ * Create new OzoneClientAdapter implementation.
+ *
+ * @throws IOException In case of a problem.
+ */
+ public BasicRootedOzoneClientAdapterImpl() throws IOException {
+ this(createConf());
+ }
+
+ private static OzoneConfiguration createConf() {
+ ClassLoader contextClassLoader =
+ Thread.currentThread().getContextClassLoader();
+ Thread.currentThread().setContextClassLoader(null);
+ try {
+ return new OzoneConfiguration();
+ } finally {
+ Thread.currentThread().setContextClassLoader(contextClassLoader);
+ }
+ }
+
+ public BasicRootedOzoneClientAdapterImpl(OzoneConfiguration conf)
+ throws IOException {
+ this(null, -1, conf);
+ }
+
+ public BasicRootedOzoneClientAdapterImpl(String omHost, int omPort,
+ Configuration hadoopConf) throws IOException {
+
+ ClassLoader contextClassLoader =
+ Thread.currentThread().getContextClassLoader();
+ Thread.currentThread().setContextClassLoader(null);
+
+ try {
+ OzoneConfiguration conf = OzoneConfiguration.of(hadoopConf);
+
+ if (omHost == null && OmUtils.isServiceIdsDefined(conf)) {
+ // When the host name or service id isn't given
+ // but ozone.om.service.ids is defined, declare failure.
+
+ // This is a safety precaution that prevents the client from
+ // accidentally failing over to an unintended OM.
+ throw new IllegalArgumentException("Service ID or host name must not"
+ + " be omitted when ozone.om.service.ids is defined.");
+ }
+
+ if (omPort != -1) {
+ // When the port number is specified, perform the following check
+ if (OmUtils.isOmHAServiceId(conf, omHost)) {
+ // If omHost is a service id, it shouldn't use a port
+ throw new IllegalArgumentException("Port " + omPort +
+ " specified in URI but host '" + omHost + "' is a "
+ + "logical (HA) OzoneManager and does not use port information.");
+ }
+ } else {
+ // When port number is not specified, read it from config
+ omPort = OmUtils.getOmRpcPort(conf);
+ }
+
+ SecurityConfig secConfig = new SecurityConfig(conf);
+
+ if (secConfig.isSecurityEnabled()) {
+ this.securityEnabled = true;
+ }
+
+ String replicationTypeConf =
+ conf.get(OzoneConfigKeys.OZONE_REPLICATION_TYPE,
+ OzoneConfigKeys.OZONE_REPLICATION_TYPE_DEFAULT);
+
+ int replicationCountConf = conf.getInt(OzoneConfigKeys.OZONE_REPLICATION,
+ OzoneConfigKeys.OZONE_REPLICATION_DEFAULT);
+
+ if (OmUtils.isOmHAServiceId(conf, omHost)) {
+ // omHost is listed as one of the service ids in the config,
+ // thus we should treat omHost as omServiceId
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(omHost, conf);
+ } else if (StringUtils.isNotEmpty(omHost) && omPort != -1) {
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(omHost, omPort, conf);
+ } else {
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(conf);
+ }
+ objectStore = ozoneClient.getObjectStore();
+ proxy = objectStore.getClientProxy();
+ this.replicationType = ReplicationType.valueOf(replicationTypeConf);
+ this.replicationFactor = ReplicationFactor.valueOf(replicationCountConf);
+ this.configuredDnPort = conf.getInt(
+ OzoneConfigKeys.DFS_CONTAINER_IPC_PORT,
+ OzoneConfigKeys.DFS_CONTAINER_IPC_PORT_DEFAULT);
+ } finally {
+ Thread.currentThread().setContextClassLoader(contextClassLoader);
+ }
+ }
+
+ OzoneBucket getBucket(OFSPath ofsPath, boolean createIfNotExist)
+ throws IOException {
+
+ return getBucket(ofsPath.getVolumeName(), ofsPath.getBucketName(),
+ createIfNotExist);
+ }
+
+ /**
+ * Get OzoneBucket object to operate in.
+ * Optionally create volume and bucket if not found.
+ *
+ * @param createIfNotExist Set this to true if the caller is a write operation
+ * in order to create the volume and bucket.
+ * @throws IOException Exceptions other than OMException with result code
+ * VOLUME_NOT_FOUND or BUCKET_NOT_FOUND.
+ */
+ private OzoneBucket getBucket(String volumeStr, String bucketStr,
+ boolean createIfNotExist) throws IOException {
+
+ OzoneBucket bucket;
+ try {
+ bucket = proxy.getBucketDetails(volumeStr, bucketStr);
+ } catch (OMException ex) {
+ if (createIfNotExist) {
+ // Note: getBucketDetails always throws BUCKET_NOT_FOUND, even if
+ // the volume doesn't exist.
+ if (ex.getResult().equals(BUCKET_NOT_FOUND)) {
+ OzoneVolume volume;
+ try {
+ volume = proxy.getVolumeDetails(volumeStr);
+ } catch (OMException getVolEx) {
+ if (getVolEx.getResult().equals(VOLUME_NOT_FOUND)) {
+ // Volume doesn't exist. Create it
+ try {
+ objectStore.createVolume(volumeStr);
+ } catch (OMException newVolEx) {
+ // Ignore the case where another client created the volume
+ if (!newVolEx.getResult().equals(VOLUME_ALREADY_EXISTS)) {
+ throw newVolEx;
+ }
+ }
+ } else {
+ throw getVolEx;
+ }
+ // Try get volume again
+ volume = proxy.getVolumeDetails(volumeStr);
+ }
+ // Create the bucket
+ try {
+ volume.createBucket(bucketStr);
+ } catch (OMException newBucEx) {
+ // Ignore the case where another client created the bucket
+ if (!newBucEx.getResult().equals(BUCKET_ALREADY_EXISTS)) {
+ throw newBucEx;
+ }
+ }
+ }
+ // Try get bucket again
+ bucket = proxy.getBucketDetails(volumeStr, bucketStr);
+ } else {
+ throw ex;
+ }
+ }
+
+ return bucket;
+ }
+
+ @Override
+ public void close() throws IOException {
+ ozoneClient.close();
+ }
+
+ @Override
+ public InputStream readFile(String pathStr) throws IOException {
+ incrementCounter(Statistic.OBJECTS_READ);
+ OFSPath ofsPath = new OFSPath(pathStr);
+ String key = ofsPath.getKeyName();
+ try {
+ OzoneBucket bucket = getBucket(ofsPath, false);
+ return bucket.readFile(key).getInputStream();
+ } catch (OMException ex) {
+ if (ex.getResult() == OMException.ResultCodes.FILE_NOT_FOUND
+ || ex.getResult() == OMException.ResultCodes.NOT_A_FILE) {
+ throw new FileNotFoundException(
+ ex.getResult().name() + ": " + ex.getMessage());
+ } else {
+ throw ex;
+ }
+ }
+ }
+
+ protected void incrementCounter(Statistic objectsRead) {
+ //noop: Use OzoneClientAdapterImpl which supports statistics.
+ }
+
+ @Override
+ public OzoneFSOutputStream createFile(String pathStr, boolean overWrite,
+ boolean recursive) throws IOException {
+ incrementCounter(Statistic.OBJECTS_CREATED);
+ OFSPath ofsPath = new OFSPath(pathStr);
+ String key = ofsPath.getKeyName();
+ try {
+ OzoneBucket bucket = getBucket(ofsPath, false);
+ OzoneOutputStream ozoneOutputStream = bucket.createFile(
+ key, 0, replicationType, replicationFactor, overWrite, recursive);
+ return new OzoneFSOutputStream(ozoneOutputStream.getOutputStream());
+ } catch (OMException ex) {
+ if (ex.getResult() == OMException.ResultCodes.FILE_ALREADY_EXISTS
+ || ex.getResult() == OMException.ResultCodes.NOT_A_FILE) {
+ throw new FileAlreadyExistsException(
+ ex.getResult().name() + ": " + ex.getMessage());
+ } else {
+ throw ex;
+ }
+ }
+ }
+
+ @Override
+ public void renamePath(String path, String newPath) throws IOException {
+ incrementCounter(Statistic.OBJECTS_RENAMED);
+ OFSPath ofsPath = new OFSPath(path);
+ OFSPath ofsNewPath = new OFSPath(newPath);
+
+ // Check path and newPathName are in the same volume and same bucket.
+ // This should have been checked in BasicRootedOzoneFileSystem#rename
+ // already via regular call path unless bypassed.
+ if (!ofsPath.isInSameBucketAs(ofsNewPath)) {
+ throw new IOException("Cannot rename a key to a different bucket");
+ }
+
+ OzoneBucket bucket = getBucket(ofsPath, false);
+ String key = ofsPath.getKeyName();
+ String newKey = ofsNewPath.getKeyName();
+ bucket.renameKey(key, newKey);
+ }
+
+ /**
+ * Package-private helper function to reduce calls to getBucket().
+ * @param bucket Bucket to operate in.
+ * @param path Existing key path.
+ * @param newPath New key path.
+ * @throws IOException IOException from bucket.renameKey().
+ */
+ void renamePath(OzoneBucket bucket, String path, String newPath)
+ throws IOException {
+ incrementCounter(Statistic.OBJECTS_RENAMED);
+ OFSPath ofsPath = new OFSPath(path);
+ OFSPath ofsNewPath = new OFSPath(newPath);
+ // No same-bucket policy check here since this call path is controlled
+ String key = ofsPath.getKeyName();
+ String newKey = ofsNewPath.getKeyName();
+ bucket.renameKey(key, newKey);
+ }
+
+ /**
+ * Helper method to create an directory specified by key name in bucket.
+ *
+ * @param pathStr path to be created as directory
+ * @return true if the key is created, false otherwise
+ */
+ @Override
+ public boolean createDirectory(String pathStr) throws IOException {
+ LOG.trace("creating dir for path: {}", pathStr);
+ incrementCounter(Statistic.OBJECTS_CREATED);
+ OFSPath ofsPath = new OFSPath(pathStr);
+ // Volume name unspecified, return failure
+ if (ofsPath.getVolumeName().length() == 0) {
+ return false;
+ }
+ // Handle where only volume is specified in pathStr
+ if (ofsPath.getBucketName().length() == 0) {
+ objectStore.createVolume(ofsPath.getVolumeName());
+ return true;
+ }
+ String keyStr = ofsPath.getKeyName();
Review comment:
what is keyStr is empty(length 0)
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] smengcl commented on issue #415: HDDS-2840.
Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
smengcl commented on issue #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#issuecomment-574954076
> Thanks for working on this @smengcl. I have started reviewing the PR.
>
> In BasicRootedOzoneClientAdapterImpl, setting volume and bucket globally could lead to discrepancies.
> Let's say my client creates one OzoneClient and multiple threads share this object.
> Thread1 - createFile /vol1/bucket1/key1
> Thread2 - createFile /vol1/bucket2/key2
>
> ```
> t1 - Thread1 sets bucket = bucket1
> t2 - Thread2 sets bucket = bucket2
> t3 - Thread1 executes bucket.createFile -> This would create key1 in bucket2.
> ```
Good point. The current implementation is not thread-safe.
I believe I can just make `volume` and `bucket` local in each call to eliminate such race condition.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] xiaoyuyao commented on a change in pull request
#415: HDDS-2840. Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
xiaoyuyao commented on a change in pull request #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#discussion_r372752982
##########
File path: hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
##########
@@ -0,0 +1,694 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.ozone;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.key.KeyProvider;
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdds.client.ReplicationFactor;
+import org.apache.hadoop.hdds.client.ReplicationType;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ozone.OmUtils;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.client.OzoneKey;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
+import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
+import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenRenewer;
+
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .BUCKET_ALREADY_EXISTS;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .VOLUME_ALREADY_EXISTS;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .VOLUME_NOT_FOUND;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .BUCKET_NOT_FOUND;
+
+/**
+ * Basic Implementation of the OzoneFileSystem calls.
+ * <p>
+ * This is the minimal version which doesn't include any statistics.
+ * <p>
+ * For full featured version use OzoneClientAdapterImpl.
+ */
+public class BasicRootedOzoneClientAdapterImpl
+ implements RootedOzoneClientAdapter {
+
+ static final Logger LOG =
+ LoggerFactory.getLogger(BasicRootedOzoneClientAdapterImpl.class);
+
+ private OzoneClient ozoneClient;
+ private ClientProtocol proxy;
+ private ObjectStore objectStore;
+ private ReplicationType replicationType;
+ private ReplicationFactor replicationFactor;
+ private boolean securityEnabled;
+ private int configuredDnPort;
+
+ /**
+ * Create new OzoneClientAdapter implementation.
+ *
+ * @throws IOException In case of a problem.
+ */
+ public BasicRootedOzoneClientAdapterImpl() throws IOException {
+ this(createConf());
+ }
+
+ private static OzoneConfiguration createConf() {
+ ClassLoader contextClassLoader =
+ Thread.currentThread().getContextClassLoader();
+ Thread.currentThread().setContextClassLoader(null);
+ try {
+ return new OzoneConfiguration();
+ } finally {
+ Thread.currentThread().setContextClassLoader(contextClassLoader);
+ }
+ }
+
+ public BasicRootedOzoneClientAdapterImpl(OzoneConfiguration conf)
+ throws IOException {
+ this(null, -1, conf);
+ }
+
+ public BasicRootedOzoneClientAdapterImpl(String omHost, int omPort,
+ Configuration hadoopConf) throws IOException {
+
+ ClassLoader contextClassLoader =
+ Thread.currentThread().getContextClassLoader();
+ Thread.currentThread().setContextClassLoader(null);
+
+ try {
+ OzoneConfiguration conf = OzoneConfiguration.of(hadoopConf);
+
+ if (omHost == null && OmUtils.isServiceIdsDefined(conf)) {
+ // When the host name or service id isn't given
+ // but ozone.om.service.ids is defined, declare failure.
+
+ // This is a safety precaution that prevents the client from
+ // accidentally failing over to an unintended OM.
+ throw new IllegalArgumentException("Service ID or host name must not"
+ + " be omitted when ozone.om.service.ids is defined.");
+ }
+
+ if (omPort != -1) {
+ // When the port number is specified, perform the following check
+ if (OmUtils.isOmHAServiceId(conf, omHost)) {
+ // If omHost is a service id, it shouldn't use a port
+ throw new IllegalArgumentException("Port " + omPort +
+ " specified in URI but host '" + omHost + "' is a "
+ + "logical (HA) OzoneManager and does not use port information.");
+ }
+ } else {
+ // When port number is not specified, read it from config
+ omPort = OmUtils.getOmRpcPort(conf);
+ }
+
+ SecurityConfig secConfig = new SecurityConfig(conf);
+
+ if (secConfig.isSecurityEnabled()) {
+ this.securityEnabled = true;
+ }
+
+ String replicationTypeConf =
+ conf.get(OzoneConfigKeys.OZONE_REPLICATION_TYPE,
+ OzoneConfigKeys.OZONE_REPLICATION_TYPE_DEFAULT);
+
+ int replicationCountConf = conf.getInt(OzoneConfigKeys.OZONE_REPLICATION,
+ OzoneConfigKeys.OZONE_REPLICATION_DEFAULT);
+
+ if (OmUtils.isOmHAServiceId(conf, omHost)) {
+ // omHost is listed as one of the service ids in the config,
+ // thus we should treat omHost as omServiceId
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(omHost, conf);
+ } else if (StringUtils.isNotEmpty(omHost) && omPort != -1) {
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(omHost, omPort, conf);
+ } else {
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(conf);
+ }
+ objectStore = ozoneClient.getObjectStore();
+ proxy = objectStore.getClientProxy();
+ this.replicationType = ReplicationType.valueOf(replicationTypeConf);
+ this.replicationFactor = ReplicationFactor.valueOf(replicationCountConf);
+ this.configuredDnPort = conf.getInt(
+ OzoneConfigKeys.DFS_CONTAINER_IPC_PORT,
+ OzoneConfigKeys.DFS_CONTAINER_IPC_PORT_DEFAULT);
+ } finally {
+ Thread.currentThread().setContextClassLoader(contextClassLoader);
+ }
+ }
+
+ OzoneBucket getBucket(OFSPath ofsPath, boolean createIfNotExist)
+ throws IOException {
+
+ return getBucket(ofsPath.getVolumeName(), ofsPath.getBucketName(),
+ createIfNotExist);
+ }
+
+ /**
+ * Get OzoneBucket object to operate in.
+ * Optionally create volume and bucket if not found.
+ *
+ * @param createIfNotExist Set this to true if the caller is a write operation
+ * in order to create the volume and bucket.
+ * @throws IOException Exceptions other than OMException with result code
+ * VOLUME_NOT_FOUND or BUCKET_NOT_FOUND.
+ */
+ private OzoneBucket getBucket(String volumeStr, String bucketStr,
+ boolean createIfNotExist) throws IOException {
+
+ OzoneBucket bucket;
+ try {
+ bucket = proxy.getBucketDetails(volumeStr, bucketStr);
+ } catch (OMException ex) {
+ if (createIfNotExist) {
+ // Note: getBucketDetails always throws BUCKET_NOT_FOUND, even if
+ // the volume doesn't exist.
+ if (ex.getResult().equals(BUCKET_NOT_FOUND)) {
+ OzoneVolume volume;
+ try {
+ volume = proxy.getVolumeDetails(volumeStr);
+ } catch (OMException getVolEx) {
+ if (getVolEx.getResult().equals(VOLUME_NOT_FOUND)) {
+ // Volume doesn't exist. Create it
+ try {
+ objectStore.createVolume(volumeStr);
+ } catch (OMException newVolEx) {
+ // Ignore the case where another client created the volume
+ if (!newVolEx.getResult().equals(VOLUME_ALREADY_EXISTS)) {
+ throw newVolEx;
+ }
+ }
+ } else {
+ throw getVolEx;
+ }
+ // Try get volume again
+ volume = proxy.getVolumeDetails(volumeStr);
+ }
+ // Create the bucket
+ try {
+ volume.createBucket(bucketStr);
+ } catch (OMException newBucEx) {
+ // Ignore the case where another client created the bucket
+ if (!newBucEx.getResult().equals(BUCKET_ALREADY_EXISTS)) {
+ throw newBucEx;
+ }
+ }
+ }
+ // Try get bucket again
+ bucket = proxy.getBucketDetails(volumeStr, bucketStr);
+ } else {
+ throw ex;
+ }
+ }
+
+ return bucket;
+ }
+
+ @Override
+ public void close() throws IOException {
+ ozoneClient.close();
+ }
+
+ @Override
+ public InputStream readFile(String pathStr) throws IOException {
+ incrementCounter(Statistic.OBJECTS_READ);
+ OFSPath ofsPath = new OFSPath(pathStr);
+ String key = ofsPath.getKeyName();
+ try {
+ OzoneBucket bucket = getBucket(ofsPath, false);
+ return bucket.readFile(key).getInputStream();
+ } catch (OMException ex) {
+ if (ex.getResult() == OMException.ResultCodes.FILE_NOT_FOUND
+ || ex.getResult() == OMException.ResultCodes.NOT_A_FILE) {
+ throw new FileNotFoundException(
+ ex.getResult().name() + ": " + ex.getMessage());
+ } else {
+ throw ex;
+ }
+ }
+ }
+
+ protected void incrementCounter(Statistic objectsRead) {
+ //noop: Use OzoneClientAdapterImpl which supports statistics.
+ }
+
+ @Override
+ public OzoneFSOutputStream createFile(String pathStr, boolean overWrite,
+ boolean recursive) throws IOException {
+ incrementCounter(Statistic.OBJECTS_CREATED);
+ OFSPath ofsPath = new OFSPath(pathStr);
+ String key = ofsPath.getKeyName();
+ try {
+ OzoneBucket bucket = getBucket(ofsPath, false);
+ OzoneOutputStream ozoneOutputStream = bucket.createFile(
+ key, 0, replicationType, replicationFactor, overWrite, recursive);
+ return new OzoneFSOutputStream(ozoneOutputStream.getOutputStream());
+ } catch (OMException ex) {
+ if (ex.getResult() == OMException.ResultCodes.FILE_ALREADY_EXISTS
+ || ex.getResult() == OMException.ResultCodes.NOT_A_FILE) {
+ throw new FileAlreadyExistsException(
+ ex.getResult().name() + ": " + ex.getMessage());
+ } else {
+ throw ex;
+ }
+ }
+ }
+
+ @Override
+ public void renamePath(String path, String newPath) throws IOException {
+ incrementCounter(Statistic.OBJECTS_RENAMED);
+ OFSPath ofsPath = new OFSPath(path);
+ OFSPath ofsNewPath = new OFSPath(newPath);
+
+ // Check path and newPathName are in the same volume and same bucket.
+ // This should have been checked in BasicRootedOzoneFileSystem#rename
+ // already via regular call path unless bypassed.
+ if (!ofsPath.isInSameBucketAs(ofsNewPath)) {
+ throw new IOException("Cannot rename a key to a different bucket");
+ }
+
+ OzoneBucket bucket = getBucket(ofsPath, false);
+ String key = ofsPath.getKeyName();
+ String newKey = ofsNewPath.getKeyName();
+ bucket.renameKey(key, newKey);
+ }
+
+ /**
+ * Package-private helper function to reduce calls to getBucket().
+ * @param bucket Bucket to operate in.
+ * @param path Existing key path.
+ * @param newPath New key path.
+ * @throws IOException IOException from bucket.renameKey().
+ */
+ void renamePath(OzoneBucket bucket, String path, String newPath)
+ throws IOException {
+ incrementCounter(Statistic.OBJECTS_RENAMED);
+ OFSPath ofsPath = new OFSPath(path);
+ OFSPath ofsNewPath = new OFSPath(newPath);
+ // No same-bucket policy check here since this call path is controlled
+ String key = ofsPath.getKeyName();
+ String newKey = ofsNewPath.getKeyName();
+ bucket.renameKey(key, newKey);
+ }
+
+ /**
+ * Helper method to create an directory specified by key name in bucket.
+ *
+ * @param pathStr path to be created as directory
+ * @return true if the key is created, false otherwise
+ */
+ @Override
+ public boolean createDirectory(String pathStr) throws IOException {
+ LOG.trace("creating dir for path: {}", pathStr);
+ incrementCounter(Statistic.OBJECTS_CREATED);
+ OFSPath ofsPath = new OFSPath(pathStr);
+ // Volume name unspecified, return failure
+ if (ofsPath.getVolumeName().length() == 0) {
+ return false;
+ }
+ // Handle where only volume is specified in pathStr
+ if (ofsPath.getBucketName().length() == 0) {
+ objectStore.createVolume(ofsPath.getVolumeName());
+ return true;
+ }
+ String keyStr = ofsPath.getKeyName();
+ try {
+ OzoneBucket bucket = getBucket(ofsPath, true);
+ // if keyStr is empty, it indicates that only volume or volume+bucket is
+ // given in pathStr, so getBucket() above should've handled the creation
+ // of volume/bucket already.
+ if (keyStr != null && keyStr.length() > 0) {
+ bucket.createDirectory(keyStr);
+ }
+ } catch (OMException e) {
+ if (e.getResult() == OMException.ResultCodes.FILE_ALREADY_EXISTS) {
+ throw new FileAlreadyExistsException(e.getMessage());
+ }
+ throw e;
+ }
+ return true;
+ }
+
+ /**
+ * Helper method to delete an object specified by key name in bucket.
+ *
+ * @param path path to a key to be deleted
+ * @return true if the key is deleted, false otherwise
+ */
+ @Override
+ public boolean deleteObject(String path) {
+ LOG.trace("issuing delete for path to key: {}", path);
+ OFSPath ofsPath = new OFSPath(path);
+ String keyName = ofsPath.getKeyName();
Review comment:
Let's follow up that on separate JIRA.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] xiaoyuyao commented on a change in pull request
#415: HDDS-2840. Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
xiaoyuyao commented on a change in pull request #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#discussion_r372027079
##########
File path: hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
##########
@@ -0,0 +1,694 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.ozone;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.key.KeyProvider;
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdds.client.ReplicationFactor;
+import org.apache.hadoop.hdds.client.ReplicationType;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ozone.OmUtils;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.client.OzoneKey;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
+import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
+import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenRenewer;
+
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .BUCKET_ALREADY_EXISTS;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .VOLUME_ALREADY_EXISTS;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .VOLUME_NOT_FOUND;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .BUCKET_NOT_FOUND;
+
+/**
+ * Basic Implementation of the OzoneFileSystem calls.
+ * <p>
+ * This is the minimal version which doesn't include any statistics.
+ * <p>
+ * For full featured version use OzoneClientAdapterImpl.
+ */
+public class BasicRootedOzoneClientAdapterImpl
+ implements RootedOzoneClientAdapter {
+
+ static final Logger LOG =
+ LoggerFactory.getLogger(BasicRootedOzoneClientAdapterImpl.class);
+
+ private OzoneClient ozoneClient;
+ private ClientProtocol proxy;
+ private ObjectStore objectStore;
+ private ReplicationType replicationType;
+ private ReplicationFactor replicationFactor;
+ private boolean securityEnabled;
+ private int configuredDnPort;
+
+ /**
+ * Create new OzoneClientAdapter implementation.
+ *
+ * @throws IOException In case of a problem.
+ */
+ public BasicRootedOzoneClientAdapterImpl() throws IOException {
+ this(createConf());
+ }
+
+ private static OzoneConfiguration createConf() {
+ ClassLoader contextClassLoader =
+ Thread.currentThread().getContextClassLoader();
+ Thread.currentThread().setContextClassLoader(null);
+ try {
+ return new OzoneConfiguration();
+ } finally {
+ Thread.currentThread().setContextClassLoader(contextClassLoader);
+ }
+ }
+
+ public BasicRootedOzoneClientAdapterImpl(OzoneConfiguration conf)
+ throws IOException {
+ this(null, -1, conf);
+ }
+
+ public BasicRootedOzoneClientAdapterImpl(String omHost, int omPort,
+ Configuration hadoopConf) throws IOException {
+
+ ClassLoader contextClassLoader =
+ Thread.currentThread().getContextClassLoader();
+ Thread.currentThread().setContextClassLoader(null);
+
+ try {
+ OzoneConfiguration conf = OzoneConfiguration.of(hadoopConf);
+
+ if (omHost == null && OmUtils.isServiceIdsDefined(conf)) {
+ // When the host name or service id isn't given
+ // but ozone.om.service.ids is defined, declare failure.
+
+ // This is a safety precaution that prevents the client from
+ // accidentally failing over to an unintended OM.
+ throw new IllegalArgumentException("Service ID or host name must not"
+ + " be omitted when ozone.om.service.ids is defined.");
+ }
+
+ if (omPort != -1) {
+ // When the port number is specified, perform the following check
+ if (OmUtils.isOmHAServiceId(conf, omHost)) {
+ // If omHost is a service id, it shouldn't use a port
+ throw new IllegalArgumentException("Port " + omPort +
+ " specified in URI but host '" + omHost + "' is a "
+ + "logical (HA) OzoneManager and does not use port information.");
+ }
+ } else {
+ // When port number is not specified, read it from config
+ omPort = OmUtils.getOmRpcPort(conf);
+ }
+
+ SecurityConfig secConfig = new SecurityConfig(conf);
+
+ if (secConfig.isSecurityEnabled()) {
+ this.securityEnabled = true;
+ }
+
+ String replicationTypeConf =
+ conf.get(OzoneConfigKeys.OZONE_REPLICATION_TYPE,
+ OzoneConfigKeys.OZONE_REPLICATION_TYPE_DEFAULT);
+
+ int replicationCountConf = conf.getInt(OzoneConfigKeys.OZONE_REPLICATION,
+ OzoneConfigKeys.OZONE_REPLICATION_DEFAULT);
+
+ if (OmUtils.isOmHAServiceId(conf, omHost)) {
+ // omHost is listed as one of the service ids in the config,
+ // thus we should treat omHost as omServiceId
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(omHost, conf);
+ } else if (StringUtils.isNotEmpty(omHost) && omPort != -1) {
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(omHost, omPort, conf);
+ } else {
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(conf);
+ }
+ objectStore = ozoneClient.getObjectStore();
+ proxy = objectStore.getClientProxy();
+ this.replicationType = ReplicationType.valueOf(replicationTypeConf);
+ this.replicationFactor = ReplicationFactor.valueOf(replicationCountConf);
+ this.configuredDnPort = conf.getInt(
+ OzoneConfigKeys.DFS_CONTAINER_IPC_PORT,
+ OzoneConfigKeys.DFS_CONTAINER_IPC_PORT_DEFAULT);
+ } finally {
+ Thread.currentThread().setContextClassLoader(contextClassLoader);
+ }
+ }
+
+ OzoneBucket getBucket(OFSPath ofsPath, boolean createIfNotExist)
+ throws IOException {
+
+ return getBucket(ofsPath.getVolumeName(), ofsPath.getBucketName(),
+ createIfNotExist);
+ }
+
+ /**
+ * Get OzoneBucket object to operate in.
+ * Optionally create volume and bucket if not found.
+ *
+ * @param createIfNotExist Set this to true if the caller is a write operation
+ * in order to create the volume and bucket.
+ * @throws IOException Exceptions other than OMException with result code
+ * VOLUME_NOT_FOUND or BUCKET_NOT_FOUND.
+ */
+ private OzoneBucket getBucket(String volumeStr, String bucketStr,
+ boolean createIfNotExist) throws IOException {
+
+ OzoneBucket bucket;
+ try {
+ bucket = proxy.getBucketDetails(volumeStr, bucketStr);
+ } catch (OMException ex) {
+ if (createIfNotExist) {
+ // Note: getBucketDetails always throws BUCKET_NOT_FOUND, even if
+ // the volume doesn't exist.
+ if (ex.getResult().equals(BUCKET_NOT_FOUND)) {
+ OzoneVolume volume;
+ try {
+ volume = proxy.getVolumeDetails(volumeStr);
+ } catch (OMException getVolEx) {
+ if (getVolEx.getResult().equals(VOLUME_NOT_FOUND)) {
+ // Volume doesn't exist. Create it
+ try {
+ objectStore.createVolume(volumeStr);
+ } catch (OMException newVolEx) {
+ // Ignore the case where another client created the volume
+ if (!newVolEx.getResult().equals(VOLUME_ALREADY_EXISTS)) {
+ throw newVolEx;
+ }
+ }
+ } else {
+ throw getVolEx;
+ }
+ // Try get volume again
+ volume = proxy.getVolumeDetails(volumeStr);
+ }
+ // Create the bucket
+ try {
+ volume.createBucket(bucketStr);
+ } catch (OMException newBucEx) {
+ // Ignore the case where another client created the bucket
+ if (!newBucEx.getResult().equals(BUCKET_ALREADY_EXISTS)) {
+ throw newBucEx;
+ }
+ }
+ }
+ // Try get bucket again
+ bucket = proxy.getBucketDetails(volumeStr, bucketStr);
+ } else {
+ throw ex;
+ }
+ }
+
+ return bucket;
+ }
+
+ @Override
+ public void close() throws IOException {
+ ozoneClient.close();
+ }
+
+ @Override
+ public InputStream readFile(String pathStr) throws IOException {
+ incrementCounter(Statistic.OBJECTS_READ);
+ OFSPath ofsPath = new OFSPath(pathStr);
+ String key = ofsPath.getKeyName();
+ try {
+ OzoneBucket bucket = getBucket(ofsPath, false);
+ return bucket.readFile(key).getInputStream();
+ } catch (OMException ex) {
+ if (ex.getResult() == OMException.ResultCodes.FILE_NOT_FOUND
+ || ex.getResult() == OMException.ResultCodes.NOT_A_FILE) {
+ throw new FileNotFoundException(
+ ex.getResult().name() + ": " + ex.getMessage());
+ } else {
+ throw ex;
+ }
+ }
+ }
+
+ protected void incrementCounter(Statistic objectsRead) {
+ //noop: Use OzoneClientAdapterImpl which supports statistics.
+ }
+
+ @Override
+ public OzoneFSOutputStream createFile(String pathStr, boolean overWrite,
+ boolean recursive) throws IOException {
+ incrementCounter(Statistic.OBJECTS_CREATED);
+ OFSPath ofsPath = new OFSPath(pathStr);
+ String key = ofsPath.getKeyName();
+ try {
+ OzoneBucket bucket = getBucket(ofsPath, false);
+ OzoneOutputStream ozoneOutputStream = bucket.createFile(
+ key, 0, replicationType, replicationFactor, overWrite, recursive);
+ return new OzoneFSOutputStream(ozoneOutputStream.getOutputStream());
+ } catch (OMException ex) {
+ if (ex.getResult() == OMException.ResultCodes.FILE_ALREADY_EXISTS
+ || ex.getResult() == OMException.ResultCodes.NOT_A_FILE) {
+ throw new FileAlreadyExistsException(
+ ex.getResult().name() + ": " + ex.getMessage());
+ } else {
+ throw ex;
+ }
+ }
+ }
+
+ @Override
+ public void renamePath(String path, String newPath) throws IOException {
+ incrementCounter(Statistic.OBJECTS_RENAMED);
+ OFSPath ofsPath = new OFSPath(path);
+ OFSPath ofsNewPath = new OFSPath(newPath);
+
+ // Check path and newPathName are in the same volume and same bucket.
+ // This should have been checked in BasicRootedOzoneFileSystem#rename
+ // already via regular call path unless bypassed.
+ if (!ofsPath.isInSameBucketAs(ofsNewPath)) {
+ throw new IOException("Cannot rename a key to a different bucket");
+ }
+
+ OzoneBucket bucket = getBucket(ofsPath, false);
+ String key = ofsPath.getKeyName();
+ String newKey = ofsNewPath.getKeyName();
+ bucket.renameKey(key, newKey);
+ }
+
+ /**
+ * Package-private helper function to reduce calls to getBucket().
+ * @param bucket Bucket to operate in.
+ * @param path Existing key path.
+ * @param newPath New key path.
+ * @throws IOException IOException from bucket.renameKey().
+ */
+ void renamePath(OzoneBucket bucket, String path, String newPath)
+ throws IOException {
+ incrementCounter(Statistic.OBJECTS_RENAMED);
+ OFSPath ofsPath = new OFSPath(path);
+ OFSPath ofsNewPath = new OFSPath(newPath);
+ // No same-bucket policy check here since this call path is controlled
+ String key = ofsPath.getKeyName();
+ String newKey = ofsNewPath.getKeyName();
+ bucket.renameKey(key, newKey);
+ }
+
+ /**
+ * Helper method to create an directory specified by key name in bucket.
+ *
+ * @param pathStr path to be created as directory
+ * @return true if the key is created, false otherwise
+ */
+ @Override
+ public boolean createDirectory(String pathStr) throws IOException {
+ LOG.trace("creating dir for path: {}", pathStr);
+ incrementCounter(Statistic.OBJECTS_CREATED);
+ OFSPath ofsPath = new OFSPath(pathStr);
+ // Volume name unspecified, return failure
+ if (ofsPath.getVolumeName().length() == 0) {
+ return false;
+ }
+ // Handle where only volume is specified in pathStr
+ if (ofsPath.getBucketName().length() == 0) {
+ objectStore.createVolume(ofsPath.getVolumeName());
+ return true;
+ }
+ String keyStr = ofsPath.getKeyName();
Review comment:
what if keyStr is empty(length 0) should we check that before getBucket?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] xiaoyuyao commented on issue #415: HDDS-2840.
Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
xiaoyuyao commented on issue #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#issuecomment-580409091
Thanks @smengcl for the update. +1 on the latest change. The last comment can be addressed in followup JIRAs.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] smengcl commented on issue #415: HDDS-2840.
Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
smengcl commented on issue #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#issuecomment-578786191
> Thanks @smengcl for working on this patiently. I know you had to rebase with master multiple times.
>
> I have posted some suggestions inline.
> Apart from them, I have some general suggestions. Since this work is being done in a branch, we can work on them in later patches (before merging into master).
>
> 1. It would be good to separate OFS and O3FS related classes into separate packages. We can have fs.ozone.o3fs for o3fs and fs.ozone.ofs for OFS. The common classes can stay under fs.ozone.
> 2. The following classes have a lot of code duplication.
> a. OzoneClientAdapter and RootedOzoneClientAdapter
> b. BasicOzoneClientAdapterImpl and BasicRootedOzoneClientAdapterImpl
> c. BasicOzoneFileSystem and BasicRootedOzoneFileSystem
> d. OzoneFileSystem and RootedOzoneFileSystem
> e. OzoneClientAdapterFactory and RootedOzoneClientAdapterFactory
> I think we can avoid most of these code duplications by creating common interfaces and extending them for O3FS and OFS.
> 3. In OzoneClientAdapter and RootedOzoneClientAdapter, since all the methods are same with just method name difference or parameter name difference, can we not have some common ground for the name convention. We could explain what the parameter denotes in the Javadoc.
> 4. We are sometimes referring to the new FS as RootedOzoneFileSystem and sometimes as O3FS. Let's follow one naming convention throughout.
> 5. We should add unit tests for testing OFSPath functionality.
Thanks so much for reviewing such a large diff. I appreciate it.
All the small issues you mentioned in the code should have a commit to address them now. For some I added TODO, others may require further discussion.
1. I agree. That is what I set out to do first (I tried to duplicate the entire `ozonefs` package). But then I realized the amount of refactoring required and maven pom editing is non-trivial. So I took @arp7 's advice to simply go out and copy those core classes for now.
2. Correct. This can be solved in the process of solving (1). This might require some intense refactoring and change on o3fs side? I think we should do this later (and eventually).
3. Do you mean we should change ALL the function names in case others misuse them when switching from o3fs world? -- I'm neutral about this (either works for me). We can discuss.
4. OFS = RootedOzoneFileSystem. Well.. I tried class name like `OFSOzoneClientAdapter` and found it unnecessarily annoying. Since `FS` in `OFS` is `FileSystem` and it doesn't mean anything in some class names. Let's just define that `OFS` is the scheme prefix, and `Rooted` prefix is for the class names (suggested by @arp7 in a call).
5. I'm on it.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] hanishakoneru commented on a change in pull request
#415: HDDS-2840. Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
hanishakoneru commented on a change in pull request #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#discussion_r369328737
##########
File path: hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
##########
@@ -0,0 +1,805 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.ozone;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.CreateFlag;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.Progressable;
+import org.apache.http.client.utils.URIBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.EnumSet;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+import static org.apache.hadoop.fs.ozone.Constants.LISTING_PAGE_SIZE;
+import static org.apache.hadoop.fs.ozone.Constants.OZONE_DEFAULT_USER;
+import static org.apache.hadoop.fs.ozone.Constants.OZONE_USER_DIR;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_OFS_URI_SCHEME;
+
+/**
+ * The minimal Ozone Filesystem implementation.
+ * <p>
+ * This is a basic version which doesn't extend
+ * KeyProviderTokenIssuer and doesn't include statistics. It can be used
+ * from older hadoop version. For newer hadoop version use the full featured
+ * OFileSystem.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class BasicRootedOzoneFileSystem extends FileSystem {
+ static final Logger LOG =
+ LoggerFactory.getLogger(BasicRootedOzoneFileSystem.class);
+
+ /**
+ * The Ozone client for connecting to Ozone server.
+ */
+
+ private URI uri;
+ private String userName;
+ private Path workingDir;
+// private String gOmHost;
+// private int gOmPort;
+// private Configuration gConf;
+// private boolean gIsolatedClassloader;
+
+ private RootedOzoneClientAdapter adapter;
+// private String adapterPath;
+
+ private static final String URI_EXCEPTION_TEXT =
+ "URL should be one of the following formats: " +
+ "ofs://om-service-id/ OR " +
+ "ofs://om-host.example.com/ OR " +
+ "ofs://om-host.example.com:5678/";
Review comment:
The URL can also have the path, right?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] hanishakoneru commented on a change in pull request
#415: HDDS-2840. Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
hanishakoneru commented on a change in pull request #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#discussion_r369830598
##########
File path: hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
##########
@@ -0,0 +1,805 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.ozone;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.CreateFlag;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.Progressable;
+import org.apache.http.client.utils.URIBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.EnumSet;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+import static org.apache.hadoop.fs.ozone.Constants.LISTING_PAGE_SIZE;
+import static org.apache.hadoop.fs.ozone.Constants.OZONE_DEFAULT_USER;
+import static org.apache.hadoop.fs.ozone.Constants.OZONE_USER_DIR;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_OFS_URI_SCHEME;
+
+/**
+ * The minimal Ozone Filesystem implementation.
+ * <p>
+ * This is a basic version which doesn't extend
+ * KeyProviderTokenIssuer and doesn't include statistics. It can be used
+ * from older hadoop version. For newer hadoop version use the full featured
+ * OFileSystem.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class BasicRootedOzoneFileSystem extends FileSystem {
+ static final Logger LOG =
+ LoggerFactory.getLogger(BasicRootedOzoneFileSystem.class);
+
+ /**
+ * The Ozone client for connecting to Ozone server.
+ */
+
+ private URI uri;
+ private String userName;
+ private Path workingDir;
+// private String gOmHost;
+// private int gOmPort;
+// private Configuration gConf;
+// private boolean gIsolatedClassloader;
+
+ private RootedOzoneClientAdapter adapter;
+// private String adapterPath;
+
+ private static final String URI_EXCEPTION_TEXT =
+ "URL should be one of the following formats: " +
+ "ofs://om-service-id/ OR " +
+ "ofs://om-host.example.com/ OR " +
+ "ofs://om-host.example.com:5678/";
+
+ @Override
+ public void initialize(URI name, Configuration conf) throws IOException {
+ super.initialize(name, conf);
+ setConf(conf);
+ Objects.requireNonNull(name.getScheme(), "No scheme provided in " + name);
+ Preconditions.checkArgument(getScheme().equals(name.getScheme()),
+ "Invalid scheme provided in " + name);
+
+ String authority = name.getAuthority();
+ if (authority == null) {
+ // authority is null when fs.defaultFS is not a qualified ofs URI and
+ // ofs:/// is passed to the client. matcher will NPE if authority is null
+ throw new IllegalArgumentException(URI_EXCEPTION_TEXT);
+ }
+
+ String omHost;
+ int omPort = -1;
+ // Parse hostname and port
+ String[] parts = authority.split(":");
+ if (parts.length > 2) {
+ throw new IllegalArgumentException(URI_EXCEPTION_TEXT);
+ }
+ omHost = parts[0];
+ if (parts.length == 2) {
+ try {
+ omPort = Integer.parseInt(parts[1]);
+ } catch (NumberFormatException e) {
+ throw new IllegalArgumentException(URI_EXCEPTION_TEXT);
+ }
+ }
+
+ try {
+ uri = new URIBuilder().setScheme(OZONE_OFS_URI_SCHEME)
+ .setHost(authority)
+ .build();
+ LOG.trace("Ozone URI for OFS initialization is " + uri);
+
+ //isolated is the default for ozonefs-lib-legacy which includes the
+ // /ozonefs.txt, otherwise the default is false. It could be overridden.
+ boolean defaultValue =
+ BasicRootedOzoneFileSystem.class.getClassLoader()
+ .getResource("ozonefs.txt") != null;
+
+ //Use string here instead of the constant as constant may not be available
+ //on the classpath of a hadoop 2.7
+ boolean isolatedClassloader =
+ conf.getBoolean("ozone.fs.isolated-classloader", defaultValue);
+
+ // adapter should be initialized in operations.
+ this.adapter = createAdapter(conf, omHost, omPort, isolatedClassloader);
+
+ try {
+ this.userName =
+ UserGroupInformation.getCurrentUser().getShortUserName();
+ } catch (IOException e) {
+ this.userName = OZONE_DEFAULT_USER;
+ }
+ this.workingDir = new Path(OZONE_USER_DIR, this.userName)
+ .makeQualified(this.uri, this.workingDir);
+ } catch (URISyntaxException ue) {
+ final String msg = "Invalid Ozone endpoint " + name;
+ LOG.error(msg, ue);
+ throw new IOException(msg, ue);
+ }
+ }
+
+ protected RootedOzoneClientAdapter createAdapter(Configuration conf,
+ String omHost, int omPort, boolean isolatedClassloader)
+ throws IOException {
+
+ if (isolatedClassloader) {
+ // TODO: Check how this code path need to be changed, for legacy Hadoop?
+ return RootedOzoneClientAdapterFactory.createAdapter();
+ } else {
+ return new BasicRootedOzoneClientAdapterImpl(omHost, omPort, conf);
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ try {
+ adapter.close();
+ } finally {
+ super.close();
+ }
+ }
+
+ @Override
+ public URI getUri() {
+ return uri;
+ }
+
+ @Override
+ public String getScheme() {
+ return OZONE_OFS_URI_SCHEME;
+ }
+
+ @Override
+ public FSDataInputStream open(Path path, int bufferSize) throws IOException {
+ incrementCounter(Statistic.INVOCATION_OPEN);
+ statistics.incrementReadOps(1);
+ LOG.trace("open() path: {}", path);
+ final String key = pathToKey(path);
+ return new FSDataInputStream(
+ new OzoneFSInputStream(adapter.readFile(key), statistics));
+ }
+
+ protected void incrementCounter(Statistic statistic) {
+ //don't do anything in this default implementation.
+ }
+
+ @Override
+ public FSDataOutputStream create(Path f, FsPermission permission,
+ boolean overwrite, int bufferSize,
+ short replication, long blockSize,
+ Progressable progress) throws IOException {
+ LOG.trace("create() path:{}", f);
+ incrementCounter(Statistic.INVOCATION_CREATE);
+ statistics.incrementWriteOps(1);
+ final String key = pathToKey(f);
+ return createOutputStream(key, overwrite, true);
+ }
+
+ @Override
+ public FSDataOutputStream createNonRecursive(Path path,
+ FsPermission permission,
+ EnumSet<CreateFlag> flags,
+ int bufferSize,
+ short replication,
+ long blockSize,
+ Progressable progress) throws IOException {
+ incrementCounter(Statistic.INVOCATION_CREATE_NON_RECURSIVE);
+ statistics.incrementWriteOps(1);
+ final String key = pathToKey(path);
+ return createOutputStream(key, flags.contains(CreateFlag.OVERWRITE), false);
+ }
+
+ private FSDataOutputStream createOutputStream(String key, boolean overwrite,
+ boolean recursive) throws IOException {
+ return new FSDataOutputStream(adapter.createFile(key, overwrite, recursive),
+ statistics);
+ }
+
+ @Override
+ public FSDataOutputStream append(Path f, int bufferSize,
+ Progressable progress) throws IOException {
+ throw new UnsupportedOperationException("append() Not implemented by the "
+ + getClass().getSimpleName() + " FileSystem implementation");
+ }
+
+ private class RenameIterator extends OzoneListingIterator {
+ private final String srcPath;
+ private final String dstPath;
+
+ RenameIterator(Path srcPath, Path dstPath)
+ throws IOException {
+ super(srcPath);
+ this.srcPath = pathToKey(srcPath);
+ this.dstPath = pathToKey(dstPath);
+ LOG.trace("rename from:{} to:{}", this.srcPath, this.dstPath);
+ }
+
+ @Override
+ boolean processKeyPath(String keyPath) throws IOException {
+ String newPath = dstPath.concat(keyPath.substring(srcPath.length()));
+ adapter.renamePath(keyPath, newPath);
+ return true;
+ }
+ }
+
+ /**
+ * Check whether the source and destination path are valid and then perform
+ * rename from source path to destination path.
+ * <p>
+ * The rename operation is performed by renaming the keys with src as prefix.
+ * For such keys the prefix is changed from src to dst.
+ *
+ * @param src source path for rename
+ * @param dst destination path for rename
+ * @return true if rename operation succeeded or
+ * if the src and dst have the same path and are of the same type
+ * @throws IOException on I/O errors or if the src/dst paths are invalid.
+ */
+ @Override
+ public boolean rename(Path src, Path dst) throws IOException {
+ incrementCounter(Statistic.INVOCATION_RENAME);
+ statistics.incrementWriteOps(1);
+ if (src.equals(dst)) {
+ return true;
+ }
+
+ LOG.trace("rename() from: {} to: {}", src, dst);
+ if (src.isRoot()) {
+ // Cannot rename root of file system
+ LOG.trace("Cannot rename the root of a filesystem");
+ return false;
+ }
+
+ // src and dst should be in the same bucket
+ OFSPath ofsSrc = new OFSPath(src);
+ OFSPath ofsDst = new OFSPath(dst);
+ if (!ofsSrc.isInSameBucketAs(ofsDst)) {
+ throw new IOException("Cannot rename a key to a different bucket");
+ }
+
+ // Cannot rename a directory to its own subdirectory
+ Path dstParent = dst.getParent();
+ while (dstParent != null && !src.equals(dstParent)) {
+ dstParent = dstParent.getParent();
+ }
+ Preconditions.checkArgument(dstParent == null,
+ "Cannot rename a directory to its own subdirectory");
+ // Check if the source exists
+ FileStatus srcStatus;
+ try {
+ srcStatus = getFileStatus(src);
+ } catch (FileNotFoundException fnfe) {
+ // source doesn't exist, return
+ return false;
+ }
+
+ // Check if the destination exists
+ FileStatus dstStatus;
+ try {
+ dstStatus = getFileStatus(dst);
+ } catch (FileNotFoundException fnde) {
+ dstStatus = null;
+ }
+
+ if (dstStatus == null) {
+ // If dst doesn't exist, check whether dst parent dir exists or not
+ // if the parent exists, the source can still be renamed to dst path
+ dstStatus = getFileStatus(dst.getParent());
+ if (!dstStatus.isDirectory()) {
+ throw new IOException(String.format(
+ "Failed to rename %s to %s, %s is a file", src, dst,
+ dst.getParent()));
+ }
+ } else {
+ // if dst exists and source and destination are same,
+ // check both the src and dst are of same type
+ if (srcStatus.getPath().equals(dstStatus.getPath())) {
+ return !srcStatus.isDirectory();
+ } else if (dstStatus.isDirectory()) {
+ // If dst is a directory, rename source as subpath of it.
+ // for example rename /source to /dst will lead to /dst/source
+ dst = new Path(dst, src.getName());
+ FileStatus[] statuses;
+ try {
+ statuses = listStatus(dst);
+ } catch (FileNotFoundException fnde) {
+ statuses = null;
+ }
+
+ if (statuses != null && statuses.length > 0) {
+ // If dst exists and not a directory not empty
+ throw new FileAlreadyExistsException(String.format(
+ "Failed to rename %s to %s, file already exists or not empty!",
+ src, dst));
+ }
+ } else {
+ // If dst is not a directory
+ throw new FileAlreadyExistsException(String.format(
+ "Failed to rename %s to %s, file already exists!", src, dst));
+ }
+ }
+
+ if (srcStatus.isDirectory()) {
+ if (dst.toString().startsWith(src.toString() + OZONE_URI_DELIMITER)) {
+ LOG.trace("Cannot rename a directory to a subdirectory of self");
+ return false;
+ }
+ }
+ RenameIterator iterator = new RenameIterator(src, dst);
+ boolean result = iterator.iterate();
+ if (result) {
+ createFakeParentDirectory(src);
+ }
+ return result;
+ }
+
+ private class DeleteIterator extends OzoneListingIterator {
+ private boolean recursive;
+
+ DeleteIterator(Path f, boolean recursive)
+ throws IOException {
+ super(f);
+ this.recursive = recursive;
+ if (getStatus().isDirectory()
+ && !this.recursive
+ && listStatus(f).length != 0) {
+ throw new PathIsNotEmptyDirectoryException(f.toString());
+ }
+ }
+
+ @Override
+ boolean processKeyPath(String keyPath) throws IOException {
+ if (keyPath.equals("")) {
+ LOG.trace("Skipping deleting root directory");
+ return true;
+ } else {
+ LOG.trace("deleting key path:" + keyPath);
+ boolean succeed = adapter.deleteObject(keyPath);
+ // if recursive delete is requested ignore the return value of
+ // deleteObject and issue deletes for other keys.
+ return recursive || succeed;
+ }
+ }
+ }
+
+ /**
+ * Deletes the children of the input dir path by iterating though the
+ * DeleteIterator.
+ *
+ * @param f directory path to be deleted
+ * @return true if successfully deletes all required keys, false otherwise
+ * @throws IOException
+ */
+ private boolean innerDelete(Path f, boolean recursive) throws IOException {
+ LOG.trace("delete() path:{} recursive:{}", f, recursive);
+ try {
+ DeleteIterator iterator = new DeleteIterator(f, recursive);
+ return iterator.iterate();
+ } catch (FileNotFoundException e) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Couldn't delete {} - does not exist", f);
+ }
+ return false;
+ }
+ }
+
+ @Override
+ public boolean delete(Path f, boolean recursive) throws IOException {
+ incrementCounter(Statistic.INVOCATION_DELETE);
+ statistics.incrementWriteOps(1);
+ LOG.debug("Delete path {} - recursive {}", f, recursive);
+ FileStatus status;
+ try {
+ status = getFileStatus(f);
+ } catch (FileNotFoundException ex) {
+ LOG.warn("delete: Path does not exist: {}", f);
+ return false;
+ }
+
+ String key = pathToKey(f);
+ boolean result;
+
+ if (status.isDirectory()) {
+ LOG.debug("delete: Path is a directory: {}", f);
+ key = addTrailingSlashIfNeeded(key);
+
+ if (key.equals("/")) {
+ LOG.warn("Cannot delete root directory.");
+ return false;
+ }
+
+ result = innerDelete(f, recursive);
+ } else {
+ LOG.debug("delete: Path is a file: {}", f);
+ result = adapter.deleteObject(key);
+ }
+
+ if (result) {
+ // If this delete operation removes all files/directories from the
+ // parent direcotry, then an empty parent directory must be created.
+ createFakeParentDirectory(f);
+ }
+
+ return result;
+ }
+
+ /**
+ * Create a fake parent directory key if it does not already exist and no
+ * other child of this parent directory exists.
+ *
+ * @param f path to the fake parent directory
+ * @throws IOException
+ */
+ private void createFakeParentDirectory(Path f) throws IOException {
+ Path parent = f.getParent();
+ if (parent != null && !parent.isRoot()) {
+ createFakeDirectoryIfNecessary(parent);
+ }
+ }
+
+ /**
+ * Create a fake directory key if it does not already exist.
+ *
+ * @param f path to the fake directory
+ * @throws IOException
+ */
+ private void createFakeDirectoryIfNecessary(Path f) throws IOException {
+ String key = pathToKey(f);
+ if (!key.isEmpty() && !o3Exists(f)) {
+ LOG.debug("Creating new fake directory at {}", f);
+ String dirKey = addTrailingSlashIfNeeded(key);
+ adapter.createDirectory(dirKey);
+ }
+ }
+
+ /**
+ * Check if a file or directory exists corresponding to given path.
+ *
+ * @param f path to file/directory.
+ * @return true if it exists, false otherwise.
+ * @throws IOException
+ */
+ private boolean o3Exists(final Path f) throws IOException {
+ Path path = makeQualified(f);
+ try {
+ getFileStatus(path);
+ return true;
+ } catch (FileNotFoundException ex) {
+ return false;
+ }
+ }
+
+ @Override
+ public FileStatus[] listStatus(Path f) throws IOException {
+ incrementCounter(Statistic.INVOCATION_LIST_STATUS);
+ statistics.incrementReadOps(1);
+ LOG.trace("listStatus() path:{}", f);
+ int numEntries = LISTING_PAGE_SIZE;
+ LinkedList<FileStatus> statuses = new LinkedList<>();
+ List<FileStatus> tmpStatusList;
+ String startPath = "";
+
+ do {
+ tmpStatusList =
+ adapter.listStatus(pathToKey(f), false, startPath,
+ numEntries, uri, workingDir, getUsername())
+ .stream()
+ .map(this::convertFileStatus)
+ .collect(Collectors.toList());
+
+ if (!tmpStatusList.isEmpty()) {
+ if (startPath.isEmpty()) {
+ statuses.addAll(tmpStatusList);
+ } else {
+ statuses.addAll(tmpStatusList.subList(1, tmpStatusList.size()));
+ }
+ startPath = pathToKey(statuses.getLast().getPath());
+ }
+ // listStatus returns entries numEntries in size if available.
+ // Any lesser number of entries indicate that the required entries have
+ // exhausted.
+ } while (tmpStatusList.size() == numEntries);
+
+ return statuses.toArray(new FileStatus[0]);
+ }
+
+ @Override
+ public void setWorkingDirectory(Path newDir) {
+ workingDir = newDir;
+ }
+
+ @Override
+ public Path getWorkingDirectory() {
+ return workingDir;
+ }
+
+ @Override
+ public Token<?> getDelegationToken(String renewer) throws IOException {
+ return adapter.getDelegationToken(renewer);
+ }
+
+ /**
+ * Get a canonical service name for this file system. If the URI is logical,
+ * the hostname part of the URI will be returned.
+ *
+ * @return a service string that uniquely identifies this file system.
+ */
+ @Override
+ public String getCanonicalServiceName() {
+ return adapter.getCanonicalServiceName();
+ }
+
+ /**
+ * Get the username of the FS.
+ *
+ * @return the short name of the user who instantiated the FS
+ */
+ public String getUsername() {
+ return userName;
+ }
+
+ /**
+ * Creates a directory. Directory is represented using a key with no value.
+ *
+ * @param path directory path to be created
+ * @return true if directory exists or created successfully.
+ * @throws IOException
+ */
+ private boolean mkdir(Path path) throws IOException {
+ return adapter.createDirectory(pathToKey(path));
+ }
+
+ @Override
+ public boolean mkdirs(Path f, FsPermission permission) throws IOException {
+ LOG.trace("mkdir() path:{} ", f);
+ String key = pathToKey(f);
+ if (isEmpty(key)) {
+ return false;
+ }
+ return mkdir(f);
+ }
+
+ @Override
+ public FileStatus getFileStatus(Path f) throws IOException {
+ incrementCounter(Statistic.INVOCATION_GET_FILE_STATUS);
+ statistics.incrementReadOps(1);
+ LOG.trace("getFileStatus() path:{}", f);
+ Path qualifiedPath = f.makeQualified(uri, workingDir);
+ String key = pathToKey(qualifiedPath);
+ FileStatus fileStatus = null;
+ try {
+ fileStatus = convertFileStatus(
+ adapter.getFileStatus(key, uri, qualifiedPath, getUsername()));
+ } catch (OMException ex) {
+ if (ex.getResult().equals(OMException.ResultCodes.KEY_NOT_FOUND)) {
+ throw new FileNotFoundException("File not found. path:" + f);
+ }
+ }
+ return fileStatus;
+ }
+
+ @Override
+ public BlockLocation[] getFileBlockLocations(FileStatus fileStatus,
+ long start, long len)
+ throws IOException {
+ if (fileStatus instanceof LocatedFileStatus) {
+ return ((LocatedFileStatus) fileStatus).getBlockLocations();
+ } else {
+ return super.getFileBlockLocations(fileStatus, start, len);
+ }
+ }
+
+ /**
+ * Turn a path (relative or otherwise) into an Ozone key.
+ *
+ * @param path the path of the file.
+ * @return the key of the object that represents the file.
+ */
+ public String pathToKey(Path path) {
+ Objects.requireNonNull(path, "Path can't be null!");
+ if (!path.isAbsolute()) {
+ path = new Path(workingDir, path);
+ }
+ // removing leading '/' char
+ String key = path.toUri().getPath().substring(1);
+ LOG.trace("path for key: {} is: {}", key, path);
+ return key;
+ }
+
+ /**
+ * Add trailing delimiter to path if it is already not present.
+ *
+ * @param key the ozone Key which needs to be appended
+ * @return delimiter appended key
+ */
+ private String addTrailingSlashIfNeeded(String key) {
+ if (!isEmpty(key) && !key.endsWith(OZONE_URI_DELIMITER)) {
+ return key + OZONE_URI_DELIMITER;
+ } else {
+ return key;
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "RootedOzoneFileSystem{URI=" + uri + ", "
+ + "workingDir=" + workingDir + ", "
+ + "userName=" + userName + ", "
+ + "statistics=" + statistics
+ + "}";
+ }
+
+ /**
+ * This class provides an interface to iterate through all the keys in the
+ * bucket prefixed with the input path key and process them.
+ * <p>
+ * Each implementing class should define how the keys should be processed
+ * through the processKeyPath() function.
+ */
+ private abstract class OzoneListingIterator {
+ private final Path path;
+ private final FileStatus status;
+ private String pathKey;
+ private Iterator<BasicKeyInfo> keyIterator;
+
+ OzoneListingIterator(Path path)
+ throws IOException {
+ this.path = path;
+ this.status = getFileStatus(path);
+ this.pathKey = pathToKey(path);
+ if (status.isDirectory()) {
+ this.pathKey = addTrailingSlashIfNeeded(pathKey);
+ }
+ keyIterator = adapter.listKeys(pathKey);
+ }
+
+ /**
+ * The output of processKey determines if further iteration through the
+ * keys should be done or not.
+ *
+ * @return true if we should continue iteration of keys, false otherwise.
+ * @throws IOException
+ */
+ abstract boolean processKeyPath(String keyPath) throws IOException;
+
+ /**
+ * Iterates through all the keys prefixed with the input path's key and
+ * processes the key though processKey().
+ * If for any key, the processKey() returns false, then the iteration is
+ * stopped and returned with false indicating that all the keys could not
+ * be processed successfully.
+ *
+ * @return true if all keys are processed successfully, false otherwise.
+ * @throws IOException
+ */
+ boolean iterate() throws IOException {
+ LOG.trace("Iterating path: {}", path);
+ if (status.isDirectory()) {
+ LOG.trace("Iterating directory: {}", pathKey);
+ OFSPath ofsPath = new OFSPath(pathKey);
+ while (keyIterator.hasNext()) {
+ BasicKeyInfo key = keyIterator.next();
+ String keyPath = ofsPath.getNonKeyPathNoPrefixDelim() +
+ OZONE_URI_DELIMITER + key.getName();
+ LOG.trace("iterating key path: {}", keyPath);
+ if (!processKeyPath(keyPath)) {
+ return false;
+ }
+ }
+ return true;
+ } else {
+ LOG.trace("iterating file: {}", path);
+ return processKeyPath(pathKey);
+ }
+ }
+
+ String getPathKey() {
+ return pathKey;
+ }
+
+ boolean pathIsDirectory() {
+ return status.isDirectory();
+ }
+
+ FileStatus getStatus() {
+ return status;
+ }
+ }
+
+ public RootedOzoneClientAdapter getAdapter() {
+ return adapter;
+ }
+
+ public boolean isEmpty(CharSequence cs) {
+ return cs == null || cs.length() == 0;
+ }
+
+ public boolean isNumber(String number) {
+ try {
+ Integer.parseInt(number);
+ } catch (NumberFormatException ex) {
+ return false;
+ }
+ return true;
+ }
+
+ private FileStatus convertFileStatus(FileStatusAdapter fileStatusAdapter) {
+ Path symLink = null;
+ try {
+ fileStatusAdapter.getSymlink();
+ } catch (Exception ex) {
+ //NOOP: If not symlink symlink remains null.
+ }
+
+ // Process path.
+ URI newUri = fileStatusAdapter.getPath().toUri();
+ try {
+ newUri = new URIBuilder().setScheme(newUri.getScheme())
+ .setHost(newUri.getAuthority())
+ .setPath(newUri.getPath())
+ .build();
+ } catch (URISyntaxException e) {
+ }
Review comment:
I did not understand what we are doing here. Could you please give an example.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] hanishakoneru commented on a change in pull request
#415: HDDS-2840. Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
hanishakoneru commented on a change in pull request #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#discussion_r369827923
##########
File path: hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
##########
@@ -0,0 +1,805 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.ozone;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.CreateFlag;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.Progressable;
+import org.apache.http.client.utils.URIBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.EnumSet;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+import static org.apache.hadoop.fs.ozone.Constants.LISTING_PAGE_SIZE;
+import static org.apache.hadoop.fs.ozone.Constants.OZONE_DEFAULT_USER;
+import static org.apache.hadoop.fs.ozone.Constants.OZONE_USER_DIR;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_OFS_URI_SCHEME;
+
+/**
+ * The minimal Ozone Filesystem implementation.
+ * <p>
+ * This is a basic version which doesn't extend
+ * KeyProviderTokenIssuer and doesn't include statistics. It can be used
+ * from older hadoop version. For newer hadoop version use the full featured
+ * OFileSystem.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class BasicRootedOzoneFileSystem extends FileSystem {
+ static final Logger LOG =
+ LoggerFactory.getLogger(BasicRootedOzoneFileSystem.class);
+
+ /**
+ * The Ozone client for connecting to Ozone server.
+ */
+
+ private URI uri;
+ private String userName;
+ private Path workingDir;
+// private String gOmHost;
+// private int gOmPort;
+// private Configuration gConf;
+// private boolean gIsolatedClassloader;
+
+ private RootedOzoneClientAdapter adapter;
+// private String adapterPath;
+
+ private static final String URI_EXCEPTION_TEXT =
+ "URL should be one of the following formats: " +
+ "ofs://om-service-id/ OR " +
+ "ofs://om-host.example.com/ OR " +
+ "ofs://om-host.example.com:5678/";
+
+ @Override
+ public void initialize(URI name, Configuration conf) throws IOException {
+ super.initialize(name, conf);
+ setConf(conf);
+ Objects.requireNonNull(name.getScheme(), "No scheme provided in " + name);
+ Preconditions.checkArgument(getScheme().equals(name.getScheme()),
+ "Invalid scheme provided in " + name);
+
+ String authority = name.getAuthority();
+ if (authority == null) {
+ // authority is null when fs.defaultFS is not a qualified ofs URI and
+ // ofs:/// is passed to the client. matcher will NPE if authority is null
+ throw new IllegalArgumentException(URI_EXCEPTION_TEXT);
+ }
+
+ String omHost;
+ int omPort = -1;
+ // Parse hostname and port
+ String[] parts = authority.split(":");
+ if (parts.length > 2) {
+ throw new IllegalArgumentException(URI_EXCEPTION_TEXT);
+ }
+ omHost = parts[0];
+ if (parts.length == 2) {
+ try {
+ omPort = Integer.parseInt(parts[1]);
+ } catch (NumberFormatException e) {
+ throw new IllegalArgumentException(URI_EXCEPTION_TEXT);
+ }
+ }
+
+ try {
+ uri = new URIBuilder().setScheme(OZONE_OFS_URI_SCHEME)
+ .setHost(authority)
+ .build();
+ LOG.trace("Ozone URI for OFS initialization is " + uri);
+
+ //isolated is the default for ozonefs-lib-legacy which includes the
+ // /ozonefs.txt, otherwise the default is false. It could be overridden.
+ boolean defaultValue =
+ BasicRootedOzoneFileSystem.class.getClassLoader()
+ .getResource("ozonefs.txt") != null;
+
+ //Use string here instead of the constant as constant may not be available
+ //on the classpath of a hadoop 2.7
+ boolean isolatedClassloader =
+ conf.getBoolean("ozone.fs.isolated-classloader", defaultValue);
+
+ // adapter should be initialized in operations.
+ this.adapter = createAdapter(conf, omHost, omPort, isolatedClassloader);
+
+ try {
+ this.userName =
+ UserGroupInformation.getCurrentUser().getShortUserName();
+ } catch (IOException e) {
+ this.userName = OZONE_DEFAULT_USER;
+ }
+ this.workingDir = new Path(OZONE_USER_DIR, this.userName)
+ .makeQualified(this.uri, this.workingDir);
+ } catch (URISyntaxException ue) {
+ final String msg = "Invalid Ozone endpoint " + name;
+ LOG.error(msg, ue);
+ throw new IOException(msg, ue);
+ }
+ }
+
+ protected RootedOzoneClientAdapter createAdapter(Configuration conf,
+ String omHost, int omPort, boolean isolatedClassloader)
+ throws IOException {
+
+ if (isolatedClassloader) {
+ // TODO: Check how this code path need to be changed, for legacy Hadoop?
+ return RootedOzoneClientAdapterFactory.createAdapter();
+ } else {
+ return new BasicRootedOzoneClientAdapterImpl(omHost, omPort, conf);
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ try {
+ adapter.close();
+ } finally {
+ super.close();
+ }
+ }
+
+ @Override
+ public URI getUri() {
+ return uri;
+ }
+
+ @Override
+ public String getScheme() {
+ return OZONE_OFS_URI_SCHEME;
+ }
+
+ @Override
+ public FSDataInputStream open(Path path, int bufferSize) throws IOException {
+ incrementCounter(Statistic.INVOCATION_OPEN);
+ statistics.incrementReadOps(1);
+ LOG.trace("open() path: {}", path);
+ final String key = pathToKey(path);
+ return new FSDataInputStream(
+ new OzoneFSInputStream(adapter.readFile(key), statistics));
+ }
+
+ protected void incrementCounter(Statistic statistic) {
+ //don't do anything in this default implementation.
+ }
+
+ @Override
+ public FSDataOutputStream create(Path f, FsPermission permission,
+ boolean overwrite, int bufferSize,
+ short replication, long blockSize,
+ Progressable progress) throws IOException {
+ LOG.trace("create() path:{}", f);
+ incrementCounter(Statistic.INVOCATION_CREATE);
+ statistics.incrementWriteOps(1);
+ final String key = pathToKey(f);
+ return createOutputStream(key, overwrite, true);
+ }
+
+ @Override
+ public FSDataOutputStream createNonRecursive(Path path,
+ FsPermission permission,
+ EnumSet<CreateFlag> flags,
+ int bufferSize,
+ short replication,
+ long blockSize,
+ Progressable progress) throws IOException {
+ incrementCounter(Statistic.INVOCATION_CREATE_NON_RECURSIVE);
+ statistics.incrementWriteOps(1);
+ final String key = pathToKey(path);
+ return createOutputStream(key, flags.contains(CreateFlag.OVERWRITE), false);
+ }
+
+ private FSDataOutputStream createOutputStream(String key, boolean overwrite,
+ boolean recursive) throws IOException {
+ return new FSDataOutputStream(adapter.createFile(key, overwrite, recursive),
+ statistics);
+ }
+
+ @Override
+ public FSDataOutputStream append(Path f, int bufferSize,
+ Progressable progress) throws IOException {
+ throw new UnsupportedOperationException("append() Not implemented by the "
+ + getClass().getSimpleName() + " FileSystem implementation");
+ }
+
+ private class RenameIterator extends OzoneListingIterator {
+ private final String srcPath;
+ private final String dstPath;
+
+ RenameIterator(Path srcPath, Path dstPath)
+ throws IOException {
+ super(srcPath);
+ this.srcPath = pathToKey(srcPath);
+ this.dstPath = pathToKey(dstPath);
+ LOG.trace("rename from:{} to:{}", this.srcPath, this.dstPath);
+ }
+
+ @Override
+ boolean processKeyPath(String keyPath) throws IOException {
+ String newPath = dstPath.concat(keyPath.substring(srcPath.length()));
+ adapter.renamePath(keyPath, newPath);
+ return true;
+ }
+ }
+
+ /**
+ * Check whether the source and destination path are valid and then perform
+ * rename from source path to destination path.
+ * <p>
+ * The rename operation is performed by renaming the keys with src as prefix.
+ * For such keys the prefix is changed from src to dst.
+ *
+ * @param src source path for rename
+ * @param dst destination path for rename
+ * @return true if rename operation succeeded or
+ * if the src and dst have the same path and are of the same type
+ * @throws IOException on I/O errors or if the src/dst paths are invalid.
+ */
+ @Override
+ public boolean rename(Path src, Path dst) throws IOException {
+ incrementCounter(Statistic.INVOCATION_RENAME);
+ statistics.incrementWriteOps(1);
+ if (src.equals(dst)) {
+ return true;
+ }
+
+ LOG.trace("rename() from: {} to: {}", src, dst);
+ if (src.isRoot()) {
+ // Cannot rename root of file system
+ LOG.trace("Cannot rename the root of a filesystem");
+ return false;
+ }
+
+ // src and dst should be in the same bucket
+ OFSPath ofsSrc = new OFSPath(src);
+ OFSPath ofsDst = new OFSPath(dst);
+ if (!ofsSrc.isInSameBucketAs(ofsDst)) {
+ throw new IOException("Cannot rename a key to a different bucket");
+ }
+
+ // Cannot rename a directory to its own subdirectory
+ Path dstParent = dst.getParent();
+ while (dstParent != null && !src.equals(dstParent)) {
+ dstParent = dstParent.getParent();
+ }
+ Preconditions.checkArgument(dstParent == null,
+ "Cannot rename a directory to its own subdirectory");
+ // Check if the source exists
+ FileStatus srcStatus;
+ try {
+ srcStatus = getFileStatus(src);
+ } catch (FileNotFoundException fnfe) {
+ // source doesn't exist, return
+ return false;
+ }
+
+ // Check if the destination exists
+ FileStatus dstStatus;
+ try {
+ dstStatus = getFileStatus(dst);
+ } catch (FileNotFoundException fnde) {
+ dstStatus = null;
+ }
+
+ if (dstStatus == null) {
+ // If dst doesn't exist, check whether dst parent dir exists or not
+ // if the parent exists, the source can still be renamed to dst path
+ dstStatus = getFileStatus(dst.getParent());
+ if (!dstStatus.isDirectory()) {
+ throw new IOException(String.format(
+ "Failed to rename %s to %s, %s is a file", src, dst,
+ dst.getParent()));
+ }
+ } else {
+ // if dst exists and source and destination are same,
+ // check both the src and dst are of same type
+ if (srcStatus.getPath().equals(dstStatus.getPath())) {
+ return !srcStatus.isDirectory();
+ } else if (dstStatus.isDirectory()) {
+ // If dst is a directory, rename source as subpath of it.
+ // for example rename /source to /dst will lead to /dst/source
+ dst = new Path(dst, src.getName());
+ FileStatus[] statuses;
+ try {
+ statuses = listStatus(dst);
+ } catch (FileNotFoundException fnde) {
+ statuses = null;
+ }
+
+ if (statuses != null && statuses.length > 0) {
+ // If dst exists and not a directory not empty
+ throw new FileAlreadyExistsException(String.format(
+ "Failed to rename %s to %s, file already exists or not empty!",
+ src, dst));
+ }
+ } else {
+ // If dst is not a directory
+ throw new FileAlreadyExistsException(String.format(
+ "Failed to rename %s to %s, file already exists!", src, dst));
+ }
+ }
+
+ if (srcStatus.isDirectory()) {
+ if (dst.toString().startsWith(src.toString() + OZONE_URI_DELIMITER)) {
+ LOG.trace("Cannot rename a directory to a subdirectory of self");
+ return false;
+ }
+ }
+ RenameIterator iterator = new RenameIterator(src, dst);
+ boolean result = iterator.iterate();
+ if (result) {
+ createFakeParentDirectory(src);
+ }
+ return result;
+ }
+
+ private class DeleteIterator extends OzoneListingIterator {
+ private boolean recursive;
+
+ DeleteIterator(Path f, boolean recursive)
+ throws IOException {
+ super(f);
+ this.recursive = recursive;
+ if (getStatus().isDirectory()
+ && !this.recursive
+ && listStatus(f).length != 0) {
+ throw new PathIsNotEmptyDirectoryException(f.toString());
+ }
+ }
+
+ @Override
+ boolean processKeyPath(String keyPath) throws IOException {
+ if (keyPath.equals("")) {
+ LOG.trace("Skipping deleting root directory");
+ return true;
+ } else {
+ LOG.trace("deleting key path:" + keyPath);
+ boolean succeed = adapter.deleteObject(keyPath);
+ // if recursive delete is requested ignore the return value of
+ // deleteObject and issue deletes for other keys.
+ return recursive || succeed;
+ }
+ }
+ }
+
+ /**
+ * Deletes the children of the input dir path by iterating though the
+ * DeleteIterator.
+ *
+ * @param f directory path to be deleted
+ * @return true if successfully deletes all required keys, false otherwise
+ * @throws IOException
+ */
+ private boolean innerDelete(Path f, boolean recursive) throws IOException {
+ LOG.trace("delete() path:{} recursive:{}", f, recursive);
+ try {
+ DeleteIterator iterator = new DeleteIterator(f, recursive);
+ return iterator.iterate();
+ } catch (FileNotFoundException e) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Couldn't delete {} - does not exist", f);
+ }
+ return false;
+ }
+ }
+
+ @Override
+ public boolean delete(Path f, boolean recursive) throws IOException {
+ incrementCounter(Statistic.INVOCATION_DELETE);
+ statistics.incrementWriteOps(1);
+ LOG.debug("Delete path {} - recursive {}", f, recursive);
+ FileStatus status;
+ try {
+ status = getFileStatus(f);
+ } catch (FileNotFoundException ex) {
+ LOG.warn("delete: Path does not exist: {}", f);
+ return false;
+ }
+
+ String key = pathToKey(f);
+ boolean result;
+
+ if (status.isDirectory()) {
+ LOG.debug("delete: Path is a directory: {}", f);
+ key = addTrailingSlashIfNeeded(key);
+
+ if (key.equals("/")) {
+ LOG.warn("Cannot delete root directory.");
+ return false;
+ }
+
+ result = innerDelete(f, recursive);
+ } else {
+ LOG.debug("delete: Path is a file: {}", f);
+ result = adapter.deleteObject(key);
+ }
+
+ if (result) {
+ // If this delete operation removes all files/directories from the
+ // parent direcotry, then an empty parent directory must be created.
+ createFakeParentDirectory(f);
+ }
+
+ return result;
+ }
+
+ /**
+ * Create a fake parent directory key if it does not already exist and no
+ * other child of this parent directory exists.
+ *
+ * @param f path to the fake parent directory
+ * @throws IOException
+ */
+ private void createFakeParentDirectory(Path f) throws IOException {
+ Path parent = f.getParent();
+ if (parent != null && !parent.isRoot()) {
+ createFakeDirectoryIfNecessary(parent);
+ }
+ }
+
+ /**
+ * Create a fake directory key if it does not already exist.
+ *
+ * @param f path to the fake directory
+ * @throws IOException
+ */
+ private void createFakeDirectoryIfNecessary(Path f) throws IOException {
+ String key = pathToKey(f);
+ if (!key.isEmpty() && !o3Exists(f)) {
+ LOG.debug("Creating new fake directory at {}", f);
+ String dirKey = addTrailingSlashIfNeeded(key);
+ adapter.createDirectory(dirKey);
+ }
+ }
+
+ /**
+ * Check if a file or directory exists corresponding to given path.
+ *
+ * @param f path to file/directory.
+ * @return true if it exists, false otherwise.
+ * @throws IOException
+ */
+ private boolean o3Exists(final Path f) throws IOException {
+ Path path = makeQualified(f);
+ try {
+ getFileStatus(path);
+ return true;
+ } catch (FileNotFoundException ex) {
+ return false;
+ }
+ }
+
+ @Override
+ public FileStatus[] listStatus(Path f) throws IOException {
+ incrementCounter(Statistic.INVOCATION_LIST_STATUS);
+ statistics.incrementReadOps(1);
+ LOG.trace("listStatus() path:{}", f);
+ int numEntries = LISTING_PAGE_SIZE;
+ LinkedList<FileStatus> statuses = new LinkedList<>();
+ List<FileStatus> tmpStatusList;
+ String startPath = "";
+
+ do {
+ tmpStatusList =
+ adapter.listStatus(pathToKey(f), false, startPath,
+ numEntries, uri, workingDir, getUsername())
+ .stream()
+ .map(this::convertFileStatus)
+ .collect(Collectors.toList());
+
+ if (!tmpStatusList.isEmpty()) {
+ if (startPath.isEmpty()) {
+ statuses.addAll(tmpStatusList);
+ } else {
+ statuses.addAll(tmpStatusList.subList(1, tmpStatusList.size()));
+ }
+ startPath = pathToKey(statuses.getLast().getPath());
+ }
+ // listStatus returns entries numEntries in size if available.
+ // Any lesser number of entries indicate that the required entries have
+ // exhausted.
+ } while (tmpStatusList.size() == numEntries);
+
+ return statuses.toArray(new FileStatus[0]);
+ }
+
+ @Override
+ public void setWorkingDirectory(Path newDir) {
+ workingDir = newDir;
+ }
+
+ @Override
+ public Path getWorkingDirectory() {
+ return workingDir;
+ }
+
+ @Override
+ public Token<?> getDelegationToken(String renewer) throws IOException {
+ return adapter.getDelegationToken(renewer);
+ }
+
+ /**
+ * Get a canonical service name for this file system. If the URI is logical,
+ * the hostname part of the URI will be returned.
+ *
+ * @return a service string that uniquely identifies this file system.
+ */
+ @Override
+ public String getCanonicalServiceName() {
+ return adapter.getCanonicalServiceName();
+ }
+
+ /**
+ * Get the username of the FS.
+ *
+ * @return the short name of the user who instantiated the FS
+ */
+ public String getUsername() {
+ return userName;
+ }
+
+ /**
+ * Creates a directory. Directory is represented using a key with no value.
+ *
+ * @param path directory path to be created
+ * @return true if directory exists or created successfully.
+ * @throws IOException
+ */
+ private boolean mkdir(Path path) throws IOException {
+ return adapter.createDirectory(pathToKey(path));
+ }
+
+ @Override
+ public boolean mkdirs(Path f, FsPermission permission) throws IOException {
+ LOG.trace("mkdir() path:{} ", f);
+ String key = pathToKey(f);
+ if (isEmpty(key)) {
+ return false;
+ }
+ return mkdir(f);
+ }
+
+ @Override
+ public FileStatus getFileStatus(Path f) throws IOException {
+ incrementCounter(Statistic.INVOCATION_GET_FILE_STATUS);
+ statistics.incrementReadOps(1);
+ LOG.trace("getFileStatus() path:{}", f);
+ Path qualifiedPath = f.makeQualified(uri, workingDir);
+ String key = pathToKey(qualifiedPath);
+ FileStatus fileStatus = null;
+ try {
+ fileStatus = convertFileStatus(
+ adapter.getFileStatus(key, uri, qualifiedPath, getUsername()));
+ } catch (OMException ex) {
+ if (ex.getResult().equals(OMException.ResultCodes.KEY_NOT_FOUND)) {
+ throw new FileNotFoundException("File not found. path:" + f);
+ }
+ }
+ return fileStatus;
+ }
+
+ @Override
+ public BlockLocation[] getFileBlockLocations(FileStatus fileStatus,
+ long start, long len)
+ throws IOException {
+ if (fileStatus instanceof LocatedFileStatus) {
+ return ((LocatedFileStatus) fileStatus).getBlockLocations();
+ } else {
+ return super.getFileBlockLocations(fileStatus, start, len);
+ }
+ }
+
+ /**
+ * Turn a path (relative or otherwise) into an Ozone key.
+ *
+ * @param path the path of the file.
+ * @return the key of the object that represents the file.
+ */
+ public String pathToKey(Path path) {
+ Objects.requireNonNull(path, "Path can't be null!");
+ if (!path.isAbsolute()) {
+ path = new Path(workingDir, path);
+ }
+ // removing leading '/' char
+ String key = path.toUri().getPath().substring(1);
+ LOG.trace("path for key: {} is: {}", key, path);
+ return key;
+ }
+
+ /**
+ * Add trailing delimiter to path if it is already not present.
+ *
+ * @param key the ozone Key which needs to be appended
+ * @return delimiter appended key
+ */
+ private String addTrailingSlashIfNeeded(String key) {
+ if (!isEmpty(key) && !key.endsWith(OZONE_URI_DELIMITER)) {
+ return key + OZONE_URI_DELIMITER;
+ } else {
+ return key;
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "RootedOzoneFileSystem{URI=" + uri + ", "
+ + "workingDir=" + workingDir + ", "
+ + "userName=" + userName + ", "
+ + "statistics=" + statistics
+ + "}";
+ }
+
+ /**
+ * This class provides an interface to iterate through all the keys in the
+ * bucket prefixed with the input path key and process them.
+ * <p>
+ * Each implementing class should define how the keys should be processed
+ * through the processKeyPath() function.
+ */
+ private abstract class OzoneListingIterator {
+ private final Path path;
+ private final FileStatus status;
+ private String pathKey;
+ private Iterator<BasicKeyInfo> keyIterator;
+
+ OzoneListingIterator(Path path)
+ throws IOException {
+ this.path = path;
+ this.status = getFileStatus(path);
+ this.pathKey = pathToKey(path);
+ if (status.isDirectory()) {
+ this.pathKey = addTrailingSlashIfNeeded(pathKey);
+ }
+ keyIterator = adapter.listKeys(pathKey);
+ }
+
+ /**
+ * The output of processKey determines if further iteration through the
+ * keys should be done or not.
+ *
+ * @return true if we should continue iteration of keys, false otherwise.
+ * @throws IOException
+ */
+ abstract boolean processKeyPath(String keyPath) throws IOException;
+
+ /**
+ * Iterates through all the keys prefixed with the input path's key and
+ * processes the key though processKey().
+ * If for any key, the processKey() returns false, then the iteration is
+ * stopped and returned with false indicating that all the keys could not
+ * be processed successfully.
+ *
+ * @return true if all keys are processed successfully, false otherwise.
+ * @throws IOException
+ */
+ boolean iterate() throws IOException {
+ LOG.trace("Iterating path: {}", path);
+ if (status.isDirectory()) {
+ LOG.trace("Iterating directory: {}", pathKey);
+ OFSPath ofsPath = new OFSPath(pathKey);
Review comment:
Do we need to convert pathKey to ofsPath here? We make this conversion when corresponding function (rename/delete) is called in processKeyPath anyway.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] xiaoyuyao commented on issue #415: HDDS-2840.
Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
xiaoyuyao commented on issue #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#issuecomment-580451681
+1, I will merge it shortly.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] smengcl commented on issue #415: [WIP] HDDS-2840.
Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
smengcl commented on issue #415: [WIP] HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#issuecomment-571714871
Status update:
1. For now mkdir works in ozone fs shell. `mkdir -p` will create volume and bucket if they don't exist. See commit message ebd2bf8e28976f370e8fc17ecf42b1b3c4497a3e how I tested it. Note: Hasn't been tested throughly!
2. In order for the code to compile after rebase to latest dev branch, I have manually applied HDDS-2777 and HDDS-2785 to OFS classes (as they were copied from existing o3fs classes and some APIs has changed).
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] xiaoyuyao commented on a change in pull request
#415: HDDS-2840. Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
xiaoyuyao commented on a change in pull request #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#discussion_r372017721
##########
File path: hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
##########
@@ -0,0 +1,694 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.ozone;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.key.KeyProvider;
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdds.client.ReplicationFactor;
+import org.apache.hadoop.hdds.client.ReplicationType;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ozone.OmUtils;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.client.OzoneKey;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
+import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
+import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenRenewer;
+
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .BUCKET_ALREADY_EXISTS;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .VOLUME_ALREADY_EXISTS;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .VOLUME_NOT_FOUND;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .BUCKET_NOT_FOUND;
+
+/**
+ * Basic Implementation of the OzoneFileSystem calls.
+ * <p>
+ * This is the minimal version which doesn't include any statistics.
+ * <p>
+ * For full featured version use OzoneClientAdapterImpl.
+ */
+public class BasicRootedOzoneClientAdapterImpl
+ implements RootedOzoneClientAdapter {
+
+ static final Logger LOG =
+ LoggerFactory.getLogger(BasicRootedOzoneClientAdapterImpl.class);
+
+ private OzoneClient ozoneClient;
+ private ClientProtocol proxy;
+ private ObjectStore objectStore;
+ private ReplicationType replicationType;
+ private ReplicationFactor replicationFactor;
+ private boolean securityEnabled;
+ private int configuredDnPort;
+
+ /**
+ * Create new OzoneClientAdapter implementation.
+ *
+ * @throws IOException In case of a problem.
+ */
+ public BasicRootedOzoneClientAdapterImpl() throws IOException {
+ this(createConf());
+ }
+
+ private static OzoneConfiguration createConf() {
+ ClassLoader contextClassLoader =
+ Thread.currentThread().getContextClassLoader();
+ Thread.currentThread().setContextClassLoader(null);
+ try {
+ return new OzoneConfiguration();
+ } finally {
+ Thread.currentThread().setContextClassLoader(contextClassLoader);
+ }
+ }
+
+ public BasicRootedOzoneClientAdapterImpl(OzoneConfiguration conf)
+ throws IOException {
+ this(null, -1, conf);
+ }
+
+ public BasicRootedOzoneClientAdapterImpl(String omHost, int omPort,
+ Configuration hadoopConf) throws IOException {
+
+ ClassLoader contextClassLoader =
+ Thread.currentThread().getContextClassLoader();
+ Thread.currentThread().setContextClassLoader(null);
+
+ try {
+ OzoneConfiguration conf = OzoneConfiguration.of(hadoopConf);
+
+ if (omHost == null && OmUtils.isServiceIdsDefined(conf)) {
+ // When the host name or service id isn't given
+ // but ozone.om.service.ids is defined, declare failure.
+
+ // This is a safety precaution that prevents the client from
+ // accidentally failing over to an unintended OM.
+ throw new IllegalArgumentException("Service ID or host name must not"
+ + " be omitted when ozone.om.service.ids is defined.");
+ }
+
+ if (omPort != -1) {
+ // When the port number is specified, perform the following check
+ if (OmUtils.isOmHAServiceId(conf, omHost)) {
+ // If omHost is a service id, it shouldn't use a port
+ throw new IllegalArgumentException("Port " + omPort +
+ " specified in URI but host '" + omHost + "' is a "
+ + "logical (HA) OzoneManager and does not use port information.");
+ }
+ } else {
+ // When port number is not specified, read it from config
+ omPort = OmUtils.getOmRpcPort(conf);
+ }
+
+ SecurityConfig secConfig = new SecurityConfig(conf);
+
+ if (secConfig.isSecurityEnabled()) {
+ this.securityEnabled = true;
+ }
+
+ String replicationTypeConf =
+ conf.get(OzoneConfigKeys.OZONE_REPLICATION_TYPE,
+ OzoneConfigKeys.OZONE_REPLICATION_TYPE_DEFAULT);
+
+ int replicationCountConf = conf.getInt(OzoneConfigKeys.OZONE_REPLICATION,
+ OzoneConfigKeys.OZONE_REPLICATION_DEFAULT);
+
+ if (OmUtils.isOmHAServiceId(conf, omHost)) {
+ // omHost is listed as one of the service ids in the config,
+ // thus we should treat omHost as omServiceId
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(omHost, conf);
+ } else if (StringUtils.isNotEmpty(omHost) && omPort != -1) {
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(omHost, omPort, conf);
+ } else {
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(conf);
+ }
+ objectStore = ozoneClient.getObjectStore();
+ proxy = objectStore.getClientProxy();
+ this.replicationType = ReplicationType.valueOf(replicationTypeConf);
+ this.replicationFactor = ReplicationFactor.valueOf(replicationCountConf);
+ this.configuredDnPort = conf.getInt(
+ OzoneConfigKeys.DFS_CONTAINER_IPC_PORT,
+ OzoneConfigKeys.DFS_CONTAINER_IPC_PORT_DEFAULT);
+ } finally {
+ Thread.currentThread().setContextClassLoader(contextClassLoader);
+ }
+ }
+
+ OzoneBucket getBucket(OFSPath ofsPath, boolean createIfNotExist)
+ throws IOException {
+
+ return getBucket(ofsPath.getVolumeName(), ofsPath.getBucketName(),
+ createIfNotExist);
+ }
+
+ /**
+ * Get OzoneBucket object to operate in.
+ * Optionally create volume and bucket if not found.
+ *
+ * @param createIfNotExist Set this to true if the caller is a write operation
+ * in order to create the volume and bucket.
+ * @throws IOException Exceptions other than OMException with result code
+ * VOLUME_NOT_FOUND or BUCKET_NOT_FOUND.
+ */
+ private OzoneBucket getBucket(String volumeStr, String bucketStr,
+ boolean createIfNotExist) throws IOException {
+
+ OzoneBucket bucket;
+ try {
+ bucket = proxy.getBucketDetails(volumeStr, bucketStr);
+ } catch (OMException ex) {
+ if (createIfNotExist) {
+ // Note: getBucketDetails always throws BUCKET_NOT_FOUND, even if
+ // the volume doesn't exist.
+ if (ex.getResult().equals(BUCKET_NOT_FOUND)) {
+ OzoneVolume volume;
+ try {
+ volume = proxy.getVolumeDetails(volumeStr);
+ } catch (OMException getVolEx) {
+ if (getVolEx.getResult().equals(VOLUME_NOT_FOUND)) {
+ // Volume doesn't exist. Create it
+ try {
+ objectStore.createVolume(volumeStr);
+ } catch (OMException newVolEx) {
+ // Ignore the case where another client created the volume
+ if (!newVolEx.getResult().equals(VOLUME_ALREADY_EXISTS)) {
+ throw newVolEx;
+ }
+ }
+ } else {
+ throw getVolEx;
+ }
+ // Try get volume again
+ volume = proxy.getVolumeDetails(volumeStr);
+ }
+ // Create the bucket
+ try {
+ volume.createBucket(bucketStr);
+ } catch (OMException newBucEx) {
+ // Ignore the case where another client created the bucket
+ if (!newBucEx.getResult().equals(BUCKET_ALREADY_EXISTS)) {
Review comment:
what is permission denied, do we recover the environment by delete the volume?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] smengcl commented on a change in pull request #415:
HDDS-2840. Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
smengcl commented on a change in pull request #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#discussion_r372543587
##########
File path: hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
##########
@@ -0,0 +1,477 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.TestDataUtil;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClientException;
+import org.apache.hadoop.ozone.client.OzoneKeyDetails;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.Set;
+import java.util.TreeSet;
+
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/**
+ * Ozone file system tests that are not covered by contract tests.
+ */
+public class TestRootedOzoneFileSystem {
Review comment:
Added TODO.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] smengcl commented on issue #415: HDDS-2840.
Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
smengcl commented on issue #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#issuecomment-577933488
> I think here we need to catch exception from createVolume, as by the time we create volume, if another client created the same volume, we get VOLUME_ALREADY_EXCEPTION, so this should be safe and ignored.
>
> Same applicable for bucket creation also.
Nice catch. Addressed in the last commit.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] smengcl commented on a change in pull request #415:
HDDS-2840. Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
smengcl commented on a change in pull request #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#discussion_r372095369
##########
File path: hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
##########
@@ -0,0 +1,477 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.TestDataUtil;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClientException;
+import org.apache.hadoop.ozone.client.OzoneKeyDetails;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.Set;
+import java.util.TreeSet;
+
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/**
+ * Ozone file system tests that are not covered by contract tests.
+ */
+public class TestRootedOzoneFileSystem {
Review comment:
Indeed most code are different, but not just in the setup. In quite a few existing test cases I changed some subtle variable to make it run on o3fs, meaning we likely would need to change o3fs's unit test a bit as well (potentially more conflicts when merging this branch with trunk). I would suggest adding a TODO for this class and do the refactoring at a later point. What do you say?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] smengcl commented on a change in pull request #415:
HDDS-2840. Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
smengcl commented on a change in pull request #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#discussion_r372556302
##########
File path: hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
##########
@@ -0,0 +1,694 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.ozone;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.key.KeyProvider;
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdds.client.ReplicationFactor;
+import org.apache.hadoop.hdds.client.ReplicationType;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ozone.OmUtils;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.client.OzoneKey;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
+import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
+import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenRenewer;
+
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .BUCKET_ALREADY_EXISTS;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .VOLUME_ALREADY_EXISTS;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .VOLUME_NOT_FOUND;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .BUCKET_NOT_FOUND;
+
+/**
+ * Basic Implementation of the OzoneFileSystem calls.
+ * <p>
+ * This is the minimal version which doesn't include any statistics.
+ * <p>
+ * For full featured version use OzoneClientAdapterImpl.
+ */
+public class BasicRootedOzoneClientAdapterImpl
+ implements RootedOzoneClientAdapter {
+
+ static final Logger LOG =
+ LoggerFactory.getLogger(BasicRootedOzoneClientAdapterImpl.class);
+
+ private OzoneClient ozoneClient;
+ private ClientProtocol proxy;
+ private ObjectStore objectStore;
+ private ReplicationType replicationType;
+ private ReplicationFactor replicationFactor;
+ private boolean securityEnabled;
+ private int configuredDnPort;
+
+ /**
+ * Create new OzoneClientAdapter implementation.
+ *
+ * @throws IOException In case of a problem.
+ */
+ public BasicRootedOzoneClientAdapterImpl() throws IOException {
+ this(createConf());
+ }
+
+ private static OzoneConfiguration createConf() {
+ ClassLoader contextClassLoader =
+ Thread.currentThread().getContextClassLoader();
+ Thread.currentThread().setContextClassLoader(null);
+ try {
+ return new OzoneConfiguration();
+ } finally {
+ Thread.currentThread().setContextClassLoader(contextClassLoader);
+ }
+ }
+
+ public BasicRootedOzoneClientAdapterImpl(OzoneConfiguration conf)
+ throws IOException {
+ this(null, -1, conf);
+ }
+
+ public BasicRootedOzoneClientAdapterImpl(String omHost, int omPort,
+ Configuration hadoopConf) throws IOException {
+
+ ClassLoader contextClassLoader =
+ Thread.currentThread().getContextClassLoader();
+ Thread.currentThread().setContextClassLoader(null);
+
+ try {
+ OzoneConfiguration conf = OzoneConfiguration.of(hadoopConf);
+
+ if (omHost == null && OmUtils.isServiceIdsDefined(conf)) {
+ // When the host name or service id isn't given
+ // but ozone.om.service.ids is defined, declare failure.
+
+ // This is a safety precaution that prevents the client from
+ // accidentally failing over to an unintended OM.
+ throw new IllegalArgumentException("Service ID or host name must not"
+ + " be omitted when ozone.om.service.ids is defined.");
+ }
+
+ if (omPort != -1) {
+ // When the port number is specified, perform the following check
+ if (OmUtils.isOmHAServiceId(conf, omHost)) {
+ // If omHost is a service id, it shouldn't use a port
+ throw new IllegalArgumentException("Port " + omPort +
+ " specified in URI but host '" + omHost + "' is a "
+ + "logical (HA) OzoneManager and does not use port information.");
+ }
+ } else {
+ // When port number is not specified, read it from config
+ omPort = OmUtils.getOmRpcPort(conf);
+ }
+
+ SecurityConfig secConfig = new SecurityConfig(conf);
+
+ if (secConfig.isSecurityEnabled()) {
+ this.securityEnabled = true;
+ }
+
+ String replicationTypeConf =
+ conf.get(OzoneConfigKeys.OZONE_REPLICATION_TYPE,
+ OzoneConfigKeys.OZONE_REPLICATION_TYPE_DEFAULT);
+
+ int replicationCountConf = conf.getInt(OzoneConfigKeys.OZONE_REPLICATION,
+ OzoneConfigKeys.OZONE_REPLICATION_DEFAULT);
+
+ if (OmUtils.isOmHAServiceId(conf, omHost)) {
+ // omHost is listed as one of the service ids in the config,
+ // thus we should treat omHost as omServiceId
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(omHost, conf);
+ } else if (StringUtils.isNotEmpty(omHost) && omPort != -1) {
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(omHost, omPort, conf);
+ } else {
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(conf);
+ }
+ objectStore = ozoneClient.getObjectStore();
+ proxy = objectStore.getClientProxy();
+ this.replicationType = ReplicationType.valueOf(replicationTypeConf);
+ this.replicationFactor = ReplicationFactor.valueOf(replicationCountConf);
+ this.configuredDnPort = conf.getInt(
+ OzoneConfigKeys.DFS_CONTAINER_IPC_PORT,
+ OzoneConfigKeys.DFS_CONTAINER_IPC_PORT_DEFAULT);
+ } finally {
+ Thread.currentThread().setContextClassLoader(contextClassLoader);
+ }
+ }
+
+ OzoneBucket getBucket(OFSPath ofsPath, boolean createIfNotExist)
+ throws IOException {
+
+ return getBucket(ofsPath.getVolumeName(), ofsPath.getBucketName(),
+ createIfNotExist);
+ }
+
+ /**
+ * Get OzoneBucket object to operate in.
+ * Optionally create volume and bucket if not found.
+ *
+ * @param createIfNotExist Set this to true if the caller is a write operation
+ * in order to create the volume and bucket.
+ * @throws IOException Exceptions other than OMException with result code
+ * VOLUME_NOT_FOUND or BUCKET_NOT_FOUND.
+ */
+ private OzoneBucket getBucket(String volumeStr, String bucketStr,
+ boolean createIfNotExist) throws IOException {
+
+ OzoneBucket bucket;
+ try {
+ bucket = proxy.getBucketDetails(volumeStr, bucketStr);
+ } catch (OMException ex) {
+ if (createIfNotExist) {
+ // Note: getBucketDetails always throws BUCKET_NOT_FOUND, even if
+ // the volume doesn't exist.
+ if (ex.getResult().equals(BUCKET_NOT_FOUND)) {
+ OzoneVolume volume;
+ try {
+ volume = proxy.getVolumeDetails(volumeStr);
+ } catch (OMException getVolEx) {
+ if (getVolEx.getResult().equals(VOLUME_NOT_FOUND)) {
+ // Volume doesn't exist. Create it
+ try {
+ objectStore.createVolume(volumeStr);
+ } catch (OMException newVolEx) {
+ // Ignore the case where another client created the volume
+ if (!newVolEx.getResult().equals(VOLUME_ALREADY_EXISTS)) {
+ throw newVolEx;
+ }
+ }
+ } else {
+ throw getVolEx;
+ }
+ // Try get volume again
+ volume = proxy.getVolumeDetails(volumeStr);
+ }
+ // Create the bucket
+ try {
+ volume.createBucket(bucketStr);
+ } catch (OMException newBucEx) {
+ // Ignore the case where another client created the bucket
+ if (!newBucEx.getResult().equals(BUCKET_ALREADY_EXISTS)) {
+ throw newBucEx;
+ }
+ }
+ }
+ // Try get bucket again
+ bucket = proxy.getBucketDetails(volumeStr, bucketStr);
+ } else {
+ throw ex;
+ }
+ }
+
+ return bucket;
+ }
+
+ @Override
+ public void close() throws IOException {
+ ozoneClient.close();
+ }
+
+ @Override
+ public InputStream readFile(String pathStr) throws IOException {
+ incrementCounter(Statistic.OBJECTS_READ);
+ OFSPath ofsPath = new OFSPath(pathStr);
+ String key = ofsPath.getKeyName();
+ try {
+ OzoneBucket bucket = getBucket(ofsPath, false);
+ return bucket.readFile(key).getInputStream();
+ } catch (OMException ex) {
+ if (ex.getResult() == OMException.ResultCodes.FILE_NOT_FOUND
+ || ex.getResult() == OMException.ResultCodes.NOT_A_FILE) {
+ throw new FileNotFoundException(
+ ex.getResult().name() + ": " + ex.getMessage());
+ } else {
+ throw ex;
+ }
+ }
+ }
+
+ protected void incrementCounter(Statistic objectsRead) {
+ //noop: Use OzoneClientAdapterImpl which supports statistics.
+ }
+
+ @Override
+ public OzoneFSOutputStream createFile(String pathStr, boolean overWrite,
+ boolean recursive) throws IOException {
+ incrementCounter(Statistic.OBJECTS_CREATED);
+ OFSPath ofsPath = new OFSPath(pathStr);
+ String key = ofsPath.getKeyName();
+ try {
+ OzoneBucket bucket = getBucket(ofsPath, false);
+ OzoneOutputStream ozoneOutputStream = bucket.createFile(
+ key, 0, replicationType, replicationFactor, overWrite, recursive);
+ return new OzoneFSOutputStream(ozoneOutputStream.getOutputStream());
+ } catch (OMException ex) {
+ if (ex.getResult() == OMException.ResultCodes.FILE_ALREADY_EXISTS
+ || ex.getResult() == OMException.ResultCodes.NOT_A_FILE) {
+ throw new FileAlreadyExistsException(
+ ex.getResult().name() + ": " + ex.getMessage());
+ } else {
+ throw ex;
+ }
+ }
+ }
+
+ @Override
+ public void renamePath(String path, String newPath) throws IOException {
+ incrementCounter(Statistic.OBJECTS_RENAMED);
+ OFSPath ofsPath = new OFSPath(path);
+ OFSPath ofsNewPath = new OFSPath(newPath);
+
+ // Check path and newPathName are in the same volume and same bucket.
+ // This should have been checked in BasicRootedOzoneFileSystem#rename
+ // already via regular call path unless bypassed.
+ if (!ofsPath.isInSameBucketAs(ofsNewPath)) {
+ throw new IOException("Cannot rename a key to a different bucket");
+ }
+
+ OzoneBucket bucket = getBucket(ofsPath, false);
+ String key = ofsPath.getKeyName();
+ String newKey = ofsNewPath.getKeyName();
+ bucket.renameKey(key, newKey);
+ }
+
+ /**
+ * Package-private helper function to reduce calls to getBucket().
+ * @param bucket Bucket to operate in.
+ * @param path Existing key path.
+ * @param newPath New key path.
+ * @throws IOException IOException from bucket.renameKey().
+ */
+ void renamePath(OzoneBucket bucket, String path, String newPath)
+ throws IOException {
+ incrementCounter(Statistic.OBJECTS_RENAMED);
+ OFSPath ofsPath = new OFSPath(path);
+ OFSPath ofsNewPath = new OFSPath(newPath);
+ // No same-bucket policy check here since this call path is controlled
+ String key = ofsPath.getKeyName();
+ String newKey = ofsNewPath.getKeyName();
+ bucket.renameKey(key, newKey);
+ }
+
+ /**
+ * Helper method to create an directory specified by key name in bucket.
+ *
+ * @param pathStr path to be created as directory
+ * @return true if the key is created, false otherwise
+ */
+ @Override
+ public boolean createDirectory(String pathStr) throws IOException {
+ LOG.trace("creating dir for path: {}", pathStr);
+ incrementCounter(Statistic.OBJECTS_CREATED);
+ OFSPath ofsPath = new OFSPath(pathStr);
+ // Volume name unspecified, return failure
+ if (ofsPath.getVolumeName().length() == 0) {
+ return false;
+ }
+ // Handle where only volume is specified in pathStr
+ if (ofsPath.getBucketName().length() == 0) {
+ objectStore.createVolume(ofsPath.getVolumeName());
+ return true;
+ }
+ String keyStr = ofsPath.getKeyName();
+ try {
+ OzoneBucket bucket = getBucket(ofsPath, true);
+ // if keyStr is empty, it indicates that only volume or volume+bucket is
+ // given in pathStr, so getBucket() above should've handled the creation
+ // of volume/bucket already.
+ if (keyStr != null && keyStr.length() > 0) {
+ bucket.createDirectory(keyStr);
+ }
+ } catch (OMException e) {
+ if (e.getResult() == OMException.ResultCodes.FILE_ALREADY_EXISTS) {
+ throw new FileAlreadyExistsException(e.getMessage());
+ }
+ throw e;
+ }
+ return true;
+ }
+
+ /**
+ * Helper method to delete an object specified by key name in bucket.
+ *
+ * @param path path to a key to be deleted
+ * @return true if the key is deleted, false otherwise
+ */
+ @Override
+ public boolean deleteObject(String path) {
+ LOG.trace("issuing delete for path to key: {}", path);
+ OFSPath ofsPath = new OFSPath(path);
+ String keyName = ofsPath.getKeyName();
+ try {
+ OzoneBucket bucket = getBucket(ofsPath, false);
+ incrementCounter(Statistic.OBJECTS_DELETED);
+ bucket.deleteKey(keyName);
+ return true;
+ } catch (IOException ioe) {
+ LOG.error("delete key failed " + ioe.getMessage());
+ return false;
+ }
+ }
+
+ /**
+ * Package-private helper function to reduce calls to getBucket().
+ * @param bucket Bucket to operate in.
+ * @param path Path to delete.
+ * @return true if operation succeeded, false upon IOException.
+ */
+ boolean deleteObject(OzoneBucket bucket, String path) {
+ LOG.trace("issuing delete for path to key: {}", path);
+ OFSPath ofsPath = new OFSPath(path);
+ String keyName = ofsPath.getKeyName();
+ try {
+ incrementCounter(Statistic.OBJECTS_DELETED);
+ bucket.deleteKey(keyName);
+ return true;
+ } catch (IOException ioe) {
+ LOG.error("delete key failed " + ioe.getMessage());
+ return false;
+ }
+ }
+
+ public FileStatusAdapter getFileStatus(String path, URI uri,
+ Path qualifiedPath, String userName)
+ throws IOException {
+ OFSPath ofsPath = new OFSPath(path);
+ String key = ofsPath.getKeyName();
+ try {
+ OzoneBucket bucket = getBucket(ofsPath, false);
+ incrementCounter(Statistic.OBJECTS_QUERY);
Review comment:
Done in a9ef567103a13fd57d2c63197e7e356b447cafd7
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] hanishakoneru commented on a change in pull request
#415: HDDS-2840. Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
hanishakoneru commented on a change in pull request #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#discussion_r369309173
##########
File path: hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
##########
@@ -0,0 +1,408 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.TestDataUtil;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClientException;
+import org.apache.hadoop.ozone.client.OzoneKeyDetails;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.Set;
+import java.util.TreeSet;
+
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/**
+ * Ozone file system tests that are not covered by contract tests.
+ */
+public class TestRootedOzoneFileSystem {
+
+ @Rule
+ public Timeout globalTimeout = new Timeout(300_000);
+
+ private static MiniOzoneCluster cluster = null;
+
+ private static FileSystem fs;
+ private static RootedOzoneFileSystem ofs;
+
+ private static ObjectStore objectStore;
+
+ private String volumeName;
+ private String bucketName;
+
+ private String rootPath;
+
+ @Before
+ public void init() throws Exception {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ cluster = MiniOzoneCluster.newBuilder(conf)
+ .setNumDatanodes(3)
+ .build();
+ cluster.waitForClusterToBeReady();
+ objectStore = cluster.getClient().getObjectStore();
+
+ // create a volume and a bucket to be used by OFileSystem
+ OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(cluster);
+ volumeName = bucket.getVolumeName();
+ bucketName = bucket.getName();
+
+ // For now:
+ rootPath = String.format("%s://%s/", OzoneConsts.OZONE_OFS_URI_SCHEME,
+ conf.get(OZONE_OM_ADDRESS_KEY));
+
+ // Set the fs.defaultFS and start the filesystem
+ conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, rootPath);
+ // Note: FileSystem#loadFileSystems won't load OFS class due to META-INF
+ // hence this workaround.
+ conf.set("fs.ofs.impl", "org.apache.hadoop.fs.ozone.RootedOzoneFileSystem");
+ fs = FileSystem.get(conf);
+ ofs = (RootedOzoneFileSystem) fs;
+ }
+
+ @After
+ public void teardown() throws IOException {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ IOUtils.closeQuietly(fs);
+ }
+
+ @Test
+ public void testOzoneFsServiceLoader() throws IOException {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ // Note: FileSystem#loadFileSystems won't load OFS class due to META-INF
+ // hence this workaround.
+ conf.set("fs.ofs.impl", "org.apache.hadoop.fs.ozone.RootedOzoneFileSystem");
+ assertEquals(
+ FileSystem.getFileSystemClass(OzoneConsts.OZONE_OFS_URI_SCHEME, conf),
+ RootedOzoneFileSystem.class);
+ }
+
+ @Test
+ public void testCreateDoesNotAddParentDirKeys() throws Exception {
+ Path rootBucket = new Path("/" + volumeName + "/" + bucketName);
+ Path grandparent = new Path(rootBucket,
+ "testCreateDoesNotAddParentDirKeys");
+ Path parent = new Path(grandparent, "parent");
+ Path child = new Path(parent, "child");
+ ContractTestUtils.touch(fs, child);
+
+ OzoneKeyDetails key = getKeyInBucket(child, false);
+ OFSPath childOFSPath = new OFSPath(child);
+ assertEquals(key.getName(), childOFSPath.getKeyName());
+
+ // Creating a child should not add parent keys to the bucket
+ try {
+ getKeyInBucket(parent, true);
+ } catch (IOException ex) {
+ assertKeyNotFoundException(ex);
+ }
+
+ // List status on the parent should show the child file
+ assertEquals("List status of parent should include the 1 child file", 1L,
+ (long)fs.listStatus(parent).length);
+ assertTrue("Parent directory does not appear to be a directory",
+ fs.getFileStatus(parent).isDirectory());
+ }
+
+ @Test
+ public void testDeleteCreatesFakeParentDir() throws Exception {
+ Path rootBucket = new Path("/" + volumeName + "/" + bucketName);
+ Path grandparent = new Path(rootBucket, "testDeleteCreatesFakeParentDir");
+ Path parent = new Path(grandparent, "parent");
+ Path child = new Path(parent, "child");
+ ContractTestUtils.touch(fs, child);
+
+ // Verify that parent dir key does not exist
+ // Creating a child should not add parent keys to the bucket
+ try {
+ getKeyInBucket(parent, true);
+ } catch (IOException ex) {
+ assertKeyNotFoundException(ex);
+ }
+
+ // Delete the child key
+ assertTrue(fs.delete(child, false));
+
+ // Deleting the only child should create the parent dir key if it does
+ // not exist
+ OFSPath parentOFSPath = new OFSPath(parent);
+ String parentKey = parentOFSPath.getKeyName() + "/";
+ OzoneKeyDetails parentKeyInfo = getKeyInBucket(parent, true);
+ assertEquals(parentKey, parentKeyInfo.getName());
+
+ // Recursive delete with DeleteIterator
+ assertTrue(fs.delete(grandparent, true));
+ }
+
+ @Test
+ public void testListStatus() throws Exception {
+ Path rootBucket = new Path("/" + volumeName + "/" + bucketName);
+ Path parent = new Path(rootBucket, "testListStatus");
+ Path file1 = new Path(parent, "key1");
+ Path file2 = new Path(parent, "key2");
+
+ FileStatus[] fileStatuses = ofs.listStatus(rootBucket);
+ assertEquals("Should be empty", 0, fileStatuses.length);
+
+ ContractTestUtils.touch(fs, file1);
+ ContractTestUtils.touch(fs, file2);
+
+ // ListStatus on a directory should return all subdirs along with
+ // files, even if there exists a file and sub-dir with the same name.
+ fileStatuses = ofs.listStatus(parent);
+ assertEquals("FileStatus did not return all children of the directory",
+ 2, fileStatuses.length);
+
+ // ListStatus should return only the immediate children of a directory.
+ Path file3 = new Path(parent, "dir1/key3");
+ Path file4 = new Path(parent, "dir1/key4");
+ ContractTestUtils.touch(fs, file3);
+ ContractTestUtils.touch(fs, file4);
+ fileStatuses = ofs.listStatus(parent);
+ assertEquals("FileStatus did not return all children of the directory",
+ 3, fileStatuses.length);
+ }
+
+ private String getRandomNonExistVolumeName() throws Exception {
+ final int numDigit = 5;
+ long retriesLeft = Math.round(Math.pow(10, 5));
+ String name = null;
+ while (name == null && retriesLeft-- > 0) {
+ name = "volume-" + RandomStringUtils.randomNumeric(numDigit);
+ // Check volume existence.
+ Iterator<? extends OzoneVolume> iter =
+ objectStore.listVolumesByUser(null, name, null);
+ if (iter.hasNext()) {
+ // If there is a match, try again.
+ // Note that volume name prefix match doesn't equal volume existence
+ // but the check is sufficient for this test.
+ name = null;
+ }
+ }
+ if (retriesLeft <= 0) {
+ throw new Exception(
+ "Failed to generate random volume name that doesn't exist already.");
+ }
+ return name;
+ }
+
+ /**
+ * Tests Mkdir operation on a volume that doesn't exist.
+ * Expect Mkdir to create the volume and bucket.
+ */
+ @Test
+ public void testMkdirOnNonExistentVolumeBucket() throws Exception {
+ String volumeNameLocal = getRandomNonExistVolumeName();
+ String bucketNameLocal = "bucket-" + RandomStringUtils.randomNumeric(5);
+ Path root = new Path("/" + volumeNameLocal + "/" + bucketNameLocal);
+ Path dir1 = new Path(root, "dir1");
+ Path dir12 = new Path(dir1, "dir12");
+ Path dir2 = new Path(root, "dir2");
+ fs.mkdirs(dir12);
+ fs.mkdirs(dir2);
+
+ // Check volume and bucket existence, they should both be created.
+ OzoneVolume ozoneVolume = objectStore.getVolume(volumeNameLocal);
+ OzoneBucket ozoneBucket = ozoneVolume.getBucket(bucketNameLocal);
+ OFSPath ofsPathDir1 = new OFSPath(dir12);
+ String key = ofsPathDir1.getKeyName() + "/";
+ OzoneKeyDetails ozoneKeyDetails = ozoneBucket.getKey(key);
+ assertEquals(key, ozoneKeyDetails.getName());
+
+ // Verify that directories are created.
+ FileStatus[] fileStatuses = ofs.listStatus(root);
+ assertEquals(fileStatuses[0].getPath().toUri().getPath(), dir1.toString());
+ assertEquals(fileStatuses[1].getPath().toUri().getPath(), dir2.toString());
+
+ fileStatuses = ofs.listStatus(dir1);
+ assertEquals(fileStatuses[0].getPath().toUri().getPath(), dir12.toString());
+ fileStatuses = ofs.listStatus(dir12);
+ assertEquals(fileStatuses.length, 0);
+ fileStatuses = ofs.listStatus(dir2);
+ assertEquals(fileStatuses.length, 0);
+ }
+
+ /**
+ * Tests listStatus operation on root directory.
+ */
+ @Test
+ public void testListStatusOnRoot() throws Exception {
+ Path root = new Path("/" + volumeName + "/" + bucketName);
+ Path dir1 = new Path(root, "dir1");
+ Path dir12 = new Path(dir1, "dir12");
+ Path dir2 = new Path(root, "dir2");
+ fs.mkdirs(dir12);
+ fs.mkdirs(dir2);
+
+ // ListStatus on root should return dir1 (even though /dir1 key does not
+ // exist) and dir2 only. dir12 is not an immediate child of root and
+ // hence should not be listed.
+ FileStatus[] fileStatuses = ofs.listStatus(root);
+ assertEquals("FileStatus should return only the immediate children", 2,
+ fileStatuses.length);
+
+ // Verify that dir12 is not included in the result of the listStatus on root
+ String fileStatus1 = fileStatuses[0].getPath().toUri().getPath();
+ String fileStatus2 = fileStatuses[1].getPath().toUri().getPath();
+ assertFalse(fileStatus1.equals(dir12.toString()));
+ assertFalse(fileStatus2.equals(dir12.toString()));
+ }
+
+ /**
+ * Tests listStatus operation on root directory.
+ */
+ @Test
+ public void testListStatusOnLargeDirectory() throws Exception {
+ Path root = new Path("/" + volumeName + "/" + bucketName);
+ Set<String> paths = new TreeSet<>();
+ int numDirs = 5111;
+ for(int i = 0; i < numDirs; i++) {
+ Path p = new Path(root, String.valueOf(i));
+ fs.mkdirs(p);
+ paths.add(p.getName());
+ }
+
+ FileStatus[] fileStatuses = ofs.listStatus(root);
+ assertEquals(
+ "Total directories listed do not match the existing directories",
+ numDirs, fileStatuses.length);
+
+ for (int i=0; i < numDirs; i++) {
+ assertTrue(paths.contains(fileStatuses[i].getPath().getName()));
+ }
+ }
+
+ /**
+ * Tests listStatus on a path with subdirs.
+ */
+ @Test
+ public void testListStatusOnSubDirs() throws Exception {
+ // Create the following key structure
+ // /dir1/dir11/dir111
+ // /dir1/dir12
+ // /dir1/dir12/file121
+ // /dir2
+ // ListStatus on /dir1 should return all its immediated subdirs only
+ // which are /dir1/dir11 and /dir1/dir12. Super child files/dirs
+ // (/dir1/dir12/file121 and /dir1/dir11/dir111) should not be returned by
+ // listStatus.
+ Path rootBucket = new Path("/" + volumeName + "/" + bucketName);
+ Path dir1 = new Path(rootBucket, "dir1");
+ Path dir11 = new Path(dir1, "dir11");
+ Path dir111 = new Path(dir11, "dir111");
+ Path dir12 = new Path(dir1, "dir12");
+ Path file121 = new Path(dir12, "file121");
+ Path dir2 = new Path(rootBucket, "dir2");
+ fs.mkdirs(dir111);
+ fs.mkdirs(dir12);
+ ContractTestUtils.touch(fs, file121);
+ fs.mkdirs(dir2);
+
+ FileStatus[] fileStatuses = ofs.listStatus(dir1);
+ assertEquals("FileStatus should return only the immediate children", 2,
+ fileStatuses.length);
+
+ // Verify that the two children of /dir1 returned by listStatus operation
+ // are /dir1/dir11 and /dir1/dir12.
+ String fileStatus1 = fileStatuses[0].getPath().toUri().getPath();
+ String fileStatus2 = fileStatuses[1].getPath().toUri().getPath();
+ assertTrue(fileStatus1.equals(dir11.toString()) ||
+ fileStatus1.equals(dir12.toString()));
+ assertTrue(fileStatus2.equals(dir11.toString()) ||
+ fileStatus2.equals(dir12.toString()));
+ }
+
+ @Test
+ public void testNonExplicitlyCreatedPathExistsAfterItsLeafsWereRemoved()
+ throws Exception {
+ Path rootBucket = new Path("/" + volumeName + "/" + bucketName);
+ Path source = new Path(rootBucket, "source");
+ Path interimPath = new Path(source, "interimPath");
+ Path leafInsideInterimPath = new Path(interimPath, "leaf");
+ Path target = new Path(rootBucket, "target");
+ Path leafInTarget = new Path(target, "leaf");
+
+ fs.mkdirs(source);
+ fs.mkdirs(target);
+ fs.mkdirs(leafInsideInterimPath);
+
+ // Attempt to rename the key to a different bucket
+ Path bucket2 = new Path("/" + volumeName + "/" + bucketName + "test");
+ Path leafInTargetInAnotherBucket = new Path(bucket2, "leaf");
+ try {
+ fs.rename(leafInsideInterimPath, leafInTargetInAnotherBucket);
+ fail("Should have thrown exception when renaming to a different bucket");
+ } catch (IOException ex) {
+ System.out.println("Exception thrown as expected");
+ }
+
+ assertTrue(fs.rename(leafInsideInterimPath, leafInTarget));
+
+ // after rename listStatus for interimPath should succeed and
+ // interimPath should have no children
+ FileStatus[] statuses = fs.listStatus(interimPath);
+ assertNotNull("liststatus returns a null array", statuses);
+ assertEquals("Statuses array is not empty", 0, statuses.length);
+ FileStatus fileStatus = fs.getFileStatus(interimPath);
+ assertEquals("FileStatus does not point to interimPath",
+ interimPath.getName(), fileStatus.getPath().getName());
+ }
+
+ private OzoneKeyDetails getKeyInBucket(Path keyPath, boolean isDirectory)
Review comment:
getKeyInBucket looks like we are getting key from a specific bucket and not the global bucket. Can we rename this back to getKey?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] smengcl commented on a change in pull request #415:
HDDS-2840. Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
smengcl commented on a change in pull request #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#discussion_r371275166
##########
File path: hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystemWithMocks.java
##########
@@ -0,0 +1,152 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.ozone;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.OmUtils;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PowerMockIgnore;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+
+import java.net.URI;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * Ozone File system tests that are light weight and use mocks.
+ */
+@RunWith(PowerMockRunner.class)
+@PrepareForTest({ OzoneClientFactory.class, UserGroupInformation.class })
+@PowerMockIgnore("javax.management.*")
+public class TestRootedOzoneFileSystemWithMocks {
+
+ @Test
+ public void testFSUriWithHostPortOverrides() throws Exception {
+ Configuration conf = new OzoneConfiguration();
+ OzoneClient ozoneClient = mock(OzoneClient.class);
+ ObjectStore objectStore = mock(ObjectStore.class);
+ OzoneVolume volume = mock(OzoneVolume.class);
+ OzoneBucket bucket = mock(OzoneBucket.class);
+
+ when(ozoneClient.getObjectStore()).thenReturn(objectStore);
+ when(objectStore.getVolume(eq("volume1"))).thenReturn(volume);
+ when(volume.getBucket("bucket1")).thenReturn(bucket);
+
+ PowerMockito.mockStatic(OzoneClientFactory.class);
+ PowerMockito.when(OzoneClientFactory.getRpcClient(eq("local.host"),
+ eq(5899), eq(conf))).thenReturn(ozoneClient);
+
+ UserGroupInformation ugi = mock(UserGroupInformation.class);
+ PowerMockito.mockStatic(UserGroupInformation.class);
+ PowerMockito.when(UserGroupInformation.getCurrentUser()).thenReturn(ugi);
+ when(ugi.getShortUserName()).thenReturn("user1");
+
+ // Note: FileSystem#loadFileSystems doesn't load OFS class because
+ // META-INF still points to org.apache.hadoop.fs.ozone.OzoneFileSystem
+ conf.set("fs.ofs.impl", "org.apache.hadoop.fs.ozone.RootedOzoneFileSystem");
+ URI uri = new URI("ofs://local.host:5899/volume1/bucket1");
+
+ FileSystem fileSystem = FileSystem.get(uri, conf);
+ RootedOzoneFileSystem ofs = (RootedOzoneFileSystem) fileSystem;
+
+ assertEquals(ofs.getUri().getAuthority(), "local.host:5899");
+ PowerMockito.verifyStatic();
+ OzoneClientFactory.getRpcClient("local.host", 5899, conf);
+ }
+
+ @Test
+ public void testFSUriWithHostPortUnspecified() throws Exception {
+ Configuration conf = new OzoneConfiguration();
+ final int omPort = OmUtils.getOmRpcPort(conf);
+
+ OzoneClient ozoneClient = mock(OzoneClient.class);
+ ObjectStore objectStore = mock(ObjectStore.class);
+ OzoneVolume volume = mock(OzoneVolume.class);
+ OzoneBucket bucket = mock(OzoneBucket.class);
+
+ when(ozoneClient.getObjectStore()).thenReturn(objectStore);
+ when(objectStore.getVolume(eq("volume1"))).thenReturn(volume);
+ when(volume.getBucket("bucket1")).thenReturn(bucket);
+
+ PowerMockito.mockStatic(OzoneClientFactory.class);
+ PowerMockito.when(OzoneClientFactory.getRpcClient(eq("local.host"),
+ eq(omPort), eq(conf))).thenReturn(ozoneClient);
+
+ UserGroupInformation ugi = mock(UserGroupInformation.class);
+ PowerMockito.mockStatic(UserGroupInformation.class);
+ PowerMockito.when(UserGroupInformation.getCurrentUser()).thenReturn(ugi);
+ when(ugi.getShortUserName()).thenReturn("user1");
+
+ URI uri = new URI("o3fs://bucket1.volume1.local.host");
+
Review comment:
Done in 5a20f9da30a25fd40fa1a73e6b0e742f40647a94.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] hanishakoneru commented on a change in pull request
#415: HDDS-2840. Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
hanishakoneru commented on a change in pull request #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#discussion_r369801507
##########
File path: hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
##########
@@ -0,0 +1,805 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.ozone;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.CreateFlag;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.Progressable;
+import org.apache.http.client.utils.URIBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.EnumSet;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+import static org.apache.hadoop.fs.ozone.Constants.LISTING_PAGE_SIZE;
+import static org.apache.hadoop.fs.ozone.Constants.OZONE_DEFAULT_USER;
+import static org.apache.hadoop.fs.ozone.Constants.OZONE_USER_DIR;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_OFS_URI_SCHEME;
+
+/**
+ * The minimal Ozone Filesystem implementation.
+ * <p>
+ * This is a basic version which doesn't extend
+ * KeyProviderTokenIssuer and doesn't include statistics. It can be used
+ * from older hadoop version. For newer hadoop version use the full featured
+ * OFileSystem.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class BasicRootedOzoneFileSystem extends FileSystem {
+ static final Logger LOG =
+ LoggerFactory.getLogger(BasicRootedOzoneFileSystem.class);
+
+ /**
+ * The Ozone client for connecting to Ozone server.
+ */
+
+ private URI uri;
+ private String userName;
+ private Path workingDir;
+// private String gOmHost;
+// private int gOmPort;
+// private Configuration gConf;
+// private boolean gIsolatedClassloader;
+
+ private RootedOzoneClientAdapter adapter;
+// private String adapterPath;
+
+ private static final String URI_EXCEPTION_TEXT =
+ "URL should be one of the following formats: " +
+ "ofs://om-service-id/ OR " +
+ "ofs://om-host.example.com/ OR " +
+ "ofs://om-host.example.com:5678/";
+
+ @Override
+ public void initialize(URI name, Configuration conf) throws IOException {
+ super.initialize(name, conf);
+ setConf(conf);
+ Objects.requireNonNull(name.getScheme(), "No scheme provided in " + name);
+ Preconditions.checkArgument(getScheme().equals(name.getScheme()),
+ "Invalid scheme provided in " + name);
+
+ String authority = name.getAuthority();
+ if (authority == null) {
+ // authority is null when fs.defaultFS is not a qualified ofs URI and
+ // ofs:/// is passed to the client. matcher will NPE if authority is null
+ throw new IllegalArgumentException(URI_EXCEPTION_TEXT);
+ }
+
+ String omHost;
+ int omPort = -1;
+ // Parse hostname and port
+ String[] parts = authority.split(":");
+ if (parts.length > 2) {
+ throw new IllegalArgumentException(URI_EXCEPTION_TEXT);
+ }
+ omHost = parts[0];
+ if (parts.length == 2) {
+ try {
+ omPort = Integer.parseInt(parts[1]);
+ } catch (NumberFormatException e) {
+ throw new IllegalArgumentException(URI_EXCEPTION_TEXT);
+ }
+ }
+
+ try {
+ uri = new URIBuilder().setScheme(OZONE_OFS_URI_SCHEME)
+ .setHost(authority)
+ .build();
+ LOG.trace("Ozone URI for OFS initialization is " + uri);
+
+ //isolated is the default for ozonefs-lib-legacy which includes the
+ // /ozonefs.txt, otherwise the default is false. It could be overridden.
+ boolean defaultValue =
+ BasicRootedOzoneFileSystem.class.getClassLoader()
+ .getResource("ozonefs.txt") != null;
+
+ //Use string here instead of the constant as constant may not be available
+ //on the classpath of a hadoop 2.7
+ boolean isolatedClassloader =
+ conf.getBoolean("ozone.fs.isolated-classloader", defaultValue);
+
+ // adapter should be initialized in operations.
+ this.adapter = createAdapter(conf, omHost, omPort, isolatedClassloader);
+
+ try {
+ this.userName =
+ UserGroupInformation.getCurrentUser().getShortUserName();
+ } catch (IOException e) {
+ this.userName = OZONE_DEFAULT_USER;
+ }
+ this.workingDir = new Path(OZONE_USER_DIR, this.userName)
+ .makeQualified(this.uri, this.workingDir);
+ } catch (URISyntaxException ue) {
+ final String msg = "Invalid Ozone endpoint " + name;
+ LOG.error(msg, ue);
+ throw new IOException(msg, ue);
+ }
+ }
+
+ protected RootedOzoneClientAdapter createAdapter(Configuration conf,
+ String omHost, int omPort, boolean isolatedClassloader)
+ throws IOException {
+
+ if (isolatedClassloader) {
+ // TODO: Check how this code path need to be changed, for legacy Hadoop?
+ return RootedOzoneClientAdapterFactory.createAdapter();
+ } else {
+ return new BasicRootedOzoneClientAdapterImpl(omHost, omPort, conf);
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ try {
+ adapter.close();
+ } finally {
+ super.close();
+ }
+ }
+
+ @Override
+ public URI getUri() {
+ return uri;
+ }
+
+ @Override
+ public String getScheme() {
+ return OZONE_OFS_URI_SCHEME;
+ }
+
+ @Override
+ public FSDataInputStream open(Path path, int bufferSize) throws IOException {
+ incrementCounter(Statistic.INVOCATION_OPEN);
+ statistics.incrementReadOps(1);
+ LOG.trace("open() path: {}", path);
+ final String key = pathToKey(path);
+ return new FSDataInputStream(
+ new OzoneFSInputStream(adapter.readFile(key), statistics));
+ }
+
+ protected void incrementCounter(Statistic statistic) {
+ //don't do anything in this default implementation.
+ }
+
+ @Override
+ public FSDataOutputStream create(Path f, FsPermission permission,
+ boolean overwrite, int bufferSize,
+ short replication, long blockSize,
+ Progressable progress) throws IOException {
+ LOG.trace("create() path:{}", f);
+ incrementCounter(Statistic.INVOCATION_CREATE);
+ statistics.incrementWriteOps(1);
+ final String key = pathToKey(f);
+ return createOutputStream(key, overwrite, true);
+ }
+
+ @Override
+ public FSDataOutputStream createNonRecursive(Path path,
+ FsPermission permission,
+ EnumSet<CreateFlag> flags,
+ int bufferSize,
+ short replication,
+ long blockSize,
+ Progressable progress) throws IOException {
+ incrementCounter(Statistic.INVOCATION_CREATE_NON_RECURSIVE);
+ statistics.incrementWriteOps(1);
+ final String key = pathToKey(path);
+ return createOutputStream(key, flags.contains(CreateFlag.OVERWRITE), false);
+ }
+
+ private FSDataOutputStream createOutputStream(String key, boolean overwrite,
+ boolean recursive) throws IOException {
+ return new FSDataOutputStream(adapter.createFile(key, overwrite, recursive),
+ statistics);
+ }
+
+ @Override
+ public FSDataOutputStream append(Path f, int bufferSize,
+ Progressable progress) throws IOException {
+ throw new UnsupportedOperationException("append() Not implemented by the "
+ + getClass().getSimpleName() + " FileSystem implementation");
+ }
+
+ private class RenameIterator extends OzoneListingIterator {
+ private final String srcPath;
+ private final String dstPath;
+
+ RenameIterator(Path srcPath, Path dstPath)
+ throws IOException {
+ super(srcPath);
+ this.srcPath = pathToKey(srcPath);
+ this.dstPath = pathToKey(dstPath);
+ LOG.trace("rename from:{} to:{}", this.srcPath, this.dstPath);
+ }
+
+ @Override
+ boolean processKeyPath(String keyPath) throws IOException {
+ String newPath = dstPath.concat(keyPath.substring(srcPath.length()));
+ adapter.renamePath(keyPath, newPath);
Review comment:
When renaming a directory, we iterate through all the keys in that directory. For each key, we are going to call adapter.renamePath(). This function would in turn get BucketInfo from OM each time. We should optimize this to avoid the extra getBucketInfo calls.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] xiaoyuyao commented on a change in pull request
#415: HDDS-2840. Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
xiaoyuyao commented on a change in pull request #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#discussion_r372001578
##########
File path: hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
##########
@@ -0,0 +1,477 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.TestDataUtil;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClientException;
+import org.apache.hadoop.ozone.client.OzoneKeyDetails;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.Set;
+import java.util.TreeSet;
+
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/**
+ * Ozone file system tests that are not covered by contract tests.
+ */
+public class TestRootedOzoneFileSystem {
Review comment:
The test cases are very similar to TestOzoneFileSystem, most of the difference are in setup.
Can we refactor to have a base class that contains all the common test methods and leave only the setup in the subclass for different file system? (This is the approach taken in the contract test to avoid duplicate code)
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] xiaoyuyao commented on a change in pull request
#415: HDDS-2840. Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
xiaoyuyao commented on a change in pull request #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#discussion_r372074995
##########
File path: hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OFSPath.java
##########
@@ -0,0 +1,143 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.ozone;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.yetus.audience.InterfaceAudience;
+import org.apache.yetus.audience.InterfaceStability;
+import java.util.StringTokenizer;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
+
+/**
+ * Utility class for Rooted Ozone Filesystem (OFS) path processing.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+class OFSPath {
+ /**
+ * Here is a table illustrating what each name variable is given an input path
+ * Assuming /tmp is mounted to /tempVol/tempBucket
Review comment:
How is the /tempVol/tempBucket created? Do we provide a CLI to admin?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] smengcl commented on a change in pull request #415:
HDDS-2840. Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
smengcl commented on a change in pull request #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#discussion_r372095369
##########
File path: hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
##########
@@ -0,0 +1,477 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.TestDataUtil;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClientException;
+import org.apache.hadoop.ozone.client.OzoneKeyDetails;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.Set;
+import java.util.TreeSet;
+
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/**
+ * Ozone file system tests that are not covered by contract tests.
+ */
+public class TestRootedOzoneFileSystem {
Review comment:
Indeed most code are the same, but not just in the setup. In quite a few existing test cases I changed some subtle variable to make it run on o3fs, meaning we likely would need to change o3fs's unit test a bit as well (potentially more conflicts when merging this branch with trunk). I would suggest adding a TODO for this class and do the refactoring at a later point. What do you say?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] hanishakoneru commented on a change in pull request
#415: HDDS-2840. Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
hanishakoneru commented on a change in pull request #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#discussion_r369832336
##########
File path: hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OFSPath.java
##########
@@ -0,0 +1,137 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.ozone;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.yetus.audience.InterfaceAudience;
+import org.apache.yetus.audience.InterfaceStability;
+import java.util.StringTokenizer;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
+
+/**
+ * Utility class for Rooted Ozone Filesystem (OFS) path processing.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+class OFSPath {
+ private String volumeName;
+ private String bucketName;
+ private String mountName;
+ private String keyName;
Review comment:
Can you please add comments explaining how vol, bucket and mounts are used.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] smengcl commented on a change in pull request #415:
HDDS-2840. Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
smengcl commented on a change in pull request #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#discussion_r371248747
##########
File path: hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
##########
@@ -0,0 +1,805 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.ozone;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.CreateFlag;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.Progressable;
+import org.apache.http.client.utils.URIBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.EnumSet;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+import static org.apache.hadoop.fs.ozone.Constants.LISTING_PAGE_SIZE;
+import static org.apache.hadoop.fs.ozone.Constants.OZONE_DEFAULT_USER;
+import static org.apache.hadoop.fs.ozone.Constants.OZONE_USER_DIR;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_OFS_URI_SCHEME;
+
+/**
+ * The minimal Ozone Filesystem implementation.
+ * <p>
+ * This is a basic version which doesn't extend
+ * KeyProviderTokenIssuer and doesn't include statistics. It can be used
+ * from older hadoop version. For newer hadoop version use the full featured
+ * OFileSystem.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class BasicRootedOzoneFileSystem extends FileSystem {
+ static final Logger LOG =
+ LoggerFactory.getLogger(BasicRootedOzoneFileSystem.class);
+
+ /**
+ * The Ozone client for connecting to Ozone server.
+ */
+
+ private URI uri;
+ private String userName;
+ private Path workingDir;
+// private String gOmHost;
+// private int gOmPort;
+// private Configuration gConf;
+// private boolean gIsolatedClassloader;
+
+ private RootedOzoneClientAdapter adapter;
+// private String adapterPath;
+
+ private static final String URI_EXCEPTION_TEXT =
+ "URL should be one of the following formats: " +
+ "ofs://om-service-id/ OR " +
+ "ofs://om-host.example.com/ OR " +
+ "ofs://om-host.example.com:5678/";
+
+ @Override
+ public void initialize(URI name, Configuration conf) throws IOException {
+ super.initialize(name, conf);
+ setConf(conf);
+ Objects.requireNonNull(name.getScheme(), "No scheme provided in " + name);
+ Preconditions.checkArgument(getScheme().equals(name.getScheme()),
+ "Invalid scheme provided in " + name);
+
+ String authority = name.getAuthority();
+ if (authority == null) {
+ // authority is null when fs.defaultFS is not a qualified ofs URI and
+ // ofs:/// is passed to the client. matcher will NPE if authority is null
+ throw new IllegalArgumentException(URI_EXCEPTION_TEXT);
+ }
+
+ String omHost;
+ int omPort = -1;
+ // Parse hostname and port
+ String[] parts = authority.split(":");
+ if (parts.length > 2) {
+ throw new IllegalArgumentException(URI_EXCEPTION_TEXT);
+ }
+ omHost = parts[0];
+ if (parts.length == 2) {
+ try {
+ omPort = Integer.parseInt(parts[1]);
+ } catch (NumberFormatException e) {
+ throw new IllegalArgumentException(URI_EXCEPTION_TEXT);
+ }
+ }
+
+ try {
+ uri = new URIBuilder().setScheme(OZONE_OFS_URI_SCHEME)
+ .setHost(authority)
+ .build();
+ LOG.trace("Ozone URI for OFS initialization is " + uri);
+
+ //isolated is the default for ozonefs-lib-legacy which includes the
+ // /ozonefs.txt, otherwise the default is false. It could be overridden.
+ boolean defaultValue =
+ BasicRootedOzoneFileSystem.class.getClassLoader()
+ .getResource("ozonefs.txt") != null;
+
+ //Use string here instead of the constant as constant may not be available
+ //on the classpath of a hadoop 2.7
+ boolean isolatedClassloader =
+ conf.getBoolean("ozone.fs.isolated-classloader", defaultValue);
+
+ // adapter should be initialized in operations.
+ this.adapter = createAdapter(conf, omHost, omPort, isolatedClassloader);
+
+ try {
+ this.userName =
+ UserGroupInformation.getCurrentUser().getShortUserName();
+ } catch (IOException e) {
+ this.userName = OZONE_DEFAULT_USER;
+ }
+ this.workingDir = new Path(OZONE_USER_DIR, this.userName)
+ .makeQualified(this.uri, this.workingDir);
+ } catch (URISyntaxException ue) {
+ final String msg = "Invalid Ozone endpoint " + name;
+ LOG.error(msg, ue);
+ throw new IOException(msg, ue);
+ }
+ }
+
+ protected RootedOzoneClientAdapter createAdapter(Configuration conf,
+ String omHost, int omPort, boolean isolatedClassloader)
+ throws IOException {
+
+ if (isolatedClassloader) {
+ // TODO: Check how this code path need to be changed, for legacy Hadoop?
+ return RootedOzoneClientAdapterFactory.createAdapter();
+ } else {
+ return new BasicRootedOzoneClientAdapterImpl(omHost, omPort, conf);
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ try {
+ adapter.close();
+ } finally {
+ super.close();
+ }
+ }
+
+ @Override
+ public URI getUri() {
+ return uri;
+ }
+
+ @Override
+ public String getScheme() {
+ return OZONE_OFS_URI_SCHEME;
+ }
+
+ @Override
+ public FSDataInputStream open(Path path, int bufferSize) throws IOException {
+ incrementCounter(Statistic.INVOCATION_OPEN);
+ statistics.incrementReadOps(1);
+ LOG.trace("open() path: {}", path);
+ final String key = pathToKey(path);
+ return new FSDataInputStream(
+ new OzoneFSInputStream(adapter.readFile(key), statistics));
+ }
+
+ protected void incrementCounter(Statistic statistic) {
+ //don't do anything in this default implementation.
+ }
+
+ @Override
+ public FSDataOutputStream create(Path f, FsPermission permission,
+ boolean overwrite, int bufferSize,
+ short replication, long blockSize,
+ Progressable progress) throws IOException {
+ LOG.trace("create() path:{}", f);
+ incrementCounter(Statistic.INVOCATION_CREATE);
+ statistics.incrementWriteOps(1);
+ final String key = pathToKey(f);
+ return createOutputStream(key, overwrite, true);
+ }
+
+ @Override
+ public FSDataOutputStream createNonRecursive(Path path,
+ FsPermission permission,
+ EnumSet<CreateFlag> flags,
+ int bufferSize,
+ short replication,
+ long blockSize,
+ Progressable progress) throws IOException {
+ incrementCounter(Statistic.INVOCATION_CREATE_NON_RECURSIVE);
+ statistics.incrementWriteOps(1);
+ final String key = pathToKey(path);
+ return createOutputStream(key, flags.contains(CreateFlag.OVERWRITE), false);
+ }
+
+ private FSDataOutputStream createOutputStream(String key, boolean overwrite,
+ boolean recursive) throws IOException {
+ return new FSDataOutputStream(adapter.createFile(key, overwrite, recursive),
+ statistics);
+ }
+
+ @Override
+ public FSDataOutputStream append(Path f, int bufferSize,
+ Progressable progress) throws IOException {
+ throw new UnsupportedOperationException("append() Not implemented by the "
+ + getClass().getSimpleName() + " FileSystem implementation");
+ }
+
+ private class RenameIterator extends OzoneListingIterator {
+ private final String srcPath;
+ private final String dstPath;
+
+ RenameIterator(Path srcPath, Path dstPath)
+ throws IOException {
+ super(srcPath);
+ this.srcPath = pathToKey(srcPath);
+ this.dstPath = pathToKey(dstPath);
+ LOG.trace("rename from:{} to:{}", this.srcPath, this.dstPath);
+ }
+
+ @Override
+ boolean processKeyPath(String keyPath) throws IOException {
+ String newPath = dstPath.concat(keyPath.substring(srcPath.length()));
+ adapter.renamePath(keyPath, newPath);
+ return true;
+ }
+ }
+
+ /**
+ * Check whether the source and destination path are valid and then perform
+ * rename from source path to destination path.
+ * <p>
+ * The rename operation is performed by renaming the keys with src as prefix.
+ * For such keys the prefix is changed from src to dst.
+ *
+ * @param src source path for rename
+ * @param dst destination path for rename
+ * @return true if rename operation succeeded or
+ * if the src and dst have the same path and are of the same type
+ * @throws IOException on I/O errors or if the src/dst paths are invalid.
+ */
+ @Override
+ public boolean rename(Path src, Path dst) throws IOException {
+ incrementCounter(Statistic.INVOCATION_RENAME);
+ statistics.incrementWriteOps(1);
+ if (src.equals(dst)) {
+ return true;
+ }
+
+ LOG.trace("rename() from: {} to: {}", src, dst);
+ if (src.isRoot()) {
+ // Cannot rename root of file system
+ LOG.trace("Cannot rename the root of a filesystem");
+ return false;
+ }
+
+ // src and dst should be in the same bucket
+ OFSPath ofsSrc = new OFSPath(src);
+ OFSPath ofsDst = new OFSPath(dst);
+ if (!ofsSrc.isInSameBucketAs(ofsDst)) {
+ throw new IOException("Cannot rename a key to a different bucket");
+ }
+
+ // Cannot rename a directory to its own subdirectory
+ Path dstParent = dst.getParent();
+ while (dstParent != null && !src.equals(dstParent)) {
+ dstParent = dstParent.getParent();
+ }
+ Preconditions.checkArgument(dstParent == null,
+ "Cannot rename a directory to its own subdirectory");
+ // Check if the source exists
+ FileStatus srcStatus;
+ try {
+ srcStatus = getFileStatus(src);
+ } catch (FileNotFoundException fnfe) {
+ // source doesn't exist, return
+ return false;
+ }
+
+ // Check if the destination exists
+ FileStatus dstStatus;
+ try {
+ dstStatus = getFileStatus(dst);
+ } catch (FileNotFoundException fnde) {
+ dstStatus = null;
+ }
+
+ if (dstStatus == null) {
+ // If dst doesn't exist, check whether dst parent dir exists or not
+ // if the parent exists, the source can still be renamed to dst path
+ dstStatus = getFileStatus(dst.getParent());
+ if (!dstStatus.isDirectory()) {
+ throw new IOException(String.format(
+ "Failed to rename %s to %s, %s is a file", src, dst,
+ dst.getParent()));
+ }
+ } else {
+ // if dst exists and source and destination are same,
+ // check both the src and dst are of same type
+ if (srcStatus.getPath().equals(dstStatus.getPath())) {
+ return !srcStatus.isDirectory();
+ } else if (dstStatus.isDirectory()) {
+ // If dst is a directory, rename source as subpath of it.
+ // for example rename /source to /dst will lead to /dst/source
+ dst = new Path(dst, src.getName());
+ FileStatus[] statuses;
+ try {
+ statuses = listStatus(dst);
+ } catch (FileNotFoundException fnde) {
+ statuses = null;
+ }
+
+ if (statuses != null && statuses.length > 0) {
+ // If dst exists and not a directory not empty
+ throw new FileAlreadyExistsException(String.format(
+ "Failed to rename %s to %s, file already exists or not empty!",
+ src, dst));
+ }
+ } else {
+ // If dst is not a directory
+ throw new FileAlreadyExistsException(String.format(
+ "Failed to rename %s to %s, file already exists!", src, dst));
+ }
+ }
+
+ if (srcStatus.isDirectory()) {
+ if (dst.toString().startsWith(src.toString() + OZONE_URI_DELIMITER)) {
+ LOG.trace("Cannot rename a directory to a subdirectory of self");
+ return false;
+ }
+ }
+ RenameIterator iterator = new RenameIterator(src, dst);
+ boolean result = iterator.iterate();
+ if (result) {
+ createFakeParentDirectory(src);
+ }
+ return result;
+ }
+
+ private class DeleteIterator extends OzoneListingIterator {
+ private boolean recursive;
+
+ DeleteIterator(Path f, boolean recursive)
+ throws IOException {
+ super(f);
+ this.recursive = recursive;
+ if (getStatus().isDirectory()
+ && !this.recursive
+ && listStatus(f).length != 0) {
+ throw new PathIsNotEmptyDirectoryException(f.toString());
+ }
+ }
+
+ @Override
+ boolean processKeyPath(String keyPath) throws IOException {
+ if (keyPath.equals("")) {
+ LOG.trace("Skipping deleting root directory");
+ return true;
+ } else {
+ LOG.trace("deleting key path:" + keyPath);
+ boolean succeed = adapter.deleteObject(keyPath);
+ // if recursive delete is requested ignore the return value of
+ // deleteObject and issue deletes for other keys.
+ return recursive || succeed;
+ }
+ }
+ }
+
+ /**
+ * Deletes the children of the input dir path by iterating though the
+ * DeleteIterator.
+ *
+ * @param f directory path to be deleted
+ * @return true if successfully deletes all required keys, false otherwise
+ * @throws IOException
+ */
+ private boolean innerDelete(Path f, boolean recursive) throws IOException {
+ LOG.trace("delete() path:{} recursive:{}", f, recursive);
+ try {
+ DeleteIterator iterator = new DeleteIterator(f, recursive);
+ return iterator.iterate();
+ } catch (FileNotFoundException e) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Couldn't delete {} - does not exist", f);
+ }
+ return false;
+ }
+ }
+
+ @Override
+ public boolean delete(Path f, boolean recursive) throws IOException {
+ incrementCounter(Statistic.INVOCATION_DELETE);
+ statistics.incrementWriteOps(1);
+ LOG.debug("Delete path {} - recursive {}", f, recursive);
+ FileStatus status;
+ try {
+ status = getFileStatus(f);
+ } catch (FileNotFoundException ex) {
+ LOG.warn("delete: Path does not exist: {}", f);
+ return false;
+ }
+
+ String key = pathToKey(f);
+ boolean result;
+
+ if (status.isDirectory()) {
+ LOG.debug("delete: Path is a directory: {}", f);
+ key = addTrailingSlashIfNeeded(key);
+
+ if (key.equals("/")) {
+ LOG.warn("Cannot delete root directory.");
+ return false;
+ }
+
+ result = innerDelete(f, recursive);
+ } else {
+ LOG.debug("delete: Path is a file: {}", f);
+ result = adapter.deleteObject(key);
+ }
+
+ if (result) {
+ // If this delete operation removes all files/directories from the
+ // parent direcotry, then an empty parent directory must be created.
+ createFakeParentDirectory(f);
+ }
+
+ return result;
+ }
+
+ /**
+ * Create a fake parent directory key if it does not already exist and no
+ * other child of this parent directory exists.
+ *
+ * @param f path to the fake parent directory
+ * @throws IOException
+ */
+ private void createFakeParentDirectory(Path f) throws IOException {
+ Path parent = f.getParent();
+ if (parent != null && !parent.isRoot()) {
+ createFakeDirectoryIfNecessary(parent);
+ }
+ }
+
+ /**
+ * Create a fake directory key if it does not already exist.
+ *
+ * @param f path to the fake directory
+ * @throws IOException
+ */
+ private void createFakeDirectoryIfNecessary(Path f) throws IOException {
+ String key = pathToKey(f);
+ if (!key.isEmpty() && !o3Exists(f)) {
+ LOG.debug("Creating new fake directory at {}", f);
+ String dirKey = addTrailingSlashIfNeeded(key);
+ adapter.createDirectory(dirKey);
+ }
+ }
+
+ /**
+ * Check if a file or directory exists corresponding to given path.
+ *
+ * @param f path to file/directory.
+ * @return true if it exists, false otherwise.
+ * @throws IOException
+ */
+ private boolean o3Exists(final Path f) throws IOException {
+ Path path = makeQualified(f);
+ try {
+ getFileStatus(path);
+ return true;
+ } catch (FileNotFoundException ex) {
+ return false;
+ }
+ }
+
+ @Override
+ public FileStatus[] listStatus(Path f) throws IOException {
+ incrementCounter(Statistic.INVOCATION_LIST_STATUS);
+ statistics.incrementReadOps(1);
+ LOG.trace("listStatus() path:{}", f);
+ int numEntries = LISTING_PAGE_SIZE;
+ LinkedList<FileStatus> statuses = new LinkedList<>();
+ List<FileStatus> tmpStatusList;
+ String startPath = "";
+
+ do {
+ tmpStatusList =
+ adapter.listStatus(pathToKey(f), false, startPath,
+ numEntries, uri, workingDir, getUsername())
+ .stream()
+ .map(this::convertFileStatus)
+ .collect(Collectors.toList());
+
+ if (!tmpStatusList.isEmpty()) {
+ if (startPath.isEmpty()) {
+ statuses.addAll(tmpStatusList);
+ } else {
+ statuses.addAll(tmpStatusList.subList(1, tmpStatusList.size()));
+ }
+ startPath = pathToKey(statuses.getLast().getPath());
+ }
+ // listStatus returns entries numEntries in size if available.
+ // Any lesser number of entries indicate that the required entries have
+ // exhausted.
+ } while (tmpStatusList.size() == numEntries);
+
+ return statuses.toArray(new FileStatus[0]);
+ }
+
+ @Override
+ public void setWorkingDirectory(Path newDir) {
+ workingDir = newDir;
+ }
+
+ @Override
+ public Path getWorkingDirectory() {
+ return workingDir;
+ }
+
+ @Override
+ public Token<?> getDelegationToken(String renewer) throws IOException {
+ return adapter.getDelegationToken(renewer);
+ }
+
+ /**
+ * Get a canonical service name for this file system. If the URI is logical,
+ * the hostname part of the URI will be returned.
+ *
+ * @return a service string that uniquely identifies this file system.
+ */
+ @Override
+ public String getCanonicalServiceName() {
+ return adapter.getCanonicalServiceName();
+ }
+
+ /**
+ * Get the username of the FS.
+ *
+ * @return the short name of the user who instantiated the FS
+ */
+ public String getUsername() {
+ return userName;
+ }
+
+ /**
+ * Creates a directory. Directory is represented using a key with no value.
+ *
+ * @param path directory path to be created
+ * @return true if directory exists or created successfully.
+ * @throws IOException
+ */
+ private boolean mkdir(Path path) throws IOException {
+ return adapter.createDirectory(pathToKey(path));
+ }
+
+ @Override
+ public boolean mkdirs(Path f, FsPermission permission) throws IOException {
+ LOG.trace("mkdir() path:{} ", f);
+ String key = pathToKey(f);
+ if (isEmpty(key)) {
+ return false;
+ }
+ return mkdir(f);
+ }
+
+ @Override
+ public FileStatus getFileStatus(Path f) throws IOException {
+ incrementCounter(Statistic.INVOCATION_GET_FILE_STATUS);
+ statistics.incrementReadOps(1);
+ LOG.trace("getFileStatus() path:{}", f);
+ Path qualifiedPath = f.makeQualified(uri, workingDir);
+ String key = pathToKey(qualifiedPath);
+ FileStatus fileStatus = null;
+ try {
+ fileStatus = convertFileStatus(
+ adapter.getFileStatus(key, uri, qualifiedPath, getUsername()));
+ } catch (OMException ex) {
+ if (ex.getResult().equals(OMException.ResultCodes.KEY_NOT_FOUND)) {
+ throw new FileNotFoundException("File not found. path:" + f);
+ }
+ }
+ return fileStatus;
+ }
+
+ @Override
+ public BlockLocation[] getFileBlockLocations(FileStatus fileStatus,
+ long start, long len)
+ throws IOException {
+ if (fileStatus instanceof LocatedFileStatus) {
+ return ((LocatedFileStatus) fileStatus).getBlockLocations();
+ } else {
+ return super.getFileBlockLocations(fileStatus, start, len);
+ }
+ }
+
+ /**
+ * Turn a path (relative or otherwise) into an Ozone key.
+ *
+ * @param path the path of the file.
+ * @return the key of the object that represents the file.
+ */
+ public String pathToKey(Path path) {
+ Objects.requireNonNull(path, "Path can't be null!");
+ if (!path.isAbsolute()) {
+ path = new Path(workingDir, path);
+ }
+ // removing leading '/' char
+ String key = path.toUri().getPath().substring(1);
+ LOG.trace("path for key: {} is: {}", key, path);
+ return key;
+ }
+
+ /**
+ * Add trailing delimiter to path if it is already not present.
+ *
+ * @param key the ozone Key which needs to be appended
+ * @return delimiter appended key
+ */
+ private String addTrailingSlashIfNeeded(String key) {
+ if (!isEmpty(key) && !key.endsWith(OZONE_URI_DELIMITER)) {
+ return key + OZONE_URI_DELIMITER;
+ } else {
+ return key;
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "RootedOzoneFileSystem{URI=" + uri + ", "
+ + "workingDir=" + workingDir + ", "
+ + "userName=" + userName + ", "
+ + "statistics=" + statistics
+ + "}";
+ }
+
+ /**
+ * This class provides an interface to iterate through all the keys in the
+ * bucket prefixed with the input path key and process them.
+ * <p>
+ * Each implementing class should define how the keys should be processed
+ * through the processKeyPath() function.
+ */
+ private abstract class OzoneListingIterator {
+ private final Path path;
+ private final FileStatus status;
+ private String pathKey;
+ private Iterator<BasicKeyInfo> keyIterator;
+
+ OzoneListingIterator(Path path)
+ throws IOException {
+ this.path = path;
+ this.status = getFileStatus(path);
+ this.pathKey = pathToKey(path);
+ if (status.isDirectory()) {
+ this.pathKey = addTrailingSlashIfNeeded(pathKey);
+ }
+ keyIterator = adapter.listKeys(pathKey);
+ }
+
+ /**
+ * The output of processKey determines if further iteration through the
+ * keys should be done or not.
+ *
+ * @return true if we should continue iteration of keys, false otherwise.
+ * @throws IOException
+ */
+ abstract boolean processKeyPath(String keyPath) throws IOException;
+
+ /**
+ * Iterates through all the keys prefixed with the input path's key and
+ * processes the key though processKey().
+ * If for any key, the processKey() returns false, then the iteration is
+ * stopped and returned with false indicating that all the keys could not
+ * be processed successfully.
+ *
+ * @return true if all keys are processed successfully, false otherwise.
+ * @throws IOException
+ */
+ boolean iterate() throws IOException {
+ LOG.trace("Iterating path: {}", path);
+ if (status.isDirectory()) {
+ LOG.trace("Iterating directory: {}", pathKey);
+ OFSPath ofsPath = new OFSPath(pathKey);
Review comment:
Yes since I intend to convert key to full path before feeding it to `processKeyPath()`. I could change it back but again, my idea of "using only full path outside of AdapterImpl" will be broken. <- this could probably be solved with refactoring but I don't think that is a priority at the moment.
Added comment and TODO in 10fe834c08cf72911b6d0409fb1138ff7d23294e.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] xiaoyuyao commented on a change in pull request
#415: HDDS-2840. Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
xiaoyuyao commented on a change in pull request #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#discussion_r372029483
##########
File path: hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
##########
@@ -0,0 +1,694 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.ozone;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.key.KeyProvider;
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdds.client.ReplicationFactor;
+import org.apache.hadoop.hdds.client.ReplicationType;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ozone.OmUtils;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.client.OzoneKey;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
+import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
+import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenRenewer;
+
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .BUCKET_ALREADY_EXISTS;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .VOLUME_ALREADY_EXISTS;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .VOLUME_NOT_FOUND;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .BUCKET_NOT_FOUND;
+
+/**
+ * Basic Implementation of the OzoneFileSystem calls.
+ * <p>
+ * This is the minimal version which doesn't include any statistics.
+ * <p>
+ * For full featured version use OzoneClientAdapterImpl.
+ */
+public class BasicRootedOzoneClientAdapterImpl
+ implements RootedOzoneClientAdapter {
+
+ static final Logger LOG =
+ LoggerFactory.getLogger(BasicRootedOzoneClientAdapterImpl.class);
+
+ private OzoneClient ozoneClient;
+ private ClientProtocol proxy;
+ private ObjectStore objectStore;
+ private ReplicationType replicationType;
+ private ReplicationFactor replicationFactor;
+ private boolean securityEnabled;
+ private int configuredDnPort;
+
+ /**
+ * Create new OzoneClientAdapter implementation.
+ *
+ * @throws IOException In case of a problem.
+ */
+ public BasicRootedOzoneClientAdapterImpl() throws IOException {
+ this(createConf());
+ }
+
+ private static OzoneConfiguration createConf() {
+ ClassLoader contextClassLoader =
+ Thread.currentThread().getContextClassLoader();
+ Thread.currentThread().setContextClassLoader(null);
+ try {
+ return new OzoneConfiguration();
+ } finally {
+ Thread.currentThread().setContextClassLoader(contextClassLoader);
+ }
+ }
+
+ public BasicRootedOzoneClientAdapterImpl(OzoneConfiguration conf)
+ throws IOException {
+ this(null, -1, conf);
+ }
+
+ public BasicRootedOzoneClientAdapterImpl(String omHost, int omPort,
+ Configuration hadoopConf) throws IOException {
+
+ ClassLoader contextClassLoader =
+ Thread.currentThread().getContextClassLoader();
+ Thread.currentThread().setContextClassLoader(null);
+
+ try {
+ OzoneConfiguration conf = OzoneConfiguration.of(hadoopConf);
+
+ if (omHost == null && OmUtils.isServiceIdsDefined(conf)) {
+ // When the host name or service id isn't given
+ // but ozone.om.service.ids is defined, declare failure.
+
+ // This is a safety precaution that prevents the client from
+ // accidentally failing over to an unintended OM.
+ throw new IllegalArgumentException("Service ID or host name must not"
+ + " be omitted when ozone.om.service.ids is defined.");
+ }
+
+ if (omPort != -1) {
+ // When the port number is specified, perform the following check
+ if (OmUtils.isOmHAServiceId(conf, omHost)) {
+ // If omHost is a service id, it shouldn't use a port
+ throw new IllegalArgumentException("Port " + omPort +
+ " specified in URI but host '" + omHost + "' is a "
+ + "logical (HA) OzoneManager and does not use port information.");
+ }
+ } else {
+ // When port number is not specified, read it from config
+ omPort = OmUtils.getOmRpcPort(conf);
+ }
+
+ SecurityConfig secConfig = new SecurityConfig(conf);
+
+ if (secConfig.isSecurityEnabled()) {
+ this.securityEnabled = true;
+ }
+
+ String replicationTypeConf =
+ conf.get(OzoneConfigKeys.OZONE_REPLICATION_TYPE,
+ OzoneConfigKeys.OZONE_REPLICATION_TYPE_DEFAULT);
+
+ int replicationCountConf = conf.getInt(OzoneConfigKeys.OZONE_REPLICATION,
+ OzoneConfigKeys.OZONE_REPLICATION_DEFAULT);
+
+ if (OmUtils.isOmHAServiceId(conf, omHost)) {
+ // omHost is listed as one of the service ids in the config,
+ // thus we should treat omHost as omServiceId
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(omHost, conf);
+ } else if (StringUtils.isNotEmpty(omHost) && omPort != -1) {
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(omHost, omPort, conf);
+ } else {
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(conf);
+ }
+ objectStore = ozoneClient.getObjectStore();
+ proxy = objectStore.getClientProxy();
+ this.replicationType = ReplicationType.valueOf(replicationTypeConf);
+ this.replicationFactor = ReplicationFactor.valueOf(replicationCountConf);
+ this.configuredDnPort = conf.getInt(
+ OzoneConfigKeys.DFS_CONTAINER_IPC_PORT,
+ OzoneConfigKeys.DFS_CONTAINER_IPC_PORT_DEFAULT);
+ } finally {
+ Thread.currentThread().setContextClassLoader(contextClassLoader);
+ }
+ }
+
+ OzoneBucket getBucket(OFSPath ofsPath, boolean createIfNotExist)
+ throws IOException {
+
+ return getBucket(ofsPath.getVolumeName(), ofsPath.getBucketName(),
+ createIfNotExist);
+ }
+
+ /**
+ * Get OzoneBucket object to operate in.
+ * Optionally create volume and bucket if not found.
+ *
+ * @param createIfNotExist Set this to true if the caller is a write operation
+ * in order to create the volume and bucket.
+ * @throws IOException Exceptions other than OMException with result code
+ * VOLUME_NOT_FOUND or BUCKET_NOT_FOUND.
+ */
+ private OzoneBucket getBucket(String volumeStr, String bucketStr,
+ boolean createIfNotExist) throws IOException {
+
+ OzoneBucket bucket;
+ try {
+ bucket = proxy.getBucketDetails(volumeStr, bucketStr);
+ } catch (OMException ex) {
+ if (createIfNotExist) {
+ // Note: getBucketDetails always throws BUCKET_NOT_FOUND, even if
+ // the volume doesn't exist.
+ if (ex.getResult().equals(BUCKET_NOT_FOUND)) {
+ OzoneVolume volume;
+ try {
+ volume = proxy.getVolumeDetails(volumeStr);
+ } catch (OMException getVolEx) {
+ if (getVolEx.getResult().equals(VOLUME_NOT_FOUND)) {
+ // Volume doesn't exist. Create it
+ try {
+ objectStore.createVolume(volumeStr);
+ } catch (OMException newVolEx) {
+ // Ignore the case where another client created the volume
+ if (!newVolEx.getResult().equals(VOLUME_ALREADY_EXISTS)) {
+ throw newVolEx;
+ }
+ }
+ } else {
+ throw getVolEx;
+ }
+ // Try get volume again
+ volume = proxy.getVolumeDetails(volumeStr);
+ }
+ // Create the bucket
+ try {
+ volume.createBucket(bucketStr);
+ } catch (OMException newBucEx) {
+ // Ignore the case where another client created the bucket
+ if (!newBucEx.getResult().equals(BUCKET_ALREADY_EXISTS)) {
+ throw newBucEx;
+ }
+ }
+ }
+ // Try get bucket again
+ bucket = proxy.getBucketDetails(volumeStr, bucketStr);
+ } else {
+ throw ex;
+ }
+ }
+
+ return bucket;
+ }
+
+ @Override
+ public void close() throws IOException {
+ ozoneClient.close();
+ }
+
+ @Override
+ public InputStream readFile(String pathStr) throws IOException {
+ incrementCounter(Statistic.OBJECTS_READ);
+ OFSPath ofsPath = new OFSPath(pathStr);
+ String key = ofsPath.getKeyName();
+ try {
+ OzoneBucket bucket = getBucket(ofsPath, false);
+ return bucket.readFile(key).getInputStream();
+ } catch (OMException ex) {
+ if (ex.getResult() == OMException.ResultCodes.FILE_NOT_FOUND
+ || ex.getResult() == OMException.ResultCodes.NOT_A_FILE) {
+ throw new FileNotFoundException(
+ ex.getResult().name() + ": " + ex.getMessage());
+ } else {
+ throw ex;
+ }
+ }
+ }
+
+ protected void incrementCounter(Statistic objectsRead) {
+ //noop: Use OzoneClientAdapterImpl which supports statistics.
+ }
+
+ @Override
+ public OzoneFSOutputStream createFile(String pathStr, boolean overWrite,
+ boolean recursive) throws IOException {
+ incrementCounter(Statistic.OBJECTS_CREATED);
+ OFSPath ofsPath = new OFSPath(pathStr);
+ String key = ofsPath.getKeyName();
+ try {
+ OzoneBucket bucket = getBucket(ofsPath, false);
+ OzoneOutputStream ozoneOutputStream = bucket.createFile(
+ key, 0, replicationType, replicationFactor, overWrite, recursive);
+ return new OzoneFSOutputStream(ozoneOutputStream.getOutputStream());
+ } catch (OMException ex) {
+ if (ex.getResult() == OMException.ResultCodes.FILE_ALREADY_EXISTS
+ || ex.getResult() == OMException.ResultCodes.NOT_A_FILE) {
+ throw new FileAlreadyExistsException(
+ ex.getResult().name() + ": " + ex.getMessage());
+ } else {
+ throw ex;
+ }
+ }
+ }
+
+ @Override
+ public void renamePath(String path, String newPath) throws IOException {
+ incrementCounter(Statistic.OBJECTS_RENAMED);
+ OFSPath ofsPath = new OFSPath(path);
+ OFSPath ofsNewPath = new OFSPath(newPath);
+
+ // Check path and newPathName are in the same volume and same bucket.
+ // This should have been checked in BasicRootedOzoneFileSystem#rename
+ // already via regular call path unless bypassed.
+ if (!ofsPath.isInSameBucketAs(ofsNewPath)) {
+ throw new IOException("Cannot rename a key to a different bucket");
+ }
+
+ OzoneBucket bucket = getBucket(ofsPath, false);
+ String key = ofsPath.getKeyName();
+ String newKey = ofsNewPath.getKeyName();
+ bucket.renameKey(key, newKey);
+ }
+
+ /**
+ * Package-private helper function to reduce calls to getBucket().
+ * @param bucket Bucket to operate in.
+ * @param path Existing key path.
+ * @param newPath New key path.
+ * @throws IOException IOException from bucket.renameKey().
+ */
+ void renamePath(OzoneBucket bucket, String path, String newPath)
+ throws IOException {
+ incrementCounter(Statistic.OBJECTS_RENAMED);
+ OFSPath ofsPath = new OFSPath(path);
+ OFSPath ofsNewPath = new OFSPath(newPath);
+ // No same-bucket policy check here since this call path is controlled
+ String key = ofsPath.getKeyName();
+ String newKey = ofsNewPath.getKeyName();
+ bucket.renameKey(key, newKey);
+ }
+
+ /**
+ * Helper method to create an directory specified by key name in bucket.
+ *
+ * @param pathStr path to be created as directory
+ * @return true if the key is created, false otherwise
+ */
+ @Override
+ public boolean createDirectory(String pathStr) throws IOException {
+ LOG.trace("creating dir for path: {}", pathStr);
+ incrementCounter(Statistic.OBJECTS_CREATED);
+ OFSPath ofsPath = new OFSPath(pathStr);
+ // Volume name unspecified, return failure
+ if (ofsPath.getVolumeName().length() == 0) {
+ return false;
+ }
+ // Handle where only volume is specified in pathStr
+ if (ofsPath.getBucketName().length() == 0) {
+ objectStore.createVolume(ofsPath.getVolumeName());
+ return true;
+ }
+ String keyStr = ofsPath.getKeyName();
+ try {
+ OzoneBucket bucket = getBucket(ofsPath, true);
+ // if keyStr is empty, it indicates that only volume or volume+bucket is
+ // given in pathStr, so getBucket() above should've handled the creation
+ // of volume/bucket already.
+ if (keyStr != null && keyStr.length() > 0) {
+ bucket.createDirectory(keyStr);
+ }
+ } catch (OMException e) {
+ if (e.getResult() == OMException.ResultCodes.FILE_ALREADY_EXISTS) {
+ throw new FileAlreadyExistsException(e.getMessage());
+ }
+ throw e;
+ }
+ return true;
+ }
+
+ /**
+ * Helper method to delete an object specified by key name in bucket.
+ *
+ * @param path path to a key to be deleted
+ * @return true if the key is deleted, false otherwise
+ */
+ @Override
+ public boolean deleteObject(String path) {
+ LOG.trace("issuing delete for path to key: {}", path);
+ OFSPath ofsPath = new OFSPath(path);
+ String keyName = ofsPath.getKeyName();
+ try {
+ OzoneBucket bucket = getBucket(ofsPath, false);
+ incrementCounter(Statistic.OBJECTS_DELETED);
+ bucket.deleteKey(keyName);
+ return true;
+ } catch (IOException ioe) {
+ LOG.error("delete key failed " + ioe.getMessage());
+ return false;
+ }
+ }
+
+ /**
+ * Package-private helper function to reduce calls to getBucket().
+ * @param bucket Bucket to operate in.
+ * @param path Path to delete.
+ * @return true if operation succeeded, false upon IOException.
+ */
+ boolean deleteObject(OzoneBucket bucket, String path) {
+ LOG.trace("issuing delete for path to key: {}", path);
+ OFSPath ofsPath = new OFSPath(path);
+ String keyName = ofsPath.getKeyName();
+ try {
+ incrementCounter(Statistic.OBJECTS_DELETED);
+ bucket.deleteKey(keyName);
+ return true;
+ } catch (IOException ioe) {
+ LOG.error("delete key failed " + ioe.getMessage());
+ return false;
+ }
+ }
+
+ public FileStatusAdapter getFileStatus(String path, URI uri,
+ Path qualifiedPath, String userName)
+ throws IOException {
+ OFSPath ofsPath = new OFSPath(path);
+ String key = ofsPath.getKeyName();
+ try {
+ OzoneBucket bucket = getBucket(ofsPath, false);
+ incrementCounter(Statistic.OBJECTS_QUERY);
Review comment:
should we move the counter incr before getBucket so that we can accurately count the ops even though getBucket may fail?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] smengcl commented on a change in pull request #415:
HDDS-2840. Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
smengcl commented on a change in pull request #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#discussion_r370417528
##########
File path: hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
##########
@@ -0,0 +1,395 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.TestDataUtil;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClientException;
+import org.apache.hadoop.ozone.client.OzoneKeyDetails;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.Set;
+import java.util.TreeSet;
+
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Ozone file system tests that are not covered by contract tests.
+ */
+public class TestRootedOzoneFileSystem {
+
+ @Rule
+ public Timeout globalTimeout = new Timeout(300_000);
+
+ private static MiniOzoneCluster cluster = null;
+
+ private static FileSystem fs;
+ private static RootedOzoneFileSystem ofs;
+
+ private static ObjectStore objectStore;
+
+ private String volumeName;
+ private String bucketName;
+
+ private String rootPath;
+
+ @Before
+ public void init() throws Exception {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ cluster = MiniOzoneCluster.newBuilder(conf)
+ .setNumDatanodes(3)
+ .build();
+ cluster.waitForClusterToBeReady();
+ objectStore = cluster.getClient().getObjectStore();
+
+ // create a volume and a bucket to be used by OFileSystem
+ OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(cluster);
+ volumeName = bucket.getVolumeName();
+ bucketName = bucket.getName();
+
+ // For now:
Review comment:
I forgot to clean up this line of comment. the `rootPath` below should be good.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] hanishakoneru commented on a change in pull request
#415: HDDS-2840. Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
hanishakoneru commented on a change in pull request #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#discussion_r369302919
##########
File path: hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
##########
@@ -0,0 +1,395 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.TestDataUtil;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClientException;
+import org.apache.hadoop.ozone.client.OzoneKeyDetails;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.Set;
+import java.util.TreeSet;
+
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Ozone file system tests that are not covered by contract tests.
+ */
+public class TestRootedOzoneFileSystem {
+
+ @Rule
+ public Timeout globalTimeout = new Timeout(300_000);
+
+ private static MiniOzoneCluster cluster = null;
+
+ private static FileSystem fs;
+ private static RootedOzoneFileSystem ofs;
+
+ private static ObjectStore objectStore;
+
+ private String volumeName;
+ private String bucketName;
+
+ private String rootPath;
+
+ @Before
+ public void init() throws Exception {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ cluster = MiniOzoneCluster.newBuilder(conf)
+ .setNumDatanodes(3)
+ .build();
+ cluster.waitForClusterToBeReady();
+ objectStore = cluster.getClient().getObjectStore();
+
+ // create a volume and a bucket to be used by OFileSystem
+ OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(cluster);
+ volumeName = bucket.getVolumeName();
+ bucketName = bucket.getName();
+
+ // For now:
+ rootPath = String.format("%s://%s/", OzoneConsts.OZONE_OFS_URI_SCHEME,
+ conf.get(OZONE_OM_ADDRESS_KEY));
+
+ // Set the fs.defaultFS and start the filesystem
+ conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, rootPath);
+ // TODO: FileSystem#loadFileSystems is not loading ofs:// class by default
+ // hence this workaround. Might need to add some config in hadoop source.
+ conf.set("fs.ofs.impl", "org.apache.hadoop.fs.ozone.RootedOzoneFileSystem");
+ fs = FileSystem.get(conf);
+ ofs = (RootedOzoneFileSystem) fs;
+ }
+
+ @After
+ public void teardown() throws IOException {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ IOUtils.closeQuietly(fs);
+ }
+
+ @Test
+ public void testOzoneFsServiceLoader() throws IOException {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ // TODO: Address this properly.
+ conf.set("fs.ofs.impl", "org.apache.hadoop.fs.ozone.RootedOzoneFileSystem");
+ assertEquals(
+ FileSystem.getFileSystemClass(OzoneConsts.OZONE_OFS_URI_SCHEME,
+ conf), RootedOzoneFileSystem.class);
+ }
+
+ @Test
+ public void testCreateDoesNotAddParentDirKeys() throws Exception {
+ Path rootBucket = new Path("/" + volumeName + "/" + bucketName);
Review comment:
rootBucket is used globally in this Test suit. Can we make it a global variable?
Also, rootBucket gives the impression that it's the bucket at the root of the FS. Could we rename it to testBucket or volumeAndBucketPath or something else?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] smengcl edited a comment on issue #415: HDDS-2840.
Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
smengcl edited a comment on issue #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#issuecomment-573229933
Status update:
1. This PR is ready for review.
2. This diff should help for the review, the Base is the commit after class copying (except RootedOzoneClientAdapterFactory): https://github.com/smengcl/hadoop-ozone/compare/0bee28acadfb2c358c6f008173e7eca6ed7fa23f...smengcl:HDDS-2840
3. I have also applied HDDS-2188 to affected OFS classes in order for code to compile.
4. Expect all operations INSIDE the same bucket to work, just like the current o3fs.
5. Expect `mkdir -p` to create volume and bucket if they don't exist. For a simple test under shell with docker-compose see commit message of ba3a21ecc2e27690e8b823bd72fa8d8976ffdb43.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] xiaoyuyao commented on a change in pull request
#415: HDDS-2840. Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
xiaoyuyao commented on a change in pull request #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#discussion_r372015527
##########
File path: hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
##########
@@ -0,0 +1,694 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.ozone;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.key.KeyProvider;
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdds.client.ReplicationFactor;
+import org.apache.hadoop.hdds.client.ReplicationType;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ozone.OmUtils;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.client.OzoneKey;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
+import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
+import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenRenewer;
+
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .BUCKET_ALREADY_EXISTS;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .VOLUME_ALREADY_EXISTS;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .VOLUME_NOT_FOUND;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .BUCKET_NOT_FOUND;
+
+/**
+ * Basic Implementation of the OzoneFileSystem calls.
+ * <p>
+ * This is the minimal version which doesn't include any statistics.
+ * <p>
+ * For full featured version use OzoneClientAdapterImpl.
+ */
+public class BasicRootedOzoneClientAdapterImpl
+ implements RootedOzoneClientAdapter {
+
+ static final Logger LOG =
+ LoggerFactory.getLogger(BasicRootedOzoneClientAdapterImpl.class);
+
+ private OzoneClient ozoneClient;
+ private ClientProtocol proxy;
+ private ObjectStore objectStore;
+ private ReplicationType replicationType;
+ private ReplicationFactor replicationFactor;
+ private boolean securityEnabled;
+ private int configuredDnPort;
+
+ /**
+ * Create new OzoneClientAdapter implementation.
+ *
+ * @throws IOException In case of a problem.
+ */
+ public BasicRootedOzoneClientAdapterImpl() throws IOException {
+ this(createConf());
+ }
+
+ private static OzoneConfiguration createConf() {
+ ClassLoader contextClassLoader =
+ Thread.currentThread().getContextClassLoader();
+ Thread.currentThread().setContextClassLoader(null);
+ try {
+ return new OzoneConfiguration();
+ } finally {
+ Thread.currentThread().setContextClassLoader(contextClassLoader);
+ }
+ }
+
+ public BasicRootedOzoneClientAdapterImpl(OzoneConfiguration conf)
+ throws IOException {
+ this(null, -1, conf);
+ }
+
+ public BasicRootedOzoneClientAdapterImpl(String omHost, int omPort,
+ Configuration hadoopConf) throws IOException {
+
+ ClassLoader contextClassLoader =
+ Thread.currentThread().getContextClassLoader();
+ Thread.currentThread().setContextClassLoader(null);
+
+ try {
+ OzoneConfiguration conf = OzoneConfiguration.of(hadoopConf);
+
+ if (omHost == null && OmUtils.isServiceIdsDefined(conf)) {
+ // When the host name or service id isn't given
+ // but ozone.om.service.ids is defined, declare failure.
+
+ // This is a safety precaution that prevents the client from
+ // accidentally failing over to an unintended OM.
+ throw new IllegalArgumentException("Service ID or host name must not"
+ + " be omitted when ozone.om.service.ids is defined.");
+ }
+
+ if (omPort != -1) {
+ // When the port number is specified, perform the following check
+ if (OmUtils.isOmHAServiceId(conf, omHost)) {
+ // If omHost is a service id, it shouldn't use a port
+ throw new IllegalArgumentException("Port " + omPort +
+ " specified in URI but host '" + omHost + "' is a "
+ + "logical (HA) OzoneManager and does not use port information.");
+ }
+ } else {
+ // When port number is not specified, read it from config
+ omPort = OmUtils.getOmRpcPort(conf);
+ }
+
+ SecurityConfig secConfig = new SecurityConfig(conf);
+
+ if (secConfig.isSecurityEnabled()) {
+ this.securityEnabled = true;
+ }
+
+ String replicationTypeConf =
+ conf.get(OzoneConfigKeys.OZONE_REPLICATION_TYPE,
+ OzoneConfigKeys.OZONE_REPLICATION_TYPE_DEFAULT);
+
+ int replicationCountConf = conf.getInt(OzoneConfigKeys.OZONE_REPLICATION,
+ OzoneConfigKeys.OZONE_REPLICATION_DEFAULT);
+
+ if (OmUtils.isOmHAServiceId(conf, omHost)) {
+ // omHost is listed as one of the service ids in the config,
+ // thus we should treat omHost as omServiceId
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(omHost, conf);
+ } else if (StringUtils.isNotEmpty(omHost) && omPort != -1) {
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(omHost, omPort, conf);
+ } else {
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(conf);
+ }
+ objectStore = ozoneClient.getObjectStore();
+ proxy = objectStore.getClientProxy();
+ this.replicationType = ReplicationType.valueOf(replicationTypeConf);
+ this.replicationFactor = ReplicationFactor.valueOf(replicationCountConf);
+ this.configuredDnPort = conf.getInt(
+ OzoneConfigKeys.DFS_CONTAINER_IPC_PORT,
+ OzoneConfigKeys.DFS_CONTAINER_IPC_PORT_DEFAULT);
+ } finally {
+ Thread.currentThread().setContextClassLoader(contextClassLoader);
+ }
+ }
+
+ OzoneBucket getBucket(OFSPath ofsPath, boolean createIfNotExist)
+ throws IOException {
+
+ return getBucket(ofsPath.getVolumeName(), ofsPath.getBucketName(),
+ createIfNotExist);
+ }
+
+ /**
+ * Get OzoneBucket object to operate in.
+ * Optionally create volume and bucket if not found.
+ *
+ * @param createIfNotExist Set this to true if the caller is a write operation
+ * in order to create the volume and bucket.
+ * @throws IOException Exceptions other than OMException with result code
+ * VOLUME_NOT_FOUND or BUCKET_NOT_FOUND.
+ */
+ private OzoneBucket getBucket(String volumeStr, String bucketStr,
+ boolean createIfNotExist) throws IOException {
+
+ OzoneBucket bucket;
+ try {
+ bucket = proxy.getBucketDetails(volumeStr, bucketStr);
+ } catch (OMException ex) {
+ if (createIfNotExist) {
+ // Note: getBucketDetails always throws BUCKET_NOT_FOUND, even if
+ // the volume doesn't exist.
+ if (ex.getResult().equals(BUCKET_NOT_FOUND)) {
+ OzoneVolume volume;
+ try {
+ volume = proxy.getVolumeDetails(volumeStr);
Review comment:
should we do getVolumeDetails/create first and if that fails, there is no need to create bucket.
As createVolume is an admin only operation, I would expect most of the client driven operation will fail here.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] hanishakoneru commented on a change in pull request
#415: HDDS-2840. Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
hanishakoneru commented on a change in pull request #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#discussion_r369795390
##########
File path: hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
##########
@@ -0,0 +1,805 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.ozone;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.CreateFlag;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.Progressable;
+import org.apache.http.client.utils.URIBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.EnumSet;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+import static org.apache.hadoop.fs.ozone.Constants.LISTING_PAGE_SIZE;
+import static org.apache.hadoop.fs.ozone.Constants.OZONE_DEFAULT_USER;
+import static org.apache.hadoop.fs.ozone.Constants.OZONE_USER_DIR;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_OFS_URI_SCHEME;
+
+/**
+ * The minimal Ozone Filesystem implementation.
+ * <p>
+ * This is a basic version which doesn't extend
+ * KeyProviderTokenIssuer and doesn't include statistics. It can be used
+ * from older hadoop version. For newer hadoop version use the full featured
+ * OFileSystem.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class BasicRootedOzoneFileSystem extends FileSystem {
+ static final Logger LOG =
+ LoggerFactory.getLogger(BasicRootedOzoneFileSystem.class);
+
+ /**
+ * The Ozone client for connecting to Ozone server.
+ */
+
+ private URI uri;
+ private String userName;
+ private Path workingDir;
+// private String gOmHost;
+// private int gOmPort;
+// private Configuration gConf;
+// private boolean gIsolatedClassloader;
+
+ private RootedOzoneClientAdapter adapter;
+// private String adapterPath;
+
+ private static final String URI_EXCEPTION_TEXT =
+ "URL should be one of the following formats: " +
+ "ofs://om-service-id/ OR " +
+ "ofs://om-host.example.com/ OR " +
+ "ofs://om-host.example.com:5678/";
+
+ @Override
+ public void initialize(URI name, Configuration conf) throws IOException {
+ super.initialize(name, conf);
+ setConf(conf);
+ Objects.requireNonNull(name.getScheme(), "No scheme provided in " + name);
+ Preconditions.checkArgument(getScheme().equals(name.getScheme()),
+ "Invalid scheme provided in " + name);
+
+ String authority = name.getAuthority();
+ if (authority == null) {
+ // authority is null when fs.defaultFS is not a qualified ofs URI and
+ // ofs:/// is passed to the client. matcher will NPE if authority is null
+ throw new IllegalArgumentException(URI_EXCEPTION_TEXT);
+ }
+
+ String omHost;
+ int omPort = -1;
+ // Parse hostname and port
+ String[] parts = authority.split(":");
+ if (parts.length > 2) {
+ throw new IllegalArgumentException(URI_EXCEPTION_TEXT);
+ }
+ omHost = parts[0];
+ if (parts.length == 2) {
+ try {
+ omPort = Integer.parseInt(parts[1]);
+ } catch (NumberFormatException e) {
+ throw new IllegalArgumentException(URI_EXCEPTION_TEXT);
+ }
+ }
+
+ try {
+ uri = new URIBuilder().setScheme(OZONE_OFS_URI_SCHEME)
+ .setHost(authority)
+ .build();
+ LOG.trace("Ozone URI for OFS initialization is " + uri);
+
+ //isolated is the default for ozonefs-lib-legacy which includes the
+ // /ozonefs.txt, otherwise the default is false. It could be overridden.
+ boolean defaultValue =
+ BasicRootedOzoneFileSystem.class.getClassLoader()
+ .getResource("ozonefs.txt") != null;
+
+ //Use string here instead of the constant as constant may not be available
+ //on the classpath of a hadoop 2.7
+ boolean isolatedClassloader =
+ conf.getBoolean("ozone.fs.isolated-classloader", defaultValue);
+
+ // adapter should be initialized in operations.
+ this.adapter = createAdapter(conf, omHost, omPort, isolatedClassloader);
+
+ try {
+ this.userName =
+ UserGroupInformation.getCurrentUser().getShortUserName();
+ } catch (IOException e) {
+ this.userName = OZONE_DEFAULT_USER;
+ }
+ this.workingDir = new Path(OZONE_USER_DIR, this.userName)
+ .makeQualified(this.uri, this.workingDir);
+ } catch (URISyntaxException ue) {
+ final String msg = "Invalid Ozone endpoint " + name;
+ LOG.error(msg, ue);
+ throw new IOException(msg, ue);
+ }
+ }
+
+ protected RootedOzoneClientAdapter createAdapter(Configuration conf,
+ String omHost, int omPort, boolean isolatedClassloader)
+ throws IOException {
+
+ if (isolatedClassloader) {
+ // TODO: Check how this code path need to be changed, for legacy Hadoop?
+ return RootedOzoneClientAdapterFactory.createAdapter();
+ } else {
+ return new BasicRootedOzoneClientAdapterImpl(omHost, omPort, conf);
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ try {
+ adapter.close();
+ } finally {
+ super.close();
+ }
+ }
+
+ @Override
+ public URI getUri() {
+ return uri;
+ }
+
+ @Override
+ public String getScheme() {
+ return OZONE_OFS_URI_SCHEME;
+ }
+
+ @Override
+ public FSDataInputStream open(Path path, int bufferSize) throws IOException {
+ incrementCounter(Statistic.INVOCATION_OPEN);
+ statistics.incrementReadOps(1);
+ LOG.trace("open() path: {}", path);
+ final String key = pathToKey(path);
+ return new FSDataInputStream(
+ new OzoneFSInputStream(adapter.readFile(key), statistics));
+ }
+
+ protected void incrementCounter(Statistic statistic) {
+ //don't do anything in this default implementation.
+ }
+
+ @Override
+ public FSDataOutputStream create(Path f, FsPermission permission,
+ boolean overwrite, int bufferSize,
+ short replication, long blockSize,
+ Progressable progress) throws IOException {
+ LOG.trace("create() path:{}", f);
+ incrementCounter(Statistic.INVOCATION_CREATE);
+ statistics.incrementWriteOps(1);
+ final String key = pathToKey(f);
+ return createOutputStream(key, overwrite, true);
+ }
+
+ @Override
+ public FSDataOutputStream createNonRecursive(Path path,
+ FsPermission permission,
+ EnumSet<CreateFlag> flags,
+ int bufferSize,
+ short replication,
+ long blockSize,
+ Progressable progress) throws IOException {
+ incrementCounter(Statistic.INVOCATION_CREATE_NON_RECURSIVE);
+ statistics.incrementWriteOps(1);
+ final String key = pathToKey(path);
+ return createOutputStream(key, flags.contains(CreateFlag.OVERWRITE), false);
+ }
+
+ private FSDataOutputStream createOutputStream(String key, boolean overwrite,
+ boolean recursive) throws IOException {
+ return new FSDataOutputStream(adapter.createFile(key, overwrite, recursive),
+ statistics);
+ }
+
+ @Override
+ public FSDataOutputStream append(Path f, int bufferSize,
+ Progressable progress) throws IOException {
+ throw new UnsupportedOperationException("append() Not implemented by the "
+ + getClass().getSimpleName() + " FileSystem implementation");
+ }
+
+ private class RenameIterator extends OzoneListingIterator {
+ private final String srcPath;
+ private final String dstPath;
+
+ RenameIterator(Path srcPath, Path dstPath)
+ throws IOException {
+ super(srcPath);
+ this.srcPath = pathToKey(srcPath);
+ this.dstPath = pathToKey(dstPath);
+ LOG.trace("rename from:{} to:{}", this.srcPath, this.dstPath);
+ }
+
+ @Override
+ boolean processKeyPath(String keyPath) throws IOException {
+ String newPath = dstPath.concat(keyPath.substring(srcPath.length()));
+ adapter.renamePath(keyPath, newPath);
+ return true;
+ }
+ }
+
+ /**
+ * Check whether the source and destination path are valid and then perform
+ * rename from source path to destination path.
+ * <p>
+ * The rename operation is performed by renaming the keys with src as prefix.
+ * For such keys the prefix is changed from src to dst.
+ *
+ * @param src source path for rename
+ * @param dst destination path for rename
+ * @return true if rename operation succeeded or
+ * if the src and dst have the same path and are of the same type
+ * @throws IOException on I/O errors or if the src/dst paths are invalid.
+ */
+ @Override
+ public boolean rename(Path src, Path dst) throws IOException {
+ incrementCounter(Statistic.INVOCATION_RENAME);
+ statistics.incrementWriteOps(1);
+ if (src.equals(dst)) {
+ return true;
+ }
+
+ LOG.trace("rename() from: {} to: {}", src, dst);
+ if (src.isRoot()) {
+ // Cannot rename root of file system
+ LOG.trace("Cannot rename the root of a filesystem");
+ return false;
+ }
+
+ // src and dst should be in the same bucket
+ OFSPath ofsSrc = new OFSPath(src);
+ OFSPath ofsDst = new OFSPath(dst);
+ if (!ofsSrc.isInSameBucketAs(ofsDst)) {
+ throw new IOException("Cannot rename a key to a different bucket");
+ }
+
+ // Cannot rename a directory to its own subdirectory
+ Path dstParent = dst.getParent();
+ while (dstParent != null && !src.equals(dstParent)) {
+ dstParent = dstParent.getParent();
+ }
+ Preconditions.checkArgument(dstParent == null,
+ "Cannot rename a directory to its own subdirectory");
+ // Check if the source exists
+ FileStatus srcStatus;
+ try {
+ srcStatus = getFileStatus(src);
+ } catch (FileNotFoundException fnfe) {
+ // source doesn't exist, return
+ return false;
+ }
+
+ // Check if the destination exists
+ FileStatus dstStatus;
+ try {
+ dstStatus = getFileStatus(dst);
+ } catch (FileNotFoundException fnde) {
+ dstStatus = null;
+ }
+
+ if (dstStatus == null) {
+ // If dst doesn't exist, check whether dst parent dir exists or not
+ // if the parent exists, the source can still be renamed to dst path
+ dstStatus = getFileStatus(dst.getParent());
+ if (!dstStatus.isDirectory()) {
+ throw new IOException(String.format(
+ "Failed to rename %s to %s, %s is a file", src, dst,
+ dst.getParent()));
+ }
+ } else {
+ // if dst exists and source and destination are same,
+ // check both the src and dst are of same type
+ if (srcStatus.getPath().equals(dstStatus.getPath())) {
+ return !srcStatus.isDirectory();
+ } else if (dstStatus.isDirectory()) {
+ // If dst is a directory, rename source as subpath of it.
+ // for example rename /source to /dst will lead to /dst/source
+ dst = new Path(dst, src.getName());
+ FileStatus[] statuses;
+ try {
+ statuses = listStatus(dst);
+ } catch (FileNotFoundException fnde) {
+ statuses = null;
+ }
+
+ if (statuses != null && statuses.length > 0) {
+ // If dst exists and not a directory not empty
+ throw new FileAlreadyExistsException(String.format(
+ "Failed to rename %s to %s, file already exists or not empty!",
+ src, dst));
+ }
+ } else {
+ // If dst is not a directory
+ throw new FileAlreadyExistsException(String.format(
+ "Failed to rename %s to %s, file already exists!", src, dst));
+ }
+ }
+
+ if (srcStatus.isDirectory()) {
+ if (dst.toString().startsWith(src.toString() + OZONE_URI_DELIMITER)) {
+ LOG.trace("Cannot rename a directory to a subdirectory of self");
+ return false;
+ }
+ }
+ RenameIterator iterator = new RenameIterator(src, dst);
+ boolean result = iterator.iterate();
+ if (result) {
+ createFakeParentDirectory(src);
+ }
+ return result;
+ }
+
+ private class DeleteIterator extends OzoneListingIterator {
+ private boolean recursive;
+
+ DeleteIterator(Path f, boolean recursive)
+ throws IOException {
+ super(f);
+ this.recursive = recursive;
+ if (getStatus().isDirectory()
+ && !this.recursive
+ && listStatus(f).length != 0) {
+ throw new PathIsNotEmptyDirectoryException(f.toString());
+ }
+ }
+
+ @Override
+ boolean processKeyPath(String keyPath) throws IOException {
+ if (keyPath.equals("")) {
+ LOG.trace("Skipping deleting root directory");
+ return true;
+ } else {
+ LOG.trace("deleting key path:" + keyPath);
+ boolean succeed = adapter.deleteObject(keyPath);
+ // if recursive delete is requested ignore the return value of
+ // deleteObject and issue deletes for other keys.
+ return recursive || succeed;
+ }
+ }
+ }
+
+ /**
+ * Deletes the children of the input dir path by iterating though the
+ * DeleteIterator.
+ *
+ * @param f directory path to be deleted
+ * @return true if successfully deletes all required keys, false otherwise
+ * @throws IOException
+ */
+ private boolean innerDelete(Path f, boolean recursive) throws IOException {
+ LOG.trace("delete() path:{} recursive:{}", f, recursive);
+ try {
+ DeleteIterator iterator = new DeleteIterator(f, recursive);
+ return iterator.iterate();
+ } catch (FileNotFoundException e) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Couldn't delete {} - does not exist", f);
+ }
+ return false;
+ }
+ }
+
+ @Override
+ public boolean delete(Path f, boolean recursive) throws IOException {
+ incrementCounter(Statistic.INVOCATION_DELETE);
+ statistics.incrementWriteOps(1);
+ LOG.debug("Delete path {} - recursive {}", f, recursive);
+ FileStatus status;
+ try {
+ status = getFileStatus(f);
+ } catch (FileNotFoundException ex) {
+ LOG.warn("delete: Path does not exist: {}", f);
+ return false;
+ }
+
+ String key = pathToKey(f);
+ boolean result;
+
+ if (status.isDirectory()) {
+ LOG.debug("delete: Path is a directory: {}", f);
+ key = addTrailingSlashIfNeeded(key);
+
+ if (key.equals("/")) {
+ LOG.warn("Cannot delete root directory.");
+ return false;
+ }
+
+ result = innerDelete(f, recursive);
+ } else {
+ LOG.debug("delete: Path is a file: {}", f);
+ result = adapter.deleteObject(key);
+ }
+
+ if (result) {
+ // If this delete operation removes all files/directories from the
+ // parent direcotry, then an empty parent directory must be created.
+ createFakeParentDirectory(f);
+ }
+
+ return result;
+ }
+
+ /**
+ * Create a fake parent directory key if it does not already exist and no
+ * other child of this parent directory exists.
+ *
+ * @param f path to the fake parent directory
+ * @throws IOException
+ */
+ private void createFakeParentDirectory(Path f) throws IOException {
+ Path parent = f.getParent();
+ if (parent != null && !parent.isRoot()) {
+ createFakeDirectoryIfNecessary(parent);
+ }
+ }
+
+ /**
+ * Create a fake directory key if it does not already exist.
+ *
+ * @param f path to the fake directory
+ * @throws IOException
+ */
+ private void createFakeDirectoryIfNecessary(Path f) throws IOException {
+ String key = pathToKey(f);
+ if (!key.isEmpty() && !o3Exists(f)) {
+ LOG.debug("Creating new fake directory at {}", f);
+ String dirKey = addTrailingSlashIfNeeded(key);
+ adapter.createDirectory(dirKey);
+ }
+ }
+
+ /**
+ * Check if a file or directory exists corresponding to given path.
+ *
+ * @param f path to file/directory.
+ * @return true if it exists, false otherwise.
+ * @throws IOException
+ */
+ private boolean o3Exists(final Path f) throws IOException {
+ Path path = makeQualified(f);
+ try {
+ getFileStatus(path);
+ return true;
+ } catch (FileNotFoundException ex) {
+ return false;
+ }
+ }
+
+ @Override
+ public FileStatus[] listStatus(Path f) throws IOException {
+ incrementCounter(Statistic.INVOCATION_LIST_STATUS);
+ statistics.incrementReadOps(1);
+ LOG.trace("listStatus() path:{}", f);
+ int numEntries = LISTING_PAGE_SIZE;
+ LinkedList<FileStatus> statuses = new LinkedList<>();
+ List<FileStatus> tmpStatusList;
+ String startPath = "";
+
+ do {
+ tmpStatusList =
+ adapter.listStatus(pathToKey(f), false, startPath,
+ numEntries, uri, workingDir, getUsername())
+ .stream()
+ .map(this::convertFileStatus)
+ .collect(Collectors.toList());
+
+ if (!tmpStatusList.isEmpty()) {
+ if (startPath.isEmpty()) {
+ statuses.addAll(tmpStatusList);
+ } else {
+ statuses.addAll(tmpStatusList.subList(1, tmpStatusList.size()));
+ }
+ startPath = pathToKey(statuses.getLast().getPath());
+ }
+ // listStatus returns entries numEntries in size if available.
+ // Any lesser number of entries indicate that the required entries have
+ // exhausted.
+ } while (tmpStatusList.size() == numEntries);
+
+ return statuses.toArray(new FileStatus[0]);
+ }
+
+ @Override
+ public void setWorkingDirectory(Path newDir) {
+ workingDir = newDir;
+ }
+
+ @Override
+ public Path getWorkingDirectory() {
+ return workingDir;
+ }
+
+ @Override
+ public Token<?> getDelegationToken(String renewer) throws IOException {
+ return adapter.getDelegationToken(renewer);
+ }
+
+ /**
+ * Get a canonical service name for this file system. If the URI is logical,
+ * the hostname part of the URI will be returned.
+ *
+ * @return a service string that uniquely identifies this file system.
+ */
+ @Override
+ public String getCanonicalServiceName() {
+ return adapter.getCanonicalServiceName();
+ }
+
+ /**
+ * Get the username of the FS.
+ *
+ * @return the short name of the user who instantiated the FS
+ */
+ public String getUsername() {
+ return userName;
+ }
+
+ /**
+ * Creates a directory. Directory is represented using a key with no value.
+ *
+ * @param path directory path to be created
+ * @return true if directory exists or created successfully.
+ * @throws IOException
+ */
+ private boolean mkdir(Path path) throws IOException {
+ return adapter.createDirectory(pathToKey(path));
+ }
+
+ @Override
+ public boolean mkdirs(Path f, FsPermission permission) throws IOException {
+ LOG.trace("mkdir() path:{} ", f);
+ String key = pathToKey(f);
+ if (isEmpty(key)) {
+ return false;
+ }
+ return mkdir(f);
+ }
+
+ @Override
+ public FileStatus getFileStatus(Path f) throws IOException {
+ incrementCounter(Statistic.INVOCATION_GET_FILE_STATUS);
+ statistics.incrementReadOps(1);
+ LOG.trace("getFileStatus() path:{}", f);
+ Path qualifiedPath = f.makeQualified(uri, workingDir);
+ String key = pathToKey(qualifiedPath);
+ FileStatus fileStatus = null;
+ try {
+ fileStatus = convertFileStatus(
+ adapter.getFileStatus(key, uri, qualifiedPath, getUsername()));
+ } catch (OMException ex) {
+ if (ex.getResult().equals(OMException.ResultCodes.KEY_NOT_FOUND)) {
+ throw new FileNotFoundException("File not found. path:" + f);
+ }
+ }
+ return fileStatus;
+ }
+
+ @Override
+ public BlockLocation[] getFileBlockLocations(FileStatus fileStatus,
+ long start, long len)
+ throws IOException {
+ if (fileStatus instanceof LocatedFileStatus) {
+ return ((LocatedFileStatus) fileStatus).getBlockLocations();
+ } else {
+ return super.getFileBlockLocations(fileStatus, start, len);
+ }
+ }
+
+ /**
+ * Turn a path (relative or otherwise) into an Ozone key.
+ *
+ * @param path the path of the file.
+ * @return the key of the object that represents the file.
+ */
+ public String pathToKey(Path path) {
+ Objects.requireNonNull(path, "Path can't be null!");
Review comment:
I am confused about pathToKey() functionality. If the input path is not an absolute path, it prepends the working dir (/user/username/) to the path. Please correct me if my understanding is wrong.
For o3fs, if the input key is lets say "dir1/dir2/key1", then pathToKey would be /user/username/dir1/dir2/key1. The volume and bucket would have been set already while creating the FS object.
The same example with ofs would result in volume being set as "user" and bucket as "username" and key as "dir1/dir2/key1". Here, we are making an assumption of the volume and bucket. Would it not be better to not allow non-absolute paths in ofs?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] smengcl commented on a change in pull request #415:
HDDS-2840. Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
smengcl commented on a change in pull request #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#discussion_r371196328
##########
File path: hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
##########
@@ -0,0 +1,604 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.ozone;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.key.KeyProvider;
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdds.client.ReplicationFactor;
+import org.apache.hadoop.hdds.client.ReplicationType;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ozone.OmUtils;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.client.OzoneKey;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
+import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
+import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenRenewer;
+
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .VOLUME_NOT_FOUND;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .BUCKET_NOT_FOUND;
+
+/**
+ * Basic Implementation of the OzoneFileSystem calls.
+ * <p>
+ * This is the minimal version which doesn't include any statistics.
+ * <p>
+ * For full featured version use OzoneClientAdapterImpl.
+ */
+public class BasicRootedOzoneClientAdapterImpl
+ implements RootedOzoneClientAdapter {
+
+ static final Logger LOG =
+ LoggerFactory.getLogger(BasicRootedOzoneClientAdapterImpl.class);
+
+ private OzoneClient ozoneClient;
+ private ClientProtocol proxy;
+ private ObjectStore objectStore;
+ private ReplicationType replicationType;
+ private ReplicationFactor replicationFactor;
+ private boolean securityEnabled;
+ private int configuredDnPort;
+
+ /**
+ * Create new OzoneClientAdapter implementation.
+ *
+ * @throws IOException In case of a problem.
+ */
+ public BasicRootedOzoneClientAdapterImpl() throws IOException {
+ this(createConf());
+ }
+
+ private static OzoneConfiguration createConf() {
+ ClassLoader contextClassLoader =
+ Thread.currentThread().getContextClassLoader();
+ Thread.currentThread().setContextClassLoader(null);
+ try {
+ return new OzoneConfiguration();
+ } finally {
+ Thread.currentThread().setContextClassLoader(contextClassLoader);
+ }
+ }
+
+ public BasicRootedOzoneClientAdapterImpl(OzoneConfiguration conf)
+ throws IOException {
+ this(null, -1, conf);
+ }
+
+ public BasicRootedOzoneClientAdapterImpl(String omHost, int omPort,
+ Configuration hadoopConf) throws IOException {
+
+ ClassLoader contextClassLoader =
+ Thread.currentThread().getContextClassLoader();
+ Thread.currentThread().setContextClassLoader(null);
+
+ try {
+ OzoneConfiguration conf = OzoneConfiguration.of(hadoopConf);
+
+ if (omHost == null && OmUtils.isServiceIdsDefined(conf)) {
+ // When the host name or service id isn't given
+ // but ozone.om.service.ids is defined, declare failure.
+
+ // This is a safety precaution that prevents the client from
+ // accidentally failing over to an unintended OM.
+ throw new IllegalArgumentException("Service ID or host name must not"
+ + " be omitted when ozone.om.service.ids is defined.");
+ }
+
+ if (omPort != -1) {
+ // When the port number is specified, perform the following check
+ if (OmUtils.isOmHAServiceId(conf, omHost)) {
+ // If omHost is a service id, it shouldn't use a port
+ throw new IllegalArgumentException("Port " + omPort +
+ " specified in URI but host '" + omHost + "' is a "
+ + "logical (HA) OzoneManager and does not use port information.");
+ }
+ } else {
+ // When port number is not specified, read it from config
+ omPort = OmUtils.getOmRpcPort(conf);
+ }
+
+ SecurityConfig secConfig = new SecurityConfig(conf);
+
+ if (secConfig.isSecurityEnabled()) {
+ this.securityEnabled = true;
+ }
+
+ String replicationTypeConf =
+ conf.get(OzoneConfigKeys.OZONE_REPLICATION_TYPE,
+ OzoneConfigKeys.OZONE_REPLICATION_TYPE_DEFAULT);
+
+ int replicationCountConf = conf.getInt(OzoneConfigKeys.OZONE_REPLICATION,
+ OzoneConfigKeys.OZONE_REPLICATION_DEFAULT);
+
+ if (OmUtils.isOmHAServiceId(conf, omHost)) {
+ // omHost is listed as one of the service ids in the config,
+ // thus we should treat omHost as omServiceId
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(omHost, conf);
+ } else if (StringUtils.isNotEmpty(omHost) && omPort != -1) {
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(omHost, omPort, conf);
+ } else {
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(conf);
+ }
+ objectStore = ozoneClient.getObjectStore();
+ proxy = objectStore.getClientProxy();
+ this.replicationType = ReplicationType.valueOf(replicationTypeConf);
+ this.replicationFactor = ReplicationFactor.valueOf(replicationCountConf);
+ this.configuredDnPort = conf.getInt(
+ OzoneConfigKeys.DFS_CONTAINER_IPC_PORT,
+ OzoneConfigKeys.DFS_CONTAINER_IPC_PORT_DEFAULT);
+ } finally {
+ Thread.currentThread().setContextClassLoader(contextClassLoader);
+ }
+ }
+
+ private OzoneBucket getBucket(OFSPath ofsPath,
+ boolean createIfNotExist) throws IOException {
+
+ return getBucket(ofsPath.getVolumeName(), ofsPath.getBucketName(),
+ createIfNotExist);
+ }
+
+ /**
+ * Get OzoneBucket object to operate in.
+ * Optionally create volume and bucket if not found.
+ *
+ * @param createIfNotExist Set this to true if the caller is a write operation
+ * in order to create the volume and bucket.
+ * @throws IOException Exceptions other than OMException with result code
+ * VOLUME_NOT_FOUND or BUCKET_NOT_FOUND.
+ */
+ private OzoneBucket getBucket(String volumeStr, String bucketStr,
+ boolean createIfNotExist) throws IOException {
+
+ OzoneBucket bucket;
+ try {
+ bucket = proxy.getBucketDetails(volumeStr, bucketStr);
+ } catch (OMException ex) {
+ if (createIfNotExist) {
+ // Note: getBucketDetails always throws BUCKET_NOT_FOUND, even if
+ // the volume doesn't exist.
+ if (ex.getResult().equals(BUCKET_NOT_FOUND)) {
+ OzoneVolume volume;
+ try {
+ volume = proxy.getVolumeDetails(volumeStr);
+ } catch (OMException volEx) {
+ if (volEx.getResult().equals(VOLUME_NOT_FOUND)) {
+ // Volume doesn't exist. Create it
+ objectStore.createVolume(volumeStr);
+ } else {
+ throw volEx;
+ }
+ // Try get volume again
+ volume = proxy.getVolumeDetails(volumeStr);
+ }
+ // Create the bucket
+ volume.createBucket(bucketStr);
+ }
+ // Try get bucket again
+ bucket = proxy.getBucketDetails(volumeStr, bucketStr);
+ } else {
+ throw ex;
+ }
+ }
+
+ return bucket;
+ }
+
+ @Override
+ public void close() throws IOException {
+ ozoneClient.close();
+ }
+
+ @Override
+ public InputStream readFile(String pathStr) throws IOException {
+ incrementCounter(Statistic.OBJECTS_READ);
+ OFSPath ofsPath = new OFSPath(pathStr);
+ String key = ofsPath.getKeyName();
+ try {
+ OzoneBucket bucket = getBucket(ofsPath, false);
+ return bucket.readFile(key).getInputStream();
+ } catch (OMException ex) {
+ if (ex.getResult() == OMException.ResultCodes.FILE_NOT_FOUND
+ || ex.getResult() == OMException.ResultCodes.NOT_A_FILE) {
+ throw new FileNotFoundException(
+ ex.getResult().name() + ": " + ex.getMessage());
+ } else {
+ throw ex;
+ }
+ }
+ }
+
+ protected void incrementCounter(Statistic objectsRead) {
+ //noop: Use OzoneClientAdapterImpl which supports statistics.
+ }
+
+ @Override
+ public OzoneFSOutputStream createFile(String pathStr, boolean overWrite,
+ boolean recursive) throws IOException {
+ incrementCounter(Statistic.OBJECTS_CREATED);
+ OFSPath ofsPath = new OFSPath(pathStr);
+ String key = ofsPath.getKeyName();
+ try {
+ OzoneBucket bucket = getBucket(ofsPath, false);
+ OzoneOutputStream ozoneOutputStream = bucket.createFile(
+ key, 0, replicationType, replicationFactor, overWrite, recursive);
+ return new OzoneFSOutputStream(ozoneOutputStream.getOutputStream());
+ } catch (OMException ex) {
+ if (ex.getResult() == OMException.ResultCodes.FILE_ALREADY_EXISTS
+ || ex.getResult() == OMException.ResultCodes.NOT_A_FILE) {
+ throw new FileAlreadyExistsException(
+ ex.getResult().name() + ": " + ex.getMessage());
+ } else {
+ throw ex;
+ }
+ }
+ }
+
+ @Override
+ public void renamePath(String path, String newPath) throws IOException {
+ incrementCounter(Statistic.OBJECTS_RENAMED);
+ OFSPath ofsPath = new OFSPath(path);
+ OFSPath ofsNewPath = new OFSPath(newPath);
+
+ // Check path and newPathName are in the same volume and same bucket.
+ // This should have been checked in BasicRootedOzoneFileSystem#rename
+ // already via regular call path unless bypassed.
+ if (!ofsPath.isInSameBucketAs(ofsNewPath)) {
+ throw new IOException("Cannot rename a key to a different bucket");
+ }
+
+ OzoneBucket bucket = getBucket(ofsPath, false);
+ String key = ofsPath.getKeyName();
+ String newKey = ofsNewPath.getKeyName();
+ bucket.renameKey(key, newKey);
+ }
+
+ /**
+ * Helper method to create an directory specified by key name in bucket.
+ *
+ * @param pathStr path to be created as directory
+ * @return true if the key is created, false otherwise
+ */
+ @Override
+ public boolean createDirectory(String pathStr) throws IOException {
+ LOG.trace("creating dir for path: {}", pathStr);
+ incrementCounter(Statistic.OBJECTS_CREATED);
+ OFSPath ofsPath = new OFSPath(pathStr);
+ String keyStr = ofsPath.getKeyName();
+ try {
+ OzoneBucket bucket = getBucket(ofsPath, true);
+ bucket.createDirectory(keyStr);
+ } catch (OMException e) {
+ if (e.getResult() == OMException.ResultCodes.FILE_ALREADY_EXISTS) {
+ throw new FileAlreadyExistsException(e.getMessage());
+ }
+ throw e;
+ }
+ return true;
+ }
+
+ /**
+ * Helper method to delete an object specified by key name in bucket.
+ *
+ * @param path path to a key to be deleted
+ * @return true if the key is deleted, false otherwise
+ */
+ @Override
+ public boolean deleteObject(String path) {
+ LOG.trace("issuing delete for path to key: {}", path);
+ OFSPath ofsPath = new OFSPath(path);
+ String keyName = ofsPath.getKeyName();
+ try {
+ OzoneBucket bucket = getBucket(ofsPath, false);
+ incrementCounter(Statistic.OBJECTS_DELETED);
+ bucket.deleteKey(keyName);
+ return true;
+ } catch (IOException ioe) {
+ LOG.error("delete key failed " + ioe.getMessage());
+ return false;
+ }
+ }
+
+ public FileStatusAdapter getFileStatus(String path, URI uri,
+ Path qualifiedPath, String userName)
+ throws IOException {
+ OFSPath ofsPath = new OFSPath(path);
+ String key = ofsPath.getKeyName();
+ try {
+ OzoneBucket bucket = getBucket(ofsPath, false);
+ incrementCounter(Statistic.OBJECTS_QUERY);
+ OzoneFileStatus status = bucket.getFileStatus(key);
+ // Note: qualifiedPath passed in is good from
+ // BasicRootedOzoneFileSystem#getFileStatus. No need to prepend here.
+ makeQualified(status, uri, qualifiedPath, userName);
+ return toFileStatusAdapter(status);
+
+ } catch (OMException e) {
+ if (e.getResult() == OMException.ResultCodes.FILE_NOT_FOUND) {
+ throw new
+ FileNotFoundException(key + ": No such file or directory!");
+ }
+ throw e;
+ }
+ }
+
+ public void makeQualified(FileStatus status, URI uri, Path path,
+ String username) {
+ if (status instanceof OzoneFileStatus) {
+ ((OzoneFileStatus) status)
+ .makeQualified(uri, path,
+ username, username);
+ }
+ }
+
+ @Override
+ public Iterator<BasicKeyInfo> listKeys(String pathStr) throws IOException {
+ incrementCounter(Statistic.OBJECTS_LIST);
+ OFSPath ofsPath = new OFSPath(pathStr);
+ String key = ofsPath.getKeyName();
+ OzoneBucket bucket = getBucket(ofsPath, false);
+ return new IteratorAdapter(bucket.listKeys(key));
+ }
+
+ public List<FileStatusAdapter> listStatus(String pathStr, boolean recursive,
+ String startPath, long numEntries, URI uri,
+ Path workingDir, String username) throws IOException {
+
+ OFSPath ofsPath = new OFSPath(pathStr);
+ String keyName = ofsPath.getKeyName();
+ OFSPath ofsStartPath = new OFSPath(startPath);
+ String startKey = ofsStartPath.getKeyName();
Review comment:
Done in b21005ebacf25f5a3a6bfdb97348a2e2e567423d.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] smengcl commented on a change in pull request #415:
HDDS-2840. Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
smengcl commented on a change in pull request #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#discussion_r371253128
##########
File path: hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
##########
@@ -0,0 +1,805 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.ozone;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.CreateFlag;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.Progressable;
+import org.apache.http.client.utils.URIBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.EnumSet;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+import static org.apache.hadoop.fs.ozone.Constants.LISTING_PAGE_SIZE;
+import static org.apache.hadoop.fs.ozone.Constants.OZONE_DEFAULT_USER;
+import static org.apache.hadoop.fs.ozone.Constants.OZONE_USER_DIR;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_OFS_URI_SCHEME;
+
+/**
+ * The minimal Ozone Filesystem implementation.
+ * <p>
+ * This is a basic version which doesn't extend
+ * KeyProviderTokenIssuer and doesn't include statistics. It can be used
+ * from older hadoop version. For newer hadoop version use the full featured
+ * OFileSystem.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class BasicRootedOzoneFileSystem extends FileSystem {
+ static final Logger LOG =
+ LoggerFactory.getLogger(BasicRootedOzoneFileSystem.class);
+
+ /**
+ * The Ozone client for connecting to Ozone server.
+ */
+
+ private URI uri;
+ private String userName;
+ private Path workingDir;
+// private String gOmHost;
+// private int gOmPort;
+// private Configuration gConf;
+// private boolean gIsolatedClassloader;
+
+ private RootedOzoneClientAdapter adapter;
+// private String adapterPath;
+
+ private static final String URI_EXCEPTION_TEXT =
+ "URL should be one of the following formats: " +
+ "ofs://om-service-id/ OR " +
+ "ofs://om-host.example.com/ OR " +
+ "ofs://om-host.example.com:5678/";
+
+ @Override
+ public void initialize(URI name, Configuration conf) throws IOException {
+ super.initialize(name, conf);
+ setConf(conf);
+ Objects.requireNonNull(name.getScheme(), "No scheme provided in " + name);
+ Preconditions.checkArgument(getScheme().equals(name.getScheme()),
+ "Invalid scheme provided in " + name);
+
+ String authority = name.getAuthority();
+ if (authority == null) {
+ // authority is null when fs.defaultFS is not a qualified ofs URI and
+ // ofs:/// is passed to the client. matcher will NPE if authority is null
+ throw new IllegalArgumentException(URI_EXCEPTION_TEXT);
+ }
+
+ String omHost;
+ int omPort = -1;
+ // Parse hostname and port
+ String[] parts = authority.split(":");
+ if (parts.length > 2) {
+ throw new IllegalArgumentException(URI_EXCEPTION_TEXT);
+ }
+ omHost = parts[0];
+ if (parts.length == 2) {
+ try {
+ omPort = Integer.parseInt(parts[1]);
+ } catch (NumberFormatException e) {
+ throw new IllegalArgumentException(URI_EXCEPTION_TEXT);
+ }
+ }
+
+ try {
+ uri = new URIBuilder().setScheme(OZONE_OFS_URI_SCHEME)
+ .setHost(authority)
+ .build();
+ LOG.trace("Ozone URI for OFS initialization is " + uri);
+
+ //isolated is the default for ozonefs-lib-legacy which includes the
+ // /ozonefs.txt, otherwise the default is false. It could be overridden.
+ boolean defaultValue =
+ BasicRootedOzoneFileSystem.class.getClassLoader()
+ .getResource("ozonefs.txt") != null;
+
+ //Use string here instead of the constant as constant may not be available
+ //on the classpath of a hadoop 2.7
+ boolean isolatedClassloader =
+ conf.getBoolean("ozone.fs.isolated-classloader", defaultValue);
+
+ // adapter should be initialized in operations.
+ this.adapter = createAdapter(conf, omHost, omPort, isolatedClassloader);
+
+ try {
+ this.userName =
+ UserGroupInformation.getCurrentUser().getShortUserName();
+ } catch (IOException e) {
+ this.userName = OZONE_DEFAULT_USER;
+ }
+ this.workingDir = new Path(OZONE_USER_DIR, this.userName)
+ .makeQualified(this.uri, this.workingDir);
+ } catch (URISyntaxException ue) {
+ final String msg = "Invalid Ozone endpoint " + name;
+ LOG.error(msg, ue);
+ throw new IOException(msg, ue);
+ }
+ }
+
+ protected RootedOzoneClientAdapter createAdapter(Configuration conf,
+ String omHost, int omPort, boolean isolatedClassloader)
+ throws IOException {
+
+ if (isolatedClassloader) {
+ // TODO: Check how this code path need to be changed, for legacy Hadoop?
+ return RootedOzoneClientAdapterFactory.createAdapter();
+ } else {
+ return new BasicRootedOzoneClientAdapterImpl(omHost, omPort, conf);
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ try {
+ adapter.close();
+ } finally {
+ super.close();
+ }
+ }
+
+ @Override
+ public URI getUri() {
+ return uri;
+ }
+
+ @Override
+ public String getScheme() {
+ return OZONE_OFS_URI_SCHEME;
+ }
+
+ @Override
+ public FSDataInputStream open(Path path, int bufferSize) throws IOException {
+ incrementCounter(Statistic.INVOCATION_OPEN);
+ statistics.incrementReadOps(1);
+ LOG.trace("open() path: {}", path);
+ final String key = pathToKey(path);
+ return new FSDataInputStream(
+ new OzoneFSInputStream(adapter.readFile(key), statistics));
+ }
+
+ protected void incrementCounter(Statistic statistic) {
+ //don't do anything in this default implementation.
+ }
+
+ @Override
+ public FSDataOutputStream create(Path f, FsPermission permission,
+ boolean overwrite, int bufferSize,
+ short replication, long blockSize,
+ Progressable progress) throws IOException {
+ LOG.trace("create() path:{}", f);
+ incrementCounter(Statistic.INVOCATION_CREATE);
+ statistics.incrementWriteOps(1);
+ final String key = pathToKey(f);
+ return createOutputStream(key, overwrite, true);
+ }
+
+ @Override
+ public FSDataOutputStream createNonRecursive(Path path,
+ FsPermission permission,
+ EnumSet<CreateFlag> flags,
+ int bufferSize,
+ short replication,
+ long blockSize,
+ Progressable progress) throws IOException {
+ incrementCounter(Statistic.INVOCATION_CREATE_NON_RECURSIVE);
+ statistics.incrementWriteOps(1);
+ final String key = pathToKey(path);
+ return createOutputStream(key, flags.contains(CreateFlag.OVERWRITE), false);
+ }
+
+ private FSDataOutputStream createOutputStream(String key, boolean overwrite,
+ boolean recursive) throws IOException {
+ return new FSDataOutputStream(adapter.createFile(key, overwrite, recursive),
+ statistics);
+ }
+
+ @Override
+ public FSDataOutputStream append(Path f, int bufferSize,
+ Progressable progress) throws IOException {
+ throw new UnsupportedOperationException("append() Not implemented by the "
+ + getClass().getSimpleName() + " FileSystem implementation");
+ }
+
+ private class RenameIterator extends OzoneListingIterator {
+ private final String srcPath;
+ private final String dstPath;
+
+ RenameIterator(Path srcPath, Path dstPath)
+ throws IOException {
+ super(srcPath);
+ this.srcPath = pathToKey(srcPath);
+ this.dstPath = pathToKey(dstPath);
+ LOG.trace("rename from:{} to:{}", this.srcPath, this.dstPath);
+ }
+
+ @Override
+ boolean processKeyPath(String keyPath) throws IOException {
+ String newPath = dstPath.concat(keyPath.substring(srcPath.length()));
+ adapter.renamePath(keyPath, newPath);
+ return true;
+ }
+ }
+
+ /**
+ * Check whether the source and destination path are valid and then perform
+ * rename from source path to destination path.
+ * <p>
+ * The rename operation is performed by renaming the keys with src as prefix.
+ * For such keys the prefix is changed from src to dst.
+ *
+ * @param src source path for rename
+ * @param dst destination path for rename
+ * @return true if rename operation succeeded or
+ * if the src and dst have the same path and are of the same type
+ * @throws IOException on I/O errors or if the src/dst paths are invalid.
+ */
+ @Override
+ public boolean rename(Path src, Path dst) throws IOException {
+ incrementCounter(Statistic.INVOCATION_RENAME);
+ statistics.incrementWriteOps(1);
+ if (src.equals(dst)) {
+ return true;
+ }
+
+ LOG.trace("rename() from: {} to: {}", src, dst);
+ if (src.isRoot()) {
+ // Cannot rename root of file system
+ LOG.trace("Cannot rename the root of a filesystem");
+ return false;
+ }
+
+ // src and dst should be in the same bucket
+ OFSPath ofsSrc = new OFSPath(src);
+ OFSPath ofsDst = new OFSPath(dst);
+ if (!ofsSrc.isInSameBucketAs(ofsDst)) {
+ throw new IOException("Cannot rename a key to a different bucket");
+ }
+
+ // Cannot rename a directory to its own subdirectory
+ Path dstParent = dst.getParent();
+ while (dstParent != null && !src.equals(dstParent)) {
+ dstParent = dstParent.getParent();
+ }
+ Preconditions.checkArgument(dstParent == null,
+ "Cannot rename a directory to its own subdirectory");
+ // Check if the source exists
+ FileStatus srcStatus;
+ try {
+ srcStatus = getFileStatus(src);
+ } catch (FileNotFoundException fnfe) {
+ // source doesn't exist, return
+ return false;
+ }
+
+ // Check if the destination exists
+ FileStatus dstStatus;
+ try {
+ dstStatus = getFileStatus(dst);
+ } catch (FileNotFoundException fnde) {
+ dstStatus = null;
+ }
+
+ if (dstStatus == null) {
+ // If dst doesn't exist, check whether dst parent dir exists or not
+ // if the parent exists, the source can still be renamed to dst path
+ dstStatus = getFileStatus(dst.getParent());
+ if (!dstStatus.isDirectory()) {
+ throw new IOException(String.format(
+ "Failed to rename %s to %s, %s is a file", src, dst,
+ dst.getParent()));
+ }
+ } else {
+ // if dst exists and source and destination are same,
+ // check both the src and dst are of same type
+ if (srcStatus.getPath().equals(dstStatus.getPath())) {
+ return !srcStatus.isDirectory();
+ } else if (dstStatus.isDirectory()) {
+ // If dst is a directory, rename source as subpath of it.
+ // for example rename /source to /dst will lead to /dst/source
+ dst = new Path(dst, src.getName());
+ FileStatus[] statuses;
+ try {
+ statuses = listStatus(dst);
+ } catch (FileNotFoundException fnde) {
+ statuses = null;
+ }
+
+ if (statuses != null && statuses.length > 0) {
+ // If dst exists and not a directory not empty
+ throw new FileAlreadyExistsException(String.format(
+ "Failed to rename %s to %s, file already exists or not empty!",
+ src, dst));
+ }
+ } else {
+ // If dst is not a directory
+ throw new FileAlreadyExistsException(String.format(
+ "Failed to rename %s to %s, file already exists!", src, dst));
+ }
+ }
+
+ if (srcStatus.isDirectory()) {
+ if (dst.toString().startsWith(src.toString() + OZONE_URI_DELIMITER)) {
+ LOG.trace("Cannot rename a directory to a subdirectory of self");
+ return false;
+ }
+ }
+ RenameIterator iterator = new RenameIterator(src, dst);
+ boolean result = iterator.iterate();
+ if (result) {
+ createFakeParentDirectory(src);
+ }
+ return result;
+ }
+
+ private class DeleteIterator extends OzoneListingIterator {
+ private boolean recursive;
+
+ DeleteIterator(Path f, boolean recursive)
+ throws IOException {
+ super(f);
+ this.recursive = recursive;
+ if (getStatus().isDirectory()
+ && !this.recursive
+ && listStatus(f).length != 0) {
+ throw new PathIsNotEmptyDirectoryException(f.toString());
+ }
+ }
+
+ @Override
+ boolean processKeyPath(String keyPath) throws IOException {
+ if (keyPath.equals("")) {
+ LOG.trace("Skipping deleting root directory");
+ return true;
+ } else {
+ LOG.trace("deleting key path:" + keyPath);
+ boolean succeed = adapter.deleteObject(keyPath);
+ // if recursive delete is requested ignore the return value of
+ // deleteObject and issue deletes for other keys.
+ return recursive || succeed;
+ }
+ }
+ }
+
+ /**
+ * Deletes the children of the input dir path by iterating though the
+ * DeleteIterator.
+ *
+ * @param f directory path to be deleted
+ * @return true if successfully deletes all required keys, false otherwise
+ * @throws IOException
+ */
+ private boolean innerDelete(Path f, boolean recursive) throws IOException {
+ LOG.trace("delete() path:{} recursive:{}", f, recursive);
+ try {
+ DeleteIterator iterator = new DeleteIterator(f, recursive);
+ return iterator.iterate();
+ } catch (FileNotFoundException e) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Couldn't delete {} - does not exist", f);
+ }
+ return false;
+ }
+ }
+
+ @Override
+ public boolean delete(Path f, boolean recursive) throws IOException {
+ incrementCounter(Statistic.INVOCATION_DELETE);
+ statistics.incrementWriteOps(1);
+ LOG.debug("Delete path {} - recursive {}", f, recursive);
+ FileStatus status;
+ try {
+ status = getFileStatus(f);
+ } catch (FileNotFoundException ex) {
+ LOG.warn("delete: Path does not exist: {}", f);
+ return false;
+ }
+
+ String key = pathToKey(f);
+ boolean result;
+
+ if (status.isDirectory()) {
+ LOG.debug("delete: Path is a directory: {}", f);
+ key = addTrailingSlashIfNeeded(key);
+
+ if (key.equals("/")) {
+ LOG.warn("Cannot delete root directory.");
+ return false;
+ }
+
+ result = innerDelete(f, recursive);
+ } else {
+ LOG.debug("delete: Path is a file: {}", f);
+ result = adapter.deleteObject(key);
+ }
+
+ if (result) {
+ // If this delete operation removes all files/directories from the
+ // parent direcotry, then an empty parent directory must be created.
+ createFakeParentDirectory(f);
+ }
+
+ return result;
+ }
+
+ /**
+ * Create a fake parent directory key if it does not already exist and no
+ * other child of this parent directory exists.
+ *
+ * @param f path to the fake parent directory
+ * @throws IOException
+ */
+ private void createFakeParentDirectory(Path f) throws IOException {
+ Path parent = f.getParent();
+ if (parent != null && !parent.isRoot()) {
+ createFakeDirectoryIfNecessary(parent);
+ }
+ }
+
+ /**
+ * Create a fake directory key if it does not already exist.
+ *
+ * @param f path to the fake directory
+ * @throws IOException
+ */
+ private void createFakeDirectoryIfNecessary(Path f) throws IOException {
+ String key = pathToKey(f);
+ if (!key.isEmpty() && !o3Exists(f)) {
+ LOG.debug("Creating new fake directory at {}", f);
+ String dirKey = addTrailingSlashIfNeeded(key);
+ adapter.createDirectory(dirKey);
+ }
+ }
+
+ /**
+ * Check if a file or directory exists corresponding to given path.
+ *
+ * @param f path to file/directory.
+ * @return true if it exists, false otherwise.
+ * @throws IOException
+ */
+ private boolean o3Exists(final Path f) throws IOException {
+ Path path = makeQualified(f);
+ try {
+ getFileStatus(path);
+ return true;
+ } catch (FileNotFoundException ex) {
+ return false;
+ }
+ }
+
+ @Override
+ public FileStatus[] listStatus(Path f) throws IOException {
+ incrementCounter(Statistic.INVOCATION_LIST_STATUS);
+ statistics.incrementReadOps(1);
+ LOG.trace("listStatus() path:{}", f);
+ int numEntries = LISTING_PAGE_SIZE;
+ LinkedList<FileStatus> statuses = new LinkedList<>();
+ List<FileStatus> tmpStatusList;
+ String startPath = "";
+
+ do {
+ tmpStatusList =
+ adapter.listStatus(pathToKey(f), false, startPath,
+ numEntries, uri, workingDir, getUsername())
+ .stream()
+ .map(this::convertFileStatus)
+ .collect(Collectors.toList());
+
+ if (!tmpStatusList.isEmpty()) {
+ if (startPath.isEmpty()) {
+ statuses.addAll(tmpStatusList);
+ } else {
+ statuses.addAll(tmpStatusList.subList(1, tmpStatusList.size()));
+ }
+ startPath = pathToKey(statuses.getLast().getPath());
+ }
+ // listStatus returns entries numEntries in size if available.
+ // Any lesser number of entries indicate that the required entries have
+ // exhausted.
+ } while (tmpStatusList.size() == numEntries);
+
+ return statuses.toArray(new FileStatus[0]);
+ }
+
+ @Override
+ public void setWorkingDirectory(Path newDir) {
+ workingDir = newDir;
+ }
+
+ @Override
+ public Path getWorkingDirectory() {
+ return workingDir;
+ }
+
+ @Override
+ public Token<?> getDelegationToken(String renewer) throws IOException {
+ return adapter.getDelegationToken(renewer);
+ }
+
+ /**
+ * Get a canonical service name for this file system. If the URI is logical,
+ * the hostname part of the URI will be returned.
+ *
+ * @return a service string that uniquely identifies this file system.
+ */
+ @Override
+ public String getCanonicalServiceName() {
+ return adapter.getCanonicalServiceName();
+ }
+
+ /**
+ * Get the username of the FS.
+ *
+ * @return the short name of the user who instantiated the FS
+ */
+ public String getUsername() {
+ return userName;
+ }
+
+ /**
+ * Creates a directory. Directory is represented using a key with no value.
+ *
+ * @param path directory path to be created
+ * @return true if directory exists or created successfully.
+ * @throws IOException
+ */
+ private boolean mkdir(Path path) throws IOException {
+ return adapter.createDirectory(pathToKey(path));
+ }
+
+ @Override
+ public boolean mkdirs(Path f, FsPermission permission) throws IOException {
+ LOG.trace("mkdir() path:{} ", f);
+ String key = pathToKey(f);
+ if (isEmpty(key)) {
+ return false;
+ }
+ return mkdir(f);
+ }
+
+ @Override
+ public FileStatus getFileStatus(Path f) throws IOException {
+ incrementCounter(Statistic.INVOCATION_GET_FILE_STATUS);
+ statistics.incrementReadOps(1);
+ LOG.trace("getFileStatus() path:{}", f);
+ Path qualifiedPath = f.makeQualified(uri, workingDir);
+ String key = pathToKey(qualifiedPath);
+ FileStatus fileStatus = null;
+ try {
+ fileStatus = convertFileStatus(
+ adapter.getFileStatus(key, uri, qualifiedPath, getUsername()));
+ } catch (OMException ex) {
+ if (ex.getResult().equals(OMException.ResultCodes.KEY_NOT_FOUND)) {
+ throw new FileNotFoundException("File not found. path:" + f);
+ }
+ }
+ return fileStatus;
+ }
+
+ @Override
+ public BlockLocation[] getFileBlockLocations(FileStatus fileStatus,
+ long start, long len)
+ throws IOException {
+ if (fileStatus instanceof LocatedFileStatus) {
+ return ((LocatedFileStatus) fileStatus).getBlockLocations();
+ } else {
+ return super.getFileBlockLocations(fileStatus, start, len);
+ }
+ }
+
+ /**
+ * Turn a path (relative or otherwise) into an Ozone key.
+ *
+ * @param path the path of the file.
+ * @return the key of the object that represents the file.
+ */
+ public String pathToKey(Path path) {
+ Objects.requireNonNull(path, "Path can't be null!");
+ if (!path.isAbsolute()) {
+ path = new Path(workingDir, path);
+ }
+ // removing leading '/' char
+ String key = path.toUri().getPath().substring(1);
+ LOG.trace("path for key: {} is: {}", key, path);
+ return key;
+ }
+
+ /**
+ * Add trailing delimiter to path if it is already not present.
+ *
+ * @param key the ozone Key which needs to be appended
+ * @return delimiter appended key
+ */
+ private String addTrailingSlashIfNeeded(String key) {
+ if (!isEmpty(key) && !key.endsWith(OZONE_URI_DELIMITER)) {
+ return key + OZONE_URI_DELIMITER;
+ } else {
+ return key;
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "RootedOzoneFileSystem{URI=" + uri + ", "
+ + "workingDir=" + workingDir + ", "
+ + "userName=" + userName + ", "
+ + "statistics=" + statistics
+ + "}";
+ }
+
+ /**
+ * This class provides an interface to iterate through all the keys in the
+ * bucket prefixed with the input path key and process them.
+ * <p>
+ * Each implementing class should define how the keys should be processed
+ * through the processKeyPath() function.
+ */
+ private abstract class OzoneListingIterator {
+ private final Path path;
+ private final FileStatus status;
+ private String pathKey;
+ private Iterator<BasicKeyInfo> keyIterator;
+
+ OzoneListingIterator(Path path)
+ throws IOException {
+ this.path = path;
+ this.status = getFileStatus(path);
+ this.pathKey = pathToKey(path);
+ if (status.isDirectory()) {
+ this.pathKey = addTrailingSlashIfNeeded(pathKey);
+ }
+ keyIterator = adapter.listKeys(pathKey);
+ }
+
+ /**
+ * The output of processKey determines if further iteration through the
+ * keys should be done or not.
+ *
+ * @return true if we should continue iteration of keys, false otherwise.
+ * @throws IOException
+ */
+ abstract boolean processKeyPath(String keyPath) throws IOException;
+
+ /**
+ * Iterates through all the keys prefixed with the input path's key and
+ * processes the key though processKey().
+ * If for any key, the processKey() returns false, then the iteration is
+ * stopped and returned with false indicating that all the keys could not
+ * be processed successfully.
+ *
+ * @return true if all keys are processed successfully, false otherwise.
+ * @throws IOException
+ */
+ boolean iterate() throws IOException {
+ LOG.trace("Iterating path: {}", path);
+ if (status.isDirectory()) {
+ LOG.trace("Iterating directory: {}", pathKey);
+ OFSPath ofsPath = new OFSPath(pathKey);
+ while (keyIterator.hasNext()) {
+ BasicKeyInfo key = keyIterator.next();
+ String keyPath = ofsPath.getNonKeyPathNoPrefixDelim() +
+ OZONE_URI_DELIMITER + key.getName();
+ LOG.trace("iterating key path: {}", keyPath);
+ if (!processKeyPath(keyPath)) {
+ return false;
+ }
+ }
+ return true;
+ } else {
+ LOG.trace("iterating file: {}", path);
+ return processKeyPath(pathKey);
+ }
+ }
+
+ String getPathKey() {
+ return pathKey;
+ }
+
+ boolean pathIsDirectory() {
+ return status.isDirectory();
+ }
+
+ FileStatus getStatus() {
+ return status;
+ }
+ }
+
+ public RootedOzoneClientAdapter getAdapter() {
+ return adapter;
+ }
+
+ public boolean isEmpty(CharSequence cs) {
+ return cs == null || cs.length() == 0;
+ }
+
+ public boolean isNumber(String number) {
+ try {
+ Integer.parseInt(number);
+ } catch (NumberFormatException ex) {
+ return false;
+ }
+ return true;
+ }
+
+ private FileStatus convertFileStatus(FileStatusAdapter fileStatusAdapter) {
+ Path symLink = null;
+ try {
+ fileStatusAdapter.getSymlink();
+ } catch (Exception ex) {
+ //NOOP: If not symlink symlink remains null.
+ }
+
+ // Process path.
+ URI newUri = fileStatusAdapter.getPath().toUri();
+ try {
+ newUri = new URIBuilder().setScheme(newUri.getScheme())
+ .setHost(newUri.getAuthority())
+ .setPath(newUri.getPath())
+ .build();
+ } catch (URISyntaxException e) {
+ }
Review comment:
This chunk of code is useless now. It was previously added for stitching the listStatus result. Just removed it.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] hanishakoneru commented on a change in pull request
#415: HDDS-2840. Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
hanishakoneru commented on a change in pull request #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#discussion_r369838294
##########
File path: hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystemWithMocks.java
##########
@@ -0,0 +1,152 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.ozone;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.OmUtils;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PowerMockIgnore;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+
+import java.net.URI;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * Ozone File system tests that are light weight and use mocks.
+ */
+@RunWith(PowerMockRunner.class)
+@PrepareForTest({ OzoneClientFactory.class, UserGroupInformation.class })
+@PowerMockIgnore("javax.management.*")
+public class TestRootedOzoneFileSystemWithMocks {
+
+ @Test
+ public void testFSUriWithHostPortOverrides() throws Exception {
+ Configuration conf = new OzoneConfiguration();
+ OzoneClient ozoneClient = mock(OzoneClient.class);
+ ObjectStore objectStore = mock(ObjectStore.class);
+ OzoneVolume volume = mock(OzoneVolume.class);
+ OzoneBucket bucket = mock(OzoneBucket.class);
+
+ when(ozoneClient.getObjectStore()).thenReturn(objectStore);
+ when(objectStore.getVolume(eq("volume1"))).thenReturn(volume);
+ when(volume.getBucket("bucket1")).thenReturn(bucket);
+
+ PowerMockito.mockStatic(OzoneClientFactory.class);
+ PowerMockito.when(OzoneClientFactory.getRpcClient(eq("local.host"),
+ eq(5899), eq(conf))).thenReturn(ozoneClient);
+
+ UserGroupInformation ugi = mock(UserGroupInformation.class);
+ PowerMockito.mockStatic(UserGroupInformation.class);
+ PowerMockito.when(UserGroupInformation.getCurrentUser()).thenReturn(ugi);
+ when(ugi.getShortUserName()).thenReturn("user1");
+
+ // Note: FileSystem#loadFileSystems doesn't load OFS class because
+ // META-INF still points to org.apache.hadoop.fs.ozone.OzoneFileSystem
+ conf.set("fs.ofs.impl", "org.apache.hadoop.fs.ozone.RootedOzoneFileSystem");
+ URI uri = new URI("ofs://local.host:5899/volume1/bucket1");
+
+ FileSystem fileSystem = FileSystem.get(uri, conf);
+ RootedOzoneFileSystem ofs = (RootedOzoneFileSystem) fileSystem;
+
+ assertEquals(ofs.getUri().getAuthority(), "local.host:5899");
+ PowerMockito.verifyStatic();
+ OzoneClientFactory.getRpcClient("local.host", 5899, conf);
+ }
+
+ @Test
+ public void testFSUriWithHostPortUnspecified() throws Exception {
+ Configuration conf = new OzoneConfiguration();
+ final int omPort = OmUtils.getOmRpcPort(conf);
+
+ OzoneClient ozoneClient = mock(OzoneClient.class);
+ ObjectStore objectStore = mock(ObjectStore.class);
+ OzoneVolume volume = mock(OzoneVolume.class);
+ OzoneBucket bucket = mock(OzoneBucket.class);
+
+ when(ozoneClient.getObjectStore()).thenReturn(objectStore);
+ when(objectStore.getVolume(eq("volume1"))).thenReturn(volume);
+ when(volume.getBucket("bucket1")).thenReturn(bucket);
+
+ PowerMockito.mockStatic(OzoneClientFactory.class);
+ PowerMockito.when(OzoneClientFactory.getRpcClient(eq("local.host"),
+ eq(omPort), eq(conf))).thenReturn(ozoneClient);
+
+ UserGroupInformation ugi = mock(UserGroupInformation.class);
+ PowerMockito.mockStatic(UserGroupInformation.class);
+ PowerMockito.when(UserGroupInformation.getCurrentUser()).thenReturn(ugi);
+ when(ugi.getShortUserName()).thenReturn("user1");
+
+ URI uri = new URI("o3fs://bucket1.volume1.local.host");
+
Review comment:
testFSUriHostVersionDefault() and testFSUriWithHostPortUnspecified() need to be modified to test ofs.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] smengcl commented on a change in pull request #415:
HDDS-2840. Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
smengcl commented on a change in pull request #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#discussion_r372552176
##########
File path: hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
##########
@@ -0,0 +1,694 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.ozone;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.key.KeyProvider;
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdds.client.ReplicationFactor;
+import org.apache.hadoop.hdds.client.ReplicationType;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ozone.OmUtils;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.client.OzoneKey;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
+import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
+import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenRenewer;
+
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .BUCKET_ALREADY_EXISTS;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .VOLUME_ALREADY_EXISTS;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .VOLUME_NOT_FOUND;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .BUCKET_NOT_FOUND;
+
+/**
+ * Basic Implementation of the OzoneFileSystem calls.
+ * <p>
+ * This is the minimal version which doesn't include any statistics.
+ * <p>
+ * For full featured version use OzoneClientAdapterImpl.
+ */
+public class BasicRootedOzoneClientAdapterImpl
+ implements RootedOzoneClientAdapter {
+
+ static final Logger LOG =
+ LoggerFactory.getLogger(BasicRootedOzoneClientAdapterImpl.class);
+
+ private OzoneClient ozoneClient;
+ private ClientProtocol proxy;
+ private ObjectStore objectStore;
+ private ReplicationType replicationType;
+ private ReplicationFactor replicationFactor;
+ private boolean securityEnabled;
+ private int configuredDnPort;
+
+ /**
+ * Create new OzoneClientAdapter implementation.
+ *
+ * @throws IOException In case of a problem.
+ */
+ public BasicRootedOzoneClientAdapterImpl() throws IOException {
+ this(createConf());
+ }
+
+ private static OzoneConfiguration createConf() {
+ ClassLoader contextClassLoader =
+ Thread.currentThread().getContextClassLoader();
+ Thread.currentThread().setContextClassLoader(null);
+ try {
+ return new OzoneConfiguration();
+ } finally {
+ Thread.currentThread().setContextClassLoader(contextClassLoader);
+ }
+ }
+
+ public BasicRootedOzoneClientAdapterImpl(OzoneConfiguration conf)
+ throws IOException {
+ this(null, -1, conf);
+ }
+
+ public BasicRootedOzoneClientAdapterImpl(String omHost, int omPort,
+ Configuration hadoopConf) throws IOException {
+
+ ClassLoader contextClassLoader =
+ Thread.currentThread().getContextClassLoader();
+ Thread.currentThread().setContextClassLoader(null);
+
+ try {
+ OzoneConfiguration conf = OzoneConfiguration.of(hadoopConf);
+
+ if (omHost == null && OmUtils.isServiceIdsDefined(conf)) {
+ // When the host name or service id isn't given
+ // but ozone.om.service.ids is defined, declare failure.
+
+ // This is a safety precaution that prevents the client from
+ // accidentally failing over to an unintended OM.
+ throw new IllegalArgumentException("Service ID or host name must not"
+ + " be omitted when ozone.om.service.ids is defined.");
+ }
+
+ if (omPort != -1) {
+ // When the port number is specified, perform the following check
+ if (OmUtils.isOmHAServiceId(conf, omHost)) {
+ // If omHost is a service id, it shouldn't use a port
+ throw new IllegalArgumentException("Port " + omPort +
+ " specified in URI but host '" + omHost + "' is a "
+ + "logical (HA) OzoneManager and does not use port information.");
+ }
+ } else {
+ // When port number is not specified, read it from config
+ omPort = OmUtils.getOmRpcPort(conf);
+ }
+
+ SecurityConfig secConfig = new SecurityConfig(conf);
+
+ if (secConfig.isSecurityEnabled()) {
+ this.securityEnabled = true;
+ }
+
+ String replicationTypeConf =
+ conf.get(OzoneConfigKeys.OZONE_REPLICATION_TYPE,
+ OzoneConfigKeys.OZONE_REPLICATION_TYPE_DEFAULT);
+
+ int replicationCountConf = conf.getInt(OzoneConfigKeys.OZONE_REPLICATION,
+ OzoneConfigKeys.OZONE_REPLICATION_DEFAULT);
+
+ if (OmUtils.isOmHAServiceId(conf, omHost)) {
+ // omHost is listed as one of the service ids in the config,
+ // thus we should treat omHost as omServiceId
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(omHost, conf);
+ } else if (StringUtils.isNotEmpty(omHost) && omPort != -1) {
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(omHost, omPort, conf);
+ } else {
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(conf);
+ }
+ objectStore = ozoneClient.getObjectStore();
+ proxy = objectStore.getClientProxy();
+ this.replicationType = ReplicationType.valueOf(replicationTypeConf);
+ this.replicationFactor = ReplicationFactor.valueOf(replicationCountConf);
+ this.configuredDnPort = conf.getInt(
+ OzoneConfigKeys.DFS_CONTAINER_IPC_PORT,
+ OzoneConfigKeys.DFS_CONTAINER_IPC_PORT_DEFAULT);
+ } finally {
+ Thread.currentThread().setContextClassLoader(contextClassLoader);
+ }
+ }
+
+ OzoneBucket getBucket(OFSPath ofsPath, boolean createIfNotExist)
+ throws IOException {
+
+ return getBucket(ofsPath.getVolumeName(), ofsPath.getBucketName(),
+ createIfNotExist);
+ }
+
+ /**
+ * Get OzoneBucket object to operate in.
+ * Optionally create volume and bucket if not found.
+ *
+ * @param createIfNotExist Set this to true if the caller is a write operation
+ * in order to create the volume and bucket.
+ * @throws IOException Exceptions other than OMException with result code
+ * VOLUME_NOT_FOUND or BUCKET_NOT_FOUND.
+ */
+ private OzoneBucket getBucket(String volumeStr, String bucketStr,
+ boolean createIfNotExist) throws IOException {
+
+ OzoneBucket bucket;
+ try {
+ bucket = proxy.getBucketDetails(volumeStr, bucketStr);
+ } catch (OMException ex) {
+ if (createIfNotExist) {
+ // Note: getBucketDetails always throws BUCKET_NOT_FOUND, even if
+ // the volume doesn't exist.
+ if (ex.getResult().equals(BUCKET_NOT_FOUND)) {
+ OzoneVolume volume;
+ try {
+ volume = proxy.getVolumeDetails(volumeStr);
+ } catch (OMException getVolEx) {
+ if (getVolEx.getResult().equals(VOLUME_NOT_FOUND)) {
+ // Volume doesn't exist. Create it
+ try {
+ objectStore.createVolume(volumeStr);
+ } catch (OMException newVolEx) {
+ // Ignore the case where another client created the volume
+ if (!newVolEx.getResult().equals(VOLUME_ALREADY_EXISTS)) {
+ throw newVolEx;
+ }
+ }
+ } else {
+ throw getVolEx;
+ }
+ // Try get volume again
+ volume = proxy.getVolumeDetails(volumeStr);
+ }
+ // Create the bucket
+ try {
+ volume.createBucket(bucketStr);
+ } catch (OMException newBucEx) {
+ // Ignore the case where another client created the bucket
+ if (!newBucEx.getResult().equals(BUCKET_ALREADY_EXISTS)) {
+ throw newBucEx;
+ }
+ }
+ }
+ // Try get bucket again
+ bucket = proxy.getBucketDetails(volumeStr, bucketStr);
+ } else {
+ throw ex;
+ }
+ }
+
+ return bucket;
+ }
+
+ @Override
+ public void close() throws IOException {
+ ozoneClient.close();
+ }
+
+ @Override
+ public InputStream readFile(String pathStr) throws IOException {
+ incrementCounter(Statistic.OBJECTS_READ);
+ OFSPath ofsPath = new OFSPath(pathStr);
+ String key = ofsPath.getKeyName();
+ try {
+ OzoneBucket bucket = getBucket(ofsPath, false);
+ return bucket.readFile(key).getInputStream();
+ } catch (OMException ex) {
+ if (ex.getResult() == OMException.ResultCodes.FILE_NOT_FOUND
+ || ex.getResult() == OMException.ResultCodes.NOT_A_FILE) {
+ throw new FileNotFoundException(
+ ex.getResult().name() + ": " + ex.getMessage());
+ } else {
+ throw ex;
+ }
+ }
+ }
+
+ protected void incrementCounter(Statistic objectsRead) {
+ //noop: Use OzoneClientAdapterImpl which supports statistics.
+ }
+
+ @Override
+ public OzoneFSOutputStream createFile(String pathStr, boolean overWrite,
+ boolean recursive) throws IOException {
+ incrementCounter(Statistic.OBJECTS_CREATED);
+ OFSPath ofsPath = new OFSPath(pathStr);
+ String key = ofsPath.getKeyName();
+ try {
+ OzoneBucket bucket = getBucket(ofsPath, false);
+ OzoneOutputStream ozoneOutputStream = bucket.createFile(
+ key, 0, replicationType, replicationFactor, overWrite, recursive);
+ return new OzoneFSOutputStream(ozoneOutputStream.getOutputStream());
+ } catch (OMException ex) {
+ if (ex.getResult() == OMException.ResultCodes.FILE_ALREADY_EXISTS
+ || ex.getResult() == OMException.ResultCodes.NOT_A_FILE) {
+ throw new FileAlreadyExistsException(
+ ex.getResult().name() + ": " + ex.getMessage());
+ } else {
+ throw ex;
+ }
+ }
+ }
+
+ @Override
+ public void renamePath(String path, String newPath) throws IOException {
+ incrementCounter(Statistic.OBJECTS_RENAMED);
+ OFSPath ofsPath = new OFSPath(path);
+ OFSPath ofsNewPath = new OFSPath(newPath);
+
+ // Check path and newPathName are in the same volume and same bucket.
+ // This should have been checked in BasicRootedOzoneFileSystem#rename
+ // already via regular call path unless bypassed.
+ if (!ofsPath.isInSameBucketAs(ofsNewPath)) {
+ throw new IOException("Cannot rename a key to a different bucket");
+ }
+
+ OzoneBucket bucket = getBucket(ofsPath, false);
+ String key = ofsPath.getKeyName();
+ String newKey = ofsNewPath.getKeyName();
+ bucket.renameKey(key, newKey);
+ }
+
+ /**
+ * Package-private helper function to reduce calls to getBucket().
+ * @param bucket Bucket to operate in.
+ * @param path Existing key path.
+ * @param newPath New key path.
+ * @throws IOException IOException from bucket.renameKey().
+ */
+ void renamePath(OzoneBucket bucket, String path, String newPath)
+ throws IOException {
+ incrementCounter(Statistic.OBJECTS_RENAMED);
+ OFSPath ofsPath = new OFSPath(path);
+ OFSPath ofsNewPath = new OFSPath(newPath);
+ // No same-bucket policy check here since this call path is controlled
+ String key = ofsPath.getKeyName();
+ String newKey = ofsNewPath.getKeyName();
+ bucket.renameKey(key, newKey);
+ }
+
+ /**
+ * Helper method to create an directory specified by key name in bucket.
+ *
+ * @param pathStr path to be created as directory
+ * @return true if the key is created, false otherwise
+ */
+ @Override
+ public boolean createDirectory(String pathStr) throws IOException {
+ LOG.trace("creating dir for path: {}", pathStr);
+ incrementCounter(Statistic.OBJECTS_CREATED);
+ OFSPath ofsPath = new OFSPath(pathStr);
+ // Volume name unspecified, return failure
+ if (ofsPath.getVolumeName().length() == 0) {
+ return false;
+ }
+ // Handle where only volume is specified in pathStr
+ if (ofsPath.getBucketName().length() == 0) {
+ objectStore.createVolume(ofsPath.getVolumeName());
+ return true;
+ }
+ String keyStr = ofsPath.getKeyName();
Review comment:
Actually here we need to put the `keyStr.length()` check after `getBucket`, since `getBucket` need to create volume and bucket if they don't exist. If we check it before `try` clause it wouldn't create the volume and bucket then.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] smengcl commented on a change in pull request #415:
HDDS-2840. Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
smengcl commented on a change in pull request #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#discussion_r370422075
##########
File path: hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
##########
@@ -0,0 +1,395 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.TestDataUtil;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClientException;
+import org.apache.hadoop.ozone.client.OzoneKeyDetails;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.Set;
+import java.util.TreeSet;
+
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Ozone file system tests that are not covered by contract tests.
+ */
+public class TestRootedOzoneFileSystem {
+
+ @Rule
+ public Timeout globalTimeout = new Timeout(300_000);
+
+ private static MiniOzoneCluster cluster = null;
+
+ private static FileSystem fs;
+ private static RootedOzoneFileSystem ofs;
+
+ private static ObjectStore objectStore;
+
+ private String volumeName;
+ private String bucketName;
+
+ private String rootPath;
+
+ @Before
+ public void init() throws Exception {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ cluster = MiniOzoneCluster.newBuilder(conf)
+ .setNumDatanodes(3)
+ .build();
+ cluster.waitForClusterToBeReady();
+ objectStore = cluster.getClient().getObjectStore();
+
+ // create a volume and a bucket to be used by OFileSystem
+ OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(cluster);
+ volumeName = bucket.getVolumeName();
+ bucketName = bucket.getName();
+
+ // For now:
+ rootPath = String.format("%s://%s/", OzoneConsts.OZONE_OFS_URI_SCHEME,
+ conf.get(OZONE_OM_ADDRESS_KEY));
+
+ // Set the fs.defaultFS and start the filesystem
+ conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, rootPath);
+ // TODO: FileSystem#loadFileSystems is not loading ofs:// class by default
+ // hence this workaround. Might need to add some config in hadoop source.
+ conf.set("fs.ofs.impl", "org.apache.hadoop.fs.ozone.RootedOzoneFileSystem");
+ fs = FileSystem.get(conf);
+ ofs = (RootedOzoneFileSystem) fs;
+ }
+
+ @After
+ public void teardown() throws IOException {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ IOUtils.closeQuietly(fs);
+ }
+
+ @Test
+ public void testOzoneFsServiceLoader() throws IOException {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ // TODO: Address this properly.
+ conf.set("fs.ofs.impl", "org.apache.hadoop.fs.ozone.RootedOzoneFileSystem");
+ assertEquals(
+ FileSystem.getFileSystemClass(OzoneConsts.OZONE_OFS_URI_SCHEME,
+ conf), RootedOzoneFileSystem.class);
+ }
+
+ @Test
+ public void testCreateDoesNotAddParentDirKeys() throws Exception {
+ Path rootBucket = new Path("/" + volumeName + "/" + bucketName);
Review comment:
Yep this looks duplicated for those **existing** test cases modified from O3FS tests.
The point is there will definitely be more OFS-specific test cases (like `testMkdirOnNonExistentVolumeBucket()`) introduced into the this test class which won't use `rootBucket`.
I would make this class global and rename it.
Done in 7a3de51e6e23abba3e91971f80e8666c1fc6c108.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] smengcl commented on issue #415: HDDS-2840.
Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
smengcl commented on issue #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#issuecomment-578954095
When writing unit tests for `OFSPath`, I realized that there is one corner case for `mkdir` that I didn't handle properly: 'mkdir /volume1'. Will push new commit to fix this in a moment.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] smengcl commented on a change in pull request #415:
HDDS-2840. Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
smengcl commented on a change in pull request #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#discussion_r372062918
##########
File path: hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
##########
@@ -0,0 +1,694 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.ozone;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.key.KeyProvider;
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdds.client.ReplicationFactor;
+import org.apache.hadoop.hdds.client.ReplicationType;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ozone.OmUtils;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.client.OzoneKey;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
+import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
+import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenRenewer;
+
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .BUCKET_ALREADY_EXISTS;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .VOLUME_ALREADY_EXISTS;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .VOLUME_NOT_FOUND;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .BUCKET_NOT_FOUND;
+
+/**
+ * Basic Implementation of the OzoneFileSystem calls.
+ * <p>
+ * This is the minimal version which doesn't include any statistics.
+ * <p>
+ * For full featured version use OzoneClientAdapterImpl.
+ */
+public class BasicRootedOzoneClientAdapterImpl
+ implements RootedOzoneClientAdapter {
+
+ static final Logger LOG =
+ LoggerFactory.getLogger(BasicRootedOzoneClientAdapterImpl.class);
+
+ private OzoneClient ozoneClient;
+ private ClientProtocol proxy;
+ private ObjectStore objectStore;
+ private ReplicationType replicationType;
+ private ReplicationFactor replicationFactor;
+ private boolean securityEnabled;
+ private int configuredDnPort;
+
+ /**
+ * Create new OzoneClientAdapter implementation.
+ *
+ * @throws IOException In case of a problem.
+ */
+ public BasicRootedOzoneClientAdapterImpl() throws IOException {
+ this(createConf());
+ }
+
+ private static OzoneConfiguration createConf() {
+ ClassLoader contextClassLoader =
+ Thread.currentThread().getContextClassLoader();
+ Thread.currentThread().setContextClassLoader(null);
+ try {
+ return new OzoneConfiguration();
+ } finally {
+ Thread.currentThread().setContextClassLoader(contextClassLoader);
+ }
+ }
+
+ public BasicRootedOzoneClientAdapterImpl(OzoneConfiguration conf)
+ throws IOException {
+ this(null, -1, conf);
+ }
+
+ public BasicRootedOzoneClientAdapterImpl(String omHost, int omPort,
+ Configuration hadoopConf) throws IOException {
+
+ ClassLoader contextClassLoader =
+ Thread.currentThread().getContextClassLoader();
+ Thread.currentThread().setContextClassLoader(null);
+
+ try {
+ OzoneConfiguration conf = OzoneConfiguration.of(hadoopConf);
+
+ if (omHost == null && OmUtils.isServiceIdsDefined(conf)) {
+ // When the host name or service id isn't given
+ // but ozone.om.service.ids is defined, declare failure.
+
+ // This is a safety precaution that prevents the client from
+ // accidentally failing over to an unintended OM.
+ throw new IllegalArgumentException("Service ID or host name must not"
+ + " be omitted when ozone.om.service.ids is defined.");
+ }
+
+ if (omPort != -1) {
+ // When the port number is specified, perform the following check
+ if (OmUtils.isOmHAServiceId(conf, omHost)) {
+ // If omHost is a service id, it shouldn't use a port
+ throw new IllegalArgumentException("Port " + omPort +
+ " specified in URI but host '" + omHost + "' is a "
+ + "logical (HA) OzoneManager and does not use port information.");
+ }
+ } else {
+ // When port number is not specified, read it from config
+ omPort = OmUtils.getOmRpcPort(conf);
+ }
+
+ SecurityConfig secConfig = new SecurityConfig(conf);
+
+ if (secConfig.isSecurityEnabled()) {
+ this.securityEnabled = true;
+ }
+
+ String replicationTypeConf =
+ conf.get(OzoneConfigKeys.OZONE_REPLICATION_TYPE,
+ OzoneConfigKeys.OZONE_REPLICATION_TYPE_DEFAULT);
+
+ int replicationCountConf = conf.getInt(OzoneConfigKeys.OZONE_REPLICATION,
+ OzoneConfigKeys.OZONE_REPLICATION_DEFAULT);
+
+ if (OmUtils.isOmHAServiceId(conf, omHost)) {
+ // omHost is listed as one of the service ids in the config,
+ // thus we should treat omHost as omServiceId
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(omHost, conf);
+ } else if (StringUtils.isNotEmpty(omHost) && omPort != -1) {
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(omHost, omPort, conf);
+ } else {
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(conf);
+ }
+ objectStore = ozoneClient.getObjectStore();
+ proxy = objectStore.getClientProxy();
+ this.replicationType = ReplicationType.valueOf(replicationTypeConf);
+ this.replicationFactor = ReplicationFactor.valueOf(replicationCountConf);
+ this.configuredDnPort = conf.getInt(
+ OzoneConfigKeys.DFS_CONTAINER_IPC_PORT,
+ OzoneConfigKeys.DFS_CONTAINER_IPC_PORT_DEFAULT);
+ } finally {
+ Thread.currentThread().setContextClassLoader(contextClassLoader);
+ }
+ }
+
+ OzoneBucket getBucket(OFSPath ofsPath, boolean createIfNotExist)
+ throws IOException {
+
+ return getBucket(ofsPath.getVolumeName(), ofsPath.getBucketName(),
+ createIfNotExist);
+ }
+
+ /**
+ * Get OzoneBucket object to operate in.
+ * Optionally create volume and bucket if not found.
+ *
+ * @param createIfNotExist Set this to true if the caller is a write operation
+ * in order to create the volume and bucket.
+ * @throws IOException Exceptions other than OMException with result code
+ * VOLUME_NOT_FOUND or BUCKET_NOT_FOUND.
+ */
+ private OzoneBucket getBucket(String volumeStr, String bucketStr,
+ boolean createIfNotExist) throws IOException {
+
+ OzoneBucket bucket;
+ try {
+ bucket = proxy.getBucketDetails(volumeStr, bucketStr);
+ } catch (OMException ex) {
+ if (createIfNotExist) {
+ // Note: getBucketDetails always throws BUCKET_NOT_FOUND, even if
+ // the volume doesn't exist.
+ if (ex.getResult().equals(BUCKET_NOT_FOUND)) {
+ OzoneVolume volume;
+ try {
+ volume = proxy.getVolumeDetails(volumeStr);
Review comment:
@xiaoyuyao Actually this part was coded previously that it checks if the volume exists first. Then @bharatviswa504 argued that this way we would always have at least two round trips for each `getBucket()` (first `getVolumeDetails`, then `getBucketDetails`) regardless of whether the bucket already exists. This change is made in https://github.com/apache/hadoop-ozone/commit/d2a01d19f0cfa2cdce5e252be2dd33fcb0d4fb6a#diff-1982358804d0a7b88f575ecd1e3b3013R202
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] smengcl commented on a change in pull request #415:
HDDS-2840. Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
smengcl commented on a change in pull request #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#discussion_r373141001
##########
File path: hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
##########
@@ -0,0 +1,482 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.TestDataUtil;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClientException;
+import org.apache.hadoop.ozone.client.OzoneKeyDetails;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.Set;
+import java.util.TreeSet;
+
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/**
+ * Ozone file system tests that are not covered by contract tests.
+ *
+ * TODO: Refactor this and TestOzoneFileSystem to eliminate most
+ * code duplication.
+ */
+public class TestRootedOzoneFileSystem {
+
+ @Rule
+ public Timeout globalTimeout = new Timeout(300_000);
+
+ private static MiniOzoneCluster cluster = null;
+
+ private static FileSystem fs;
+ private static RootedOzoneFileSystem ofs;
+
+ private static ObjectStore objectStore;
+
+ private String volumeName;
+ private String bucketName;
+
+ private String rootPath;
+
+ // Store path commonly used by tests that test functionality within a bucket
+ private String testBucketStr;
+ private Path testBucketPath;
+
+ @Before
+ public void init() throws Exception {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ cluster = MiniOzoneCluster.newBuilder(conf)
+ .setNumDatanodes(3)
+ .build();
+ cluster.waitForClusterToBeReady();
+ objectStore = cluster.getClient().getObjectStore();
+
+ // create a volume and a bucket to be used by RootedOzoneFileSystem (OFS)
+ OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(cluster);
+ volumeName = bucket.getVolumeName();
+ bucketName = bucket.getName();
+ testBucketStr = "/" + volumeName + "/" + bucketName;
+ testBucketPath = new Path(testBucketStr);
+
+ rootPath = String.format("%s://%s/", OzoneConsts.OZONE_OFS_URI_SCHEME,
+ conf.get(OZONE_OM_ADDRESS_KEY));
+
+ // Set the fs.defaultFS and start the filesystem
+ conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, rootPath);
+ // Note: FileSystem#loadFileSystems won't load OFS class due to META-INF
+ // hence this workaround.
+ conf.set("fs.ofs.impl", "org.apache.hadoop.fs.ozone.RootedOzoneFileSystem");
+ fs = FileSystem.get(conf);
+ ofs = (RootedOzoneFileSystem) fs;
+ }
+
+ @After
+ public void teardown() {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ IOUtils.closeQuietly(fs);
+ }
+
+ @Test
+ public void testOzoneFsServiceLoader() throws IOException {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ // Note: FileSystem#loadFileSystems won't load OFS class due to META-INF
+ // hence this workaround.
+ conf.set("fs.ofs.impl", "org.apache.hadoop.fs.ozone.RootedOzoneFileSystem");
+ assertEquals(
+ FileSystem.getFileSystemClass(OzoneConsts.OZONE_OFS_URI_SCHEME, conf),
+ RootedOzoneFileSystem.class);
+ }
+
+ @Test
+ public void testCreateDoesNotAddParentDirKeys() throws Exception {
+ Path grandparent = new Path(testBucketPath,
+ "testCreateDoesNotAddParentDirKeys");
+ Path parent = new Path(grandparent, "parent");
+ Path child = new Path(parent, "child");
+ ContractTestUtils.touch(fs, child);
+
+ OzoneKeyDetails key = getKey(child, false);
+ OFSPath childOFSPath = new OFSPath(child);
+ assertEquals(key.getName(), childOFSPath.getKeyName());
+
+ // Creating a child should not add parent keys to the bucket
+ try {
+ getKey(parent, true);
+ } catch (IOException ex) {
+ assertKeyNotFoundException(ex);
+ }
+
+ // List status on the parent should show the child file
+ assertEquals("List status of parent should include the 1 child file", 1L,
+ (long)fs.listStatus(parent).length);
+ assertTrue("Parent directory does not appear to be a directory",
+ fs.getFileStatus(parent).isDirectory());
+ }
+
+ @Test
+ public void testDeleteCreatesFakeParentDir() throws Exception {
+ Path grandparent = new Path(testBucketPath,
+ "testDeleteCreatesFakeParentDir");
+ Path parent = new Path(grandparent, "parent");
+ Path child = new Path(parent, "child");
+ ContractTestUtils.touch(fs, child);
+
+ // Verify that parent dir key does not exist
+ // Creating a child should not add parent keys to the bucket
+ try {
+ getKey(parent, true);
+ } catch (IOException ex) {
+ assertKeyNotFoundException(ex);
+ }
+
+ // Delete the child key
+ assertTrue(fs.delete(child, false));
+
+ // Deleting the only child should create the parent dir key if it does
+ // not exist
+ OFSPath parentOFSPath = new OFSPath(parent);
+ String parentKey = parentOFSPath.getKeyName() + "/";
+ OzoneKeyDetails parentKeyInfo = getKey(parent, true);
+ assertEquals(parentKey, parentKeyInfo.getName());
+
+ // Recursive delete with DeleteIterator
+ assertTrue(fs.delete(grandparent, true));
+ }
+
+ @Test
+ public void testListStatus() throws Exception {
+ Path parent = new Path(testBucketPath, "testListStatus");
+ Path file1 = new Path(parent, "key1");
+ Path file2 = new Path(parent, "key2");
+
+ FileStatus[] fileStatuses = ofs.listStatus(testBucketPath);
+ assertEquals("Should be empty", 0, fileStatuses.length);
+
+ ContractTestUtils.touch(fs, file1);
+ ContractTestUtils.touch(fs, file2);
+
+ fileStatuses = ofs.listStatus(testBucketPath);
+ assertEquals("Should have created parent",
+ 1, fileStatuses.length);
+ assertEquals("Parent path doesn't match",
+ fileStatuses[0].getPath().toUri().getPath(), parent.toString());
+
+ // ListStatus on a directory should return all subdirs along with
+ // files, even if there exists a file and sub-dir with the same name.
+ fileStatuses = ofs.listStatus(parent);
+ assertEquals("FileStatus did not return all children of the directory",
+ 2, fileStatuses.length);
+
+ // ListStatus should return only the immediate children of a directory.
+ Path file3 = new Path(parent, "dir1/key3");
+ Path file4 = new Path(parent, "dir1/key4");
+ ContractTestUtils.touch(fs, file3);
+ ContractTestUtils.touch(fs, file4);
+ fileStatuses = ofs.listStatus(parent);
+ assertEquals("FileStatus did not return all children of the directory",
+ 3, fileStatuses.length);
+ }
+
+ /**
+ * OFS: Helper function for tests. Return a volume name that doesn't exist.
+ */
+ private String getRandomNonExistVolumeName() throws Exception {
+ final int numDigit = 5;
+ long retriesLeft = Math.round(Math.pow(10, 5));
+ String name = null;
+ while (name == null && retriesLeft-- > 0) {
+ name = "volume-" + RandomStringUtils.randomNumeric(numDigit);
+ // Check volume existence.
+ Iterator<? extends OzoneVolume> iter =
+ objectStore.listVolumesByUser(null, name, null);
+ if (iter.hasNext()) {
+ // If there is a match, try again.
+ // Note that volume name prefix match doesn't equal volume existence
+ // but the check is sufficient for this test.
+ name = null;
+ }
+ }
+ if (retriesLeft <= 0) {
+ throw new Exception(
+ "Failed to generate random volume name that doesn't exist already.");
+ }
+ return name;
+ }
+
+ /**
+ * OFS: Test mkdir on volume, bucket and dir that doesn't exist.
+ */
+ @Test
+ public void testMkdirOnNonExistentVolumeBucketDir() throws Exception {
+ String volumeNameLocal = getRandomNonExistVolumeName();
+ String bucketNameLocal = "bucket-" + RandomStringUtils.randomNumeric(5);
+ Path root = new Path("/" + volumeNameLocal + "/" + bucketNameLocal);
+ Path dir1 = new Path(root, "dir1");
+ Path dir12 = new Path(dir1, "dir12");
+ Path dir2 = new Path(root, "dir2");
+ fs.mkdirs(dir12);
+ fs.mkdirs(dir2);
+
+ // Check volume and bucket existence, they should both be created.
+ OzoneVolume ozoneVolume = objectStore.getVolume(volumeNameLocal);
+ OzoneBucket ozoneBucket = ozoneVolume.getBucket(bucketNameLocal);
+ OFSPath ofsPathDir1 = new OFSPath(dir12);
+ String key = ofsPathDir1.getKeyName() + "/";
+ OzoneKeyDetails ozoneKeyDetails = ozoneBucket.getKey(key);
+ assertEquals(key, ozoneKeyDetails.getName());
+
+ // Verify that directories are created.
+ FileStatus[] fileStatuses = ofs.listStatus(root);
+ assertEquals(fileStatuses[0].getPath().toUri().getPath(), dir1.toString());
+ assertEquals(fileStatuses[1].getPath().toUri().getPath(), dir2.toString());
+
+ fileStatuses = ofs.listStatus(dir1);
+ assertEquals(fileStatuses[0].getPath().toUri().getPath(), dir12.toString());
+ fileStatuses = ofs.listStatus(dir12);
+ assertEquals(fileStatuses.length, 0);
+ fileStatuses = ofs.listStatus(dir2);
+ assertEquals(fileStatuses.length, 0);
+ }
+
+ /**
+ * OFS: Tests mkdir on a volume and bucket that doesn't exist.
+ */
+ @Test
+ public void testMkdirNonExistentVolumeBucket() throws Exception {
+ String volumeNameLocal = getRandomNonExistVolumeName();
+ String bucketNameLocal = "bucket-" + RandomStringUtils.randomNumeric(5);
+ Path newVolBucket = new Path(
+ "/" + volumeNameLocal + "/" + bucketNameLocal);
+ fs.mkdirs(newVolBucket);
+
+ // Verify with listVolumes and listBuckets
+ Iterator<? extends OzoneVolume> iterVol =
+ objectStore.listVolumesByUser(null, volumeNameLocal, null);
+ OzoneVolume ozoneVolume = iterVol.next();
+ assertNotNull(ozoneVolume);
+ assertEquals(volumeNameLocal, ozoneVolume.getName());
+
+ Iterator<? extends OzoneBucket> iterBuc =
+ ozoneVolume.listBuckets("bucket-");
+ OzoneBucket ozoneBucket = iterBuc.next();
+ assertNotNull(ozoneBucket);
+ assertEquals(bucketNameLocal, ozoneBucket.getName());
+
+ // TODO: Use listStatus to check volume and bucket creation in HDDS-2928.
+ }
+
+ /**
+ * OFS: Tests mkdir on a volume that doesn't exist.
+ */
+ @Test
+ public void testMkdirNonExistentVolume() throws Exception {
+ String volumeNameLocal = getRandomNonExistVolumeName();
+ Path newVolume = new Path("/" + volumeNameLocal);
+ fs.mkdirs(newVolume);
+
+ // Verify with listVolumes and listBuckets
+ Iterator<? extends OzoneVolume> iterVol =
+ objectStore.listVolumesByUser(null, volumeNameLocal, null);
+ OzoneVolume ozoneVolume = iterVol.next();
+ assertNotNull(ozoneVolume);
+ assertEquals(volumeNameLocal, ozoneVolume.getName());
+
+ // TODO: Use listStatus to check volume and bucket creation in HDDS-2928.
+ }
+
+ /**
+ * Tests listStatus operation in a bucket.
+ */
+ @Test
+ public void testListStatusOnRoot() throws Exception {
+ Path root = new Path("/" + volumeName + "/" + bucketName);
+ Path dir1 = new Path(root, "dir1");
+ Path dir12 = new Path(dir1, "dir12");
+ Path dir2 = new Path(root, "dir2");
+ fs.mkdirs(dir12);
+ fs.mkdirs(dir2);
+
+ // ListStatus on root should return dir1 (even though /dir1 key does not
+ // exist) and dir2 only. dir12 is not an immediate child of root and
+ // hence should not be listed.
+ FileStatus[] fileStatuses = ofs.listStatus(root);
+ assertEquals("FileStatus should return only the immediate children", 2,
+ fileStatuses.length);
+
+ // Verify that dir12 is not included in the result of the listStatus on root
+ String fileStatus1 = fileStatuses[0].getPath().toUri().getPath();
+ String fileStatus2 = fileStatuses[1].getPath().toUri().getPath();
+ assertFalse(fileStatus1.equals(dir12.toString()));
+ assertFalse(fileStatus2.equals(dir12.toString()));
+ }
+
+ /**
+ * Tests listStatus operation on root directory.
+ */
+ @Test
+ public void testListStatusOnLargeDirectory() throws Exception {
+ Path root = new Path("/" + volumeName + "/" + bucketName);
+ Set<String> paths = new TreeSet<>();
+ int numDirs = 5111;
Review comment:
I'm not sure why o3fs test chose this random number. But basically `numDirs` only need to be larger than `LISTING_PAGE_SIZE` so it tests pagination. And BTW I believe integration tests are disabled in PR by default now. (They are not run in PR checks.)
Reduced in `0d7100604334f06fe304b143a3b906c9cbfc88dd`
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] smengcl edited a comment on issue #415: HDDS-2840.
Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
smengcl edited a comment on issue #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#issuecomment-578954095
When writing unit tests for `OFSPath`, I realized that there is one corner case for `mkdir` that I didn't handle properly: `mkdir /volume1` <- we should expect this to create a volume `volume1` only. Will push new commit to fix this in a moment.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] xiaoyuyao commented on a change in pull request
#415: HDDS-2840. Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
xiaoyuyao commented on a change in pull request #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#discussion_r372028254
##########
File path: hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
##########
@@ -0,0 +1,694 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.ozone;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.key.KeyProvider;
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdds.client.ReplicationFactor;
+import org.apache.hadoop.hdds.client.ReplicationType;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ozone.OmUtils;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.client.OzoneKey;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
+import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
+import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenRenewer;
+
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .BUCKET_ALREADY_EXISTS;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .VOLUME_ALREADY_EXISTS;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .VOLUME_NOT_FOUND;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .BUCKET_NOT_FOUND;
+
+/**
+ * Basic Implementation of the OzoneFileSystem calls.
+ * <p>
+ * This is the minimal version which doesn't include any statistics.
+ * <p>
+ * For full featured version use OzoneClientAdapterImpl.
+ */
+public class BasicRootedOzoneClientAdapterImpl
+ implements RootedOzoneClientAdapter {
+
+ static final Logger LOG =
+ LoggerFactory.getLogger(BasicRootedOzoneClientAdapterImpl.class);
+
+ private OzoneClient ozoneClient;
+ private ClientProtocol proxy;
+ private ObjectStore objectStore;
+ private ReplicationType replicationType;
+ private ReplicationFactor replicationFactor;
+ private boolean securityEnabled;
+ private int configuredDnPort;
+
+ /**
+ * Create new OzoneClientAdapter implementation.
+ *
+ * @throws IOException In case of a problem.
+ */
+ public BasicRootedOzoneClientAdapterImpl() throws IOException {
+ this(createConf());
+ }
+
+ private static OzoneConfiguration createConf() {
+ ClassLoader contextClassLoader =
+ Thread.currentThread().getContextClassLoader();
+ Thread.currentThread().setContextClassLoader(null);
+ try {
+ return new OzoneConfiguration();
+ } finally {
+ Thread.currentThread().setContextClassLoader(contextClassLoader);
+ }
+ }
+
+ public BasicRootedOzoneClientAdapterImpl(OzoneConfiguration conf)
+ throws IOException {
+ this(null, -1, conf);
+ }
+
+ public BasicRootedOzoneClientAdapterImpl(String omHost, int omPort,
+ Configuration hadoopConf) throws IOException {
+
+ ClassLoader contextClassLoader =
+ Thread.currentThread().getContextClassLoader();
+ Thread.currentThread().setContextClassLoader(null);
+
+ try {
+ OzoneConfiguration conf = OzoneConfiguration.of(hadoopConf);
+
+ if (omHost == null && OmUtils.isServiceIdsDefined(conf)) {
+ // When the host name or service id isn't given
+ // but ozone.om.service.ids is defined, declare failure.
+
+ // This is a safety precaution that prevents the client from
+ // accidentally failing over to an unintended OM.
+ throw new IllegalArgumentException("Service ID or host name must not"
+ + " be omitted when ozone.om.service.ids is defined.");
+ }
+
+ if (omPort != -1) {
+ // When the port number is specified, perform the following check
+ if (OmUtils.isOmHAServiceId(conf, omHost)) {
+ // If omHost is a service id, it shouldn't use a port
+ throw new IllegalArgumentException("Port " + omPort +
+ " specified in URI but host '" + omHost + "' is a "
+ + "logical (HA) OzoneManager and does not use port information.");
+ }
+ } else {
+ // When port number is not specified, read it from config
+ omPort = OmUtils.getOmRpcPort(conf);
+ }
+
+ SecurityConfig secConfig = new SecurityConfig(conf);
+
+ if (secConfig.isSecurityEnabled()) {
+ this.securityEnabled = true;
+ }
+
+ String replicationTypeConf =
+ conf.get(OzoneConfigKeys.OZONE_REPLICATION_TYPE,
+ OzoneConfigKeys.OZONE_REPLICATION_TYPE_DEFAULT);
+
+ int replicationCountConf = conf.getInt(OzoneConfigKeys.OZONE_REPLICATION,
+ OzoneConfigKeys.OZONE_REPLICATION_DEFAULT);
+
+ if (OmUtils.isOmHAServiceId(conf, omHost)) {
+ // omHost is listed as one of the service ids in the config,
+ // thus we should treat omHost as omServiceId
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(omHost, conf);
+ } else if (StringUtils.isNotEmpty(omHost) && omPort != -1) {
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(omHost, omPort, conf);
+ } else {
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(conf);
+ }
+ objectStore = ozoneClient.getObjectStore();
+ proxy = objectStore.getClientProxy();
+ this.replicationType = ReplicationType.valueOf(replicationTypeConf);
+ this.replicationFactor = ReplicationFactor.valueOf(replicationCountConf);
+ this.configuredDnPort = conf.getInt(
+ OzoneConfigKeys.DFS_CONTAINER_IPC_PORT,
+ OzoneConfigKeys.DFS_CONTAINER_IPC_PORT_DEFAULT);
+ } finally {
+ Thread.currentThread().setContextClassLoader(contextClassLoader);
+ }
+ }
+
+ OzoneBucket getBucket(OFSPath ofsPath, boolean createIfNotExist)
+ throws IOException {
+
+ return getBucket(ofsPath.getVolumeName(), ofsPath.getBucketName(),
+ createIfNotExist);
+ }
+
+ /**
+ * Get OzoneBucket object to operate in.
+ * Optionally create volume and bucket if not found.
+ *
+ * @param createIfNotExist Set this to true if the caller is a write operation
+ * in order to create the volume and bucket.
+ * @throws IOException Exceptions other than OMException with result code
+ * VOLUME_NOT_FOUND or BUCKET_NOT_FOUND.
+ */
+ private OzoneBucket getBucket(String volumeStr, String bucketStr,
+ boolean createIfNotExist) throws IOException {
+
+ OzoneBucket bucket;
+ try {
+ bucket = proxy.getBucketDetails(volumeStr, bucketStr);
+ } catch (OMException ex) {
+ if (createIfNotExist) {
+ // Note: getBucketDetails always throws BUCKET_NOT_FOUND, even if
+ // the volume doesn't exist.
+ if (ex.getResult().equals(BUCKET_NOT_FOUND)) {
+ OzoneVolume volume;
+ try {
+ volume = proxy.getVolumeDetails(volumeStr);
+ } catch (OMException getVolEx) {
+ if (getVolEx.getResult().equals(VOLUME_NOT_FOUND)) {
+ // Volume doesn't exist. Create it
+ try {
+ objectStore.createVolume(volumeStr);
+ } catch (OMException newVolEx) {
+ // Ignore the case where another client created the volume
+ if (!newVolEx.getResult().equals(VOLUME_ALREADY_EXISTS)) {
+ throw newVolEx;
+ }
+ }
+ } else {
+ throw getVolEx;
+ }
+ // Try get volume again
+ volume = proxy.getVolumeDetails(volumeStr);
+ }
+ // Create the bucket
+ try {
+ volume.createBucket(bucketStr);
+ } catch (OMException newBucEx) {
+ // Ignore the case where another client created the bucket
+ if (!newBucEx.getResult().equals(BUCKET_ALREADY_EXISTS)) {
+ throw newBucEx;
+ }
+ }
+ }
+ // Try get bucket again
+ bucket = proxy.getBucketDetails(volumeStr, bucketStr);
+ } else {
+ throw ex;
+ }
+ }
+
+ return bucket;
+ }
+
+ @Override
+ public void close() throws IOException {
+ ozoneClient.close();
+ }
+
+ @Override
+ public InputStream readFile(String pathStr) throws IOException {
+ incrementCounter(Statistic.OBJECTS_READ);
+ OFSPath ofsPath = new OFSPath(pathStr);
+ String key = ofsPath.getKeyName();
+ try {
+ OzoneBucket bucket = getBucket(ofsPath, false);
+ return bucket.readFile(key).getInputStream();
+ } catch (OMException ex) {
+ if (ex.getResult() == OMException.ResultCodes.FILE_NOT_FOUND
+ || ex.getResult() == OMException.ResultCodes.NOT_A_FILE) {
+ throw new FileNotFoundException(
+ ex.getResult().name() + ": " + ex.getMessage());
+ } else {
+ throw ex;
+ }
+ }
+ }
+
+ protected void incrementCounter(Statistic objectsRead) {
+ //noop: Use OzoneClientAdapterImpl which supports statistics.
+ }
+
+ @Override
+ public OzoneFSOutputStream createFile(String pathStr, boolean overWrite,
+ boolean recursive) throws IOException {
+ incrementCounter(Statistic.OBJECTS_CREATED);
+ OFSPath ofsPath = new OFSPath(pathStr);
+ String key = ofsPath.getKeyName();
+ try {
+ OzoneBucket bucket = getBucket(ofsPath, false);
+ OzoneOutputStream ozoneOutputStream = bucket.createFile(
+ key, 0, replicationType, replicationFactor, overWrite, recursive);
+ return new OzoneFSOutputStream(ozoneOutputStream.getOutputStream());
+ } catch (OMException ex) {
+ if (ex.getResult() == OMException.ResultCodes.FILE_ALREADY_EXISTS
+ || ex.getResult() == OMException.ResultCodes.NOT_A_FILE) {
+ throw new FileAlreadyExistsException(
+ ex.getResult().name() + ": " + ex.getMessage());
+ } else {
+ throw ex;
+ }
+ }
+ }
+
+ @Override
+ public void renamePath(String path, String newPath) throws IOException {
+ incrementCounter(Statistic.OBJECTS_RENAMED);
+ OFSPath ofsPath = new OFSPath(path);
+ OFSPath ofsNewPath = new OFSPath(newPath);
+
+ // Check path and newPathName are in the same volume and same bucket.
+ // This should have been checked in BasicRootedOzoneFileSystem#rename
+ // already via regular call path unless bypassed.
+ if (!ofsPath.isInSameBucketAs(ofsNewPath)) {
+ throw new IOException("Cannot rename a key to a different bucket");
+ }
+
+ OzoneBucket bucket = getBucket(ofsPath, false);
+ String key = ofsPath.getKeyName();
+ String newKey = ofsNewPath.getKeyName();
+ bucket.renameKey(key, newKey);
+ }
+
+ /**
+ * Package-private helper function to reduce calls to getBucket().
+ * @param bucket Bucket to operate in.
+ * @param path Existing key path.
+ * @param newPath New key path.
+ * @throws IOException IOException from bucket.renameKey().
+ */
+ void renamePath(OzoneBucket bucket, String path, String newPath)
+ throws IOException {
+ incrementCounter(Statistic.OBJECTS_RENAMED);
+ OFSPath ofsPath = new OFSPath(path);
+ OFSPath ofsNewPath = new OFSPath(newPath);
+ // No same-bucket policy check here since this call path is controlled
+ String key = ofsPath.getKeyName();
+ String newKey = ofsNewPath.getKeyName();
+ bucket.renameKey(key, newKey);
+ }
+
+ /**
+ * Helper method to create an directory specified by key name in bucket.
+ *
+ * @param pathStr path to be created as directory
+ * @return true if the key is created, false otherwise
+ */
+ @Override
+ public boolean createDirectory(String pathStr) throws IOException {
+ LOG.trace("creating dir for path: {}", pathStr);
+ incrementCounter(Statistic.OBJECTS_CREATED);
+ OFSPath ofsPath = new OFSPath(pathStr);
+ // Volume name unspecified, return failure
+ if (ofsPath.getVolumeName().length() == 0) {
+ return false;
+ }
+ // Handle where only volume is specified in pathStr
+ if (ofsPath.getBucketName().length() == 0) {
+ objectStore.createVolume(ofsPath.getVolumeName());
+ return true;
+ }
+ String keyStr = ofsPath.getKeyName();
+ try {
+ OzoneBucket bucket = getBucket(ofsPath, true);
+ // if keyStr is empty, it indicates that only volume or volume+bucket is
+ // given in pathStr, so getBucket() above should've handled the creation
+ // of volume/bucket already.
+ if (keyStr != null && keyStr.length() > 0) {
+ bucket.createDirectory(keyStr);
+ }
+ } catch (OMException e) {
+ if (e.getResult() == OMException.ResultCodes.FILE_ALREADY_EXISTS) {
+ throw new FileAlreadyExistsException(e.getMessage());
+ }
+ throw e;
+ }
+ return true;
+ }
+
+ /**
+ * Helper method to delete an object specified by key name in bucket.
+ *
+ * @param path path to a key to be deleted
+ * @return true if the key is deleted, false otherwise
+ */
+ @Override
+ public boolean deleteObject(String path) {
+ LOG.trace("issuing delete for path to key: {}", path);
+ OFSPath ofsPath = new OFSPath(path);
+ String keyName = ofsPath.getKeyName();
Review comment:
check keyName is empty?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] smengcl commented on a change in pull request #415:
HDDS-2840. Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
smengcl commented on a change in pull request #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#discussion_r372555375
##########
File path: hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
##########
@@ -0,0 +1,694 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.ozone;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.key.KeyProvider;
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdds.client.ReplicationFactor;
+import org.apache.hadoop.hdds.client.ReplicationType;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ozone.OmUtils;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.client.OzoneKey;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
+import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
+import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenRenewer;
+
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .BUCKET_ALREADY_EXISTS;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .VOLUME_ALREADY_EXISTS;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .VOLUME_NOT_FOUND;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .BUCKET_NOT_FOUND;
+
+/**
+ * Basic Implementation of the OzoneFileSystem calls.
+ * <p>
+ * This is the minimal version which doesn't include any statistics.
+ * <p>
+ * For full featured version use OzoneClientAdapterImpl.
+ */
+public class BasicRootedOzoneClientAdapterImpl
+ implements RootedOzoneClientAdapter {
+
+ static final Logger LOG =
+ LoggerFactory.getLogger(BasicRootedOzoneClientAdapterImpl.class);
+
+ private OzoneClient ozoneClient;
+ private ClientProtocol proxy;
+ private ObjectStore objectStore;
+ private ReplicationType replicationType;
+ private ReplicationFactor replicationFactor;
+ private boolean securityEnabled;
+ private int configuredDnPort;
+
+ /**
+ * Create new OzoneClientAdapter implementation.
+ *
+ * @throws IOException In case of a problem.
+ */
+ public BasicRootedOzoneClientAdapterImpl() throws IOException {
+ this(createConf());
+ }
+
+ private static OzoneConfiguration createConf() {
+ ClassLoader contextClassLoader =
+ Thread.currentThread().getContextClassLoader();
+ Thread.currentThread().setContextClassLoader(null);
+ try {
+ return new OzoneConfiguration();
+ } finally {
+ Thread.currentThread().setContextClassLoader(contextClassLoader);
+ }
+ }
+
+ public BasicRootedOzoneClientAdapterImpl(OzoneConfiguration conf)
+ throws IOException {
+ this(null, -1, conf);
+ }
+
+ public BasicRootedOzoneClientAdapterImpl(String omHost, int omPort,
+ Configuration hadoopConf) throws IOException {
+
+ ClassLoader contextClassLoader =
+ Thread.currentThread().getContextClassLoader();
+ Thread.currentThread().setContextClassLoader(null);
+
+ try {
+ OzoneConfiguration conf = OzoneConfiguration.of(hadoopConf);
+
+ if (omHost == null && OmUtils.isServiceIdsDefined(conf)) {
+ // When the host name or service id isn't given
+ // but ozone.om.service.ids is defined, declare failure.
+
+ // This is a safety precaution that prevents the client from
+ // accidentally failing over to an unintended OM.
+ throw new IllegalArgumentException("Service ID or host name must not"
+ + " be omitted when ozone.om.service.ids is defined.");
+ }
+
+ if (omPort != -1) {
+ // When the port number is specified, perform the following check
+ if (OmUtils.isOmHAServiceId(conf, omHost)) {
+ // If omHost is a service id, it shouldn't use a port
+ throw new IllegalArgumentException("Port " + omPort +
+ " specified in URI but host '" + omHost + "' is a "
+ + "logical (HA) OzoneManager and does not use port information.");
+ }
+ } else {
+ // When port number is not specified, read it from config
+ omPort = OmUtils.getOmRpcPort(conf);
+ }
+
+ SecurityConfig secConfig = new SecurityConfig(conf);
+
+ if (secConfig.isSecurityEnabled()) {
+ this.securityEnabled = true;
+ }
+
+ String replicationTypeConf =
+ conf.get(OzoneConfigKeys.OZONE_REPLICATION_TYPE,
+ OzoneConfigKeys.OZONE_REPLICATION_TYPE_DEFAULT);
+
+ int replicationCountConf = conf.getInt(OzoneConfigKeys.OZONE_REPLICATION,
+ OzoneConfigKeys.OZONE_REPLICATION_DEFAULT);
+
+ if (OmUtils.isOmHAServiceId(conf, omHost)) {
+ // omHost is listed as one of the service ids in the config,
+ // thus we should treat omHost as omServiceId
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(omHost, conf);
+ } else if (StringUtils.isNotEmpty(omHost) && omPort != -1) {
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(omHost, omPort, conf);
+ } else {
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(conf);
+ }
+ objectStore = ozoneClient.getObjectStore();
+ proxy = objectStore.getClientProxy();
+ this.replicationType = ReplicationType.valueOf(replicationTypeConf);
+ this.replicationFactor = ReplicationFactor.valueOf(replicationCountConf);
+ this.configuredDnPort = conf.getInt(
+ OzoneConfigKeys.DFS_CONTAINER_IPC_PORT,
+ OzoneConfigKeys.DFS_CONTAINER_IPC_PORT_DEFAULT);
+ } finally {
+ Thread.currentThread().setContextClassLoader(contextClassLoader);
+ }
+ }
+
+ OzoneBucket getBucket(OFSPath ofsPath, boolean createIfNotExist)
+ throws IOException {
+
+ return getBucket(ofsPath.getVolumeName(), ofsPath.getBucketName(),
+ createIfNotExist);
+ }
+
+ /**
+ * Get OzoneBucket object to operate in.
+ * Optionally create volume and bucket if not found.
+ *
+ * @param createIfNotExist Set this to true if the caller is a write operation
+ * in order to create the volume and bucket.
+ * @throws IOException Exceptions other than OMException with result code
+ * VOLUME_NOT_FOUND or BUCKET_NOT_FOUND.
+ */
+ private OzoneBucket getBucket(String volumeStr, String bucketStr,
+ boolean createIfNotExist) throws IOException {
+
+ OzoneBucket bucket;
+ try {
+ bucket = proxy.getBucketDetails(volumeStr, bucketStr);
+ } catch (OMException ex) {
+ if (createIfNotExist) {
+ // Note: getBucketDetails always throws BUCKET_NOT_FOUND, even if
+ // the volume doesn't exist.
+ if (ex.getResult().equals(BUCKET_NOT_FOUND)) {
+ OzoneVolume volume;
+ try {
+ volume = proxy.getVolumeDetails(volumeStr);
+ } catch (OMException getVolEx) {
+ if (getVolEx.getResult().equals(VOLUME_NOT_FOUND)) {
+ // Volume doesn't exist. Create it
+ try {
+ objectStore.createVolume(volumeStr);
+ } catch (OMException newVolEx) {
+ // Ignore the case where another client created the volume
+ if (!newVolEx.getResult().equals(VOLUME_ALREADY_EXISTS)) {
+ throw newVolEx;
+ }
+ }
+ } else {
+ throw getVolEx;
+ }
+ // Try get volume again
+ volume = proxy.getVolumeDetails(volumeStr);
+ }
+ // Create the bucket
+ try {
+ volume.createBucket(bucketStr);
+ } catch (OMException newBucEx) {
+ // Ignore the case where another client created the bucket
+ if (!newBucEx.getResult().equals(BUCKET_ALREADY_EXISTS)) {
+ throw newBucEx;
+ }
+ }
+ }
+ // Try get bucket again
+ bucket = proxy.getBucketDetails(volumeStr, bucketStr);
+ } else {
+ throw ex;
+ }
+ }
+
+ return bucket;
+ }
+
+ @Override
+ public void close() throws IOException {
+ ozoneClient.close();
+ }
+
+ @Override
+ public InputStream readFile(String pathStr) throws IOException {
+ incrementCounter(Statistic.OBJECTS_READ);
+ OFSPath ofsPath = new OFSPath(pathStr);
+ String key = ofsPath.getKeyName();
+ try {
+ OzoneBucket bucket = getBucket(ofsPath, false);
+ return bucket.readFile(key).getInputStream();
+ } catch (OMException ex) {
+ if (ex.getResult() == OMException.ResultCodes.FILE_NOT_FOUND
+ || ex.getResult() == OMException.ResultCodes.NOT_A_FILE) {
+ throw new FileNotFoundException(
+ ex.getResult().name() + ": " + ex.getMessage());
+ } else {
+ throw ex;
+ }
+ }
+ }
+
+ protected void incrementCounter(Statistic objectsRead) {
+ //noop: Use OzoneClientAdapterImpl which supports statistics.
+ }
+
+ @Override
+ public OzoneFSOutputStream createFile(String pathStr, boolean overWrite,
+ boolean recursive) throws IOException {
+ incrementCounter(Statistic.OBJECTS_CREATED);
+ OFSPath ofsPath = new OFSPath(pathStr);
+ String key = ofsPath.getKeyName();
+ try {
+ OzoneBucket bucket = getBucket(ofsPath, false);
+ OzoneOutputStream ozoneOutputStream = bucket.createFile(
+ key, 0, replicationType, replicationFactor, overWrite, recursive);
+ return new OzoneFSOutputStream(ozoneOutputStream.getOutputStream());
+ } catch (OMException ex) {
+ if (ex.getResult() == OMException.ResultCodes.FILE_ALREADY_EXISTS
+ || ex.getResult() == OMException.ResultCodes.NOT_A_FILE) {
+ throw new FileAlreadyExistsException(
+ ex.getResult().name() + ": " + ex.getMessage());
+ } else {
+ throw ex;
+ }
+ }
+ }
+
+ @Override
+ public void renamePath(String path, String newPath) throws IOException {
+ incrementCounter(Statistic.OBJECTS_RENAMED);
+ OFSPath ofsPath = new OFSPath(path);
+ OFSPath ofsNewPath = new OFSPath(newPath);
+
+ // Check path and newPathName are in the same volume and same bucket.
+ // This should have been checked in BasicRootedOzoneFileSystem#rename
+ // already via regular call path unless bypassed.
+ if (!ofsPath.isInSameBucketAs(ofsNewPath)) {
+ throw new IOException("Cannot rename a key to a different bucket");
+ }
+
+ OzoneBucket bucket = getBucket(ofsPath, false);
+ String key = ofsPath.getKeyName();
+ String newKey = ofsNewPath.getKeyName();
+ bucket.renameKey(key, newKey);
+ }
+
+ /**
+ * Package-private helper function to reduce calls to getBucket().
+ * @param bucket Bucket to operate in.
+ * @param path Existing key path.
+ * @param newPath New key path.
+ * @throws IOException IOException from bucket.renameKey().
+ */
+ void renamePath(OzoneBucket bucket, String path, String newPath)
+ throws IOException {
+ incrementCounter(Statistic.OBJECTS_RENAMED);
+ OFSPath ofsPath = new OFSPath(path);
+ OFSPath ofsNewPath = new OFSPath(newPath);
+ // No same-bucket policy check here since this call path is controlled
+ String key = ofsPath.getKeyName();
+ String newKey = ofsNewPath.getKeyName();
+ bucket.renameKey(key, newKey);
+ }
+
+ /**
+ * Helper method to create an directory specified by key name in bucket.
+ *
+ * @param pathStr path to be created as directory
+ * @return true if the key is created, false otherwise
+ */
+ @Override
+ public boolean createDirectory(String pathStr) throws IOException {
+ LOG.trace("creating dir for path: {}", pathStr);
+ incrementCounter(Statistic.OBJECTS_CREATED);
+ OFSPath ofsPath = new OFSPath(pathStr);
+ // Volume name unspecified, return failure
+ if (ofsPath.getVolumeName().length() == 0) {
+ return false;
+ }
+ // Handle where only volume is specified in pathStr
+ if (ofsPath.getBucketName().length() == 0) {
+ objectStore.createVolume(ofsPath.getVolumeName());
+ return true;
+ }
+ String keyStr = ofsPath.getKeyName();
+ try {
+ OzoneBucket bucket = getBucket(ofsPath, true);
+ // if keyStr is empty, it indicates that only volume or volume+bucket is
+ // given in pathStr, so getBucket() above should've handled the creation
+ // of volume/bucket already.
+ if (keyStr != null && keyStr.length() > 0) {
+ bucket.createDirectory(keyStr);
+ }
+ } catch (OMException e) {
+ if (e.getResult() == OMException.ResultCodes.FILE_ALREADY_EXISTS) {
+ throw new FileAlreadyExistsException(e.getMessage());
+ }
+ throw e;
+ }
+ return true;
+ }
+
+ /**
+ * Helper method to delete an object specified by key name in bucket.
+ *
+ * @param path path to a key to be deleted
+ * @return true if the key is deleted, false otherwise
+ */
+ @Override
+ public boolean deleteObject(String path) {
+ LOG.trace("issuing delete for path to key: {}", path);
+ OFSPath ofsPath = new OFSPath(path);
+ String keyName = ofsPath.getKeyName();
Review comment:
keyName check done in ac0cd77f9c22c19c7442f7e248ab401bbb9ffb73.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] xiaoyuyao commented on a change in pull request
#415: HDDS-2840. Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
xiaoyuyao commented on a change in pull request #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#discussion_r372028598
##########
File path: hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
##########
@@ -0,0 +1,694 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.ozone;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.key.KeyProvider;
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdds.client.ReplicationFactor;
+import org.apache.hadoop.hdds.client.ReplicationType;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ozone.OmUtils;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.client.OzoneKey;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
+import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
+import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenRenewer;
+
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .BUCKET_ALREADY_EXISTS;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .VOLUME_ALREADY_EXISTS;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .VOLUME_NOT_FOUND;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .BUCKET_NOT_FOUND;
+
+/**
+ * Basic Implementation of the OzoneFileSystem calls.
+ * <p>
+ * This is the minimal version which doesn't include any statistics.
+ * <p>
+ * For full featured version use OzoneClientAdapterImpl.
+ */
+public class BasicRootedOzoneClientAdapterImpl
+ implements RootedOzoneClientAdapter {
+
+ static final Logger LOG =
+ LoggerFactory.getLogger(BasicRootedOzoneClientAdapterImpl.class);
+
+ private OzoneClient ozoneClient;
+ private ClientProtocol proxy;
+ private ObjectStore objectStore;
+ private ReplicationType replicationType;
+ private ReplicationFactor replicationFactor;
+ private boolean securityEnabled;
+ private int configuredDnPort;
+
+ /**
+ * Create new OzoneClientAdapter implementation.
+ *
+ * @throws IOException In case of a problem.
+ */
+ public BasicRootedOzoneClientAdapterImpl() throws IOException {
+ this(createConf());
+ }
+
+ private static OzoneConfiguration createConf() {
+ ClassLoader contextClassLoader =
+ Thread.currentThread().getContextClassLoader();
+ Thread.currentThread().setContextClassLoader(null);
+ try {
+ return new OzoneConfiguration();
+ } finally {
+ Thread.currentThread().setContextClassLoader(contextClassLoader);
+ }
+ }
+
+ public BasicRootedOzoneClientAdapterImpl(OzoneConfiguration conf)
+ throws IOException {
+ this(null, -1, conf);
+ }
+
+ public BasicRootedOzoneClientAdapterImpl(String omHost, int omPort,
+ Configuration hadoopConf) throws IOException {
+
+ ClassLoader contextClassLoader =
+ Thread.currentThread().getContextClassLoader();
+ Thread.currentThread().setContextClassLoader(null);
+
+ try {
+ OzoneConfiguration conf = OzoneConfiguration.of(hadoopConf);
+
+ if (omHost == null && OmUtils.isServiceIdsDefined(conf)) {
+ // When the host name or service id isn't given
+ // but ozone.om.service.ids is defined, declare failure.
+
+ // This is a safety precaution that prevents the client from
+ // accidentally failing over to an unintended OM.
+ throw new IllegalArgumentException("Service ID or host name must not"
+ + " be omitted when ozone.om.service.ids is defined.");
+ }
+
+ if (omPort != -1) {
+ // When the port number is specified, perform the following check
+ if (OmUtils.isOmHAServiceId(conf, omHost)) {
+ // If omHost is a service id, it shouldn't use a port
+ throw new IllegalArgumentException("Port " + omPort +
+ " specified in URI but host '" + omHost + "' is a "
+ + "logical (HA) OzoneManager and does not use port information.");
+ }
+ } else {
+ // When port number is not specified, read it from config
+ omPort = OmUtils.getOmRpcPort(conf);
+ }
+
+ SecurityConfig secConfig = new SecurityConfig(conf);
+
+ if (secConfig.isSecurityEnabled()) {
+ this.securityEnabled = true;
+ }
+
+ String replicationTypeConf =
+ conf.get(OzoneConfigKeys.OZONE_REPLICATION_TYPE,
+ OzoneConfigKeys.OZONE_REPLICATION_TYPE_DEFAULT);
+
+ int replicationCountConf = conf.getInt(OzoneConfigKeys.OZONE_REPLICATION,
+ OzoneConfigKeys.OZONE_REPLICATION_DEFAULT);
+
+ if (OmUtils.isOmHAServiceId(conf, omHost)) {
+ // omHost is listed as one of the service ids in the config,
+ // thus we should treat omHost as omServiceId
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(omHost, conf);
+ } else if (StringUtils.isNotEmpty(omHost) && omPort != -1) {
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(omHost, omPort, conf);
+ } else {
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(conf);
+ }
+ objectStore = ozoneClient.getObjectStore();
+ proxy = objectStore.getClientProxy();
+ this.replicationType = ReplicationType.valueOf(replicationTypeConf);
+ this.replicationFactor = ReplicationFactor.valueOf(replicationCountConf);
+ this.configuredDnPort = conf.getInt(
+ OzoneConfigKeys.DFS_CONTAINER_IPC_PORT,
+ OzoneConfigKeys.DFS_CONTAINER_IPC_PORT_DEFAULT);
+ } finally {
+ Thread.currentThread().setContextClassLoader(contextClassLoader);
+ }
+ }
+
+ OzoneBucket getBucket(OFSPath ofsPath, boolean createIfNotExist)
+ throws IOException {
+
+ return getBucket(ofsPath.getVolumeName(), ofsPath.getBucketName(),
+ createIfNotExist);
+ }
+
+ /**
+ * Get OzoneBucket object to operate in.
+ * Optionally create volume and bucket if not found.
+ *
+ * @param createIfNotExist Set this to true if the caller is a write operation
+ * in order to create the volume and bucket.
+ * @throws IOException Exceptions other than OMException with result code
+ * VOLUME_NOT_FOUND or BUCKET_NOT_FOUND.
+ */
+ private OzoneBucket getBucket(String volumeStr, String bucketStr,
+ boolean createIfNotExist) throws IOException {
+
+ OzoneBucket bucket;
+ try {
+ bucket = proxy.getBucketDetails(volumeStr, bucketStr);
+ } catch (OMException ex) {
+ if (createIfNotExist) {
+ // Note: getBucketDetails always throws BUCKET_NOT_FOUND, even if
+ // the volume doesn't exist.
+ if (ex.getResult().equals(BUCKET_NOT_FOUND)) {
+ OzoneVolume volume;
+ try {
+ volume = proxy.getVolumeDetails(volumeStr);
+ } catch (OMException getVolEx) {
+ if (getVolEx.getResult().equals(VOLUME_NOT_FOUND)) {
+ // Volume doesn't exist. Create it
+ try {
+ objectStore.createVolume(volumeStr);
+ } catch (OMException newVolEx) {
+ // Ignore the case where another client created the volume
+ if (!newVolEx.getResult().equals(VOLUME_ALREADY_EXISTS)) {
+ throw newVolEx;
+ }
+ }
+ } else {
+ throw getVolEx;
+ }
+ // Try get volume again
+ volume = proxy.getVolumeDetails(volumeStr);
+ }
+ // Create the bucket
+ try {
+ volume.createBucket(bucketStr);
+ } catch (OMException newBucEx) {
+ // Ignore the case where another client created the bucket
+ if (!newBucEx.getResult().equals(BUCKET_ALREADY_EXISTS)) {
+ throw newBucEx;
+ }
+ }
+ }
+ // Try get bucket again
+ bucket = proxy.getBucketDetails(volumeStr, bucketStr);
+ } else {
+ throw ex;
+ }
+ }
+
+ return bucket;
+ }
+
+ @Override
+ public void close() throws IOException {
+ ozoneClient.close();
+ }
+
+ @Override
+ public InputStream readFile(String pathStr) throws IOException {
+ incrementCounter(Statistic.OBJECTS_READ);
+ OFSPath ofsPath = new OFSPath(pathStr);
+ String key = ofsPath.getKeyName();
+ try {
+ OzoneBucket bucket = getBucket(ofsPath, false);
+ return bucket.readFile(key).getInputStream();
+ } catch (OMException ex) {
+ if (ex.getResult() == OMException.ResultCodes.FILE_NOT_FOUND
+ || ex.getResult() == OMException.ResultCodes.NOT_A_FILE) {
+ throw new FileNotFoundException(
+ ex.getResult().name() + ": " + ex.getMessage());
+ } else {
+ throw ex;
+ }
+ }
+ }
+
+ protected void incrementCounter(Statistic objectsRead) {
+ //noop: Use OzoneClientAdapterImpl which supports statistics.
+ }
+
+ @Override
+ public OzoneFSOutputStream createFile(String pathStr, boolean overWrite,
+ boolean recursive) throws IOException {
+ incrementCounter(Statistic.OBJECTS_CREATED);
+ OFSPath ofsPath = new OFSPath(pathStr);
+ String key = ofsPath.getKeyName();
+ try {
+ OzoneBucket bucket = getBucket(ofsPath, false);
+ OzoneOutputStream ozoneOutputStream = bucket.createFile(
+ key, 0, replicationType, replicationFactor, overWrite, recursive);
+ return new OzoneFSOutputStream(ozoneOutputStream.getOutputStream());
+ } catch (OMException ex) {
+ if (ex.getResult() == OMException.ResultCodes.FILE_ALREADY_EXISTS
+ || ex.getResult() == OMException.ResultCodes.NOT_A_FILE) {
+ throw new FileAlreadyExistsException(
+ ex.getResult().name() + ": " + ex.getMessage());
+ } else {
+ throw ex;
+ }
+ }
+ }
+
+ @Override
+ public void renamePath(String path, String newPath) throws IOException {
+ incrementCounter(Statistic.OBJECTS_RENAMED);
+ OFSPath ofsPath = new OFSPath(path);
+ OFSPath ofsNewPath = new OFSPath(newPath);
+
+ // Check path and newPathName are in the same volume and same bucket.
+ // This should have been checked in BasicRootedOzoneFileSystem#rename
+ // already via regular call path unless bypassed.
+ if (!ofsPath.isInSameBucketAs(ofsNewPath)) {
+ throw new IOException("Cannot rename a key to a different bucket");
+ }
+
+ OzoneBucket bucket = getBucket(ofsPath, false);
+ String key = ofsPath.getKeyName();
+ String newKey = ofsNewPath.getKeyName();
+ bucket.renameKey(key, newKey);
+ }
+
+ /**
+ * Package-private helper function to reduce calls to getBucket().
+ * @param bucket Bucket to operate in.
+ * @param path Existing key path.
+ * @param newPath New key path.
+ * @throws IOException IOException from bucket.renameKey().
+ */
+ void renamePath(OzoneBucket bucket, String path, String newPath)
+ throws IOException {
+ incrementCounter(Statistic.OBJECTS_RENAMED);
+ OFSPath ofsPath = new OFSPath(path);
+ OFSPath ofsNewPath = new OFSPath(newPath);
+ // No same-bucket policy check here since this call path is controlled
+ String key = ofsPath.getKeyName();
+ String newKey = ofsNewPath.getKeyName();
+ bucket.renameKey(key, newKey);
+ }
+
+ /**
+ * Helper method to create an directory specified by key name in bucket.
+ *
+ * @param pathStr path to be created as directory
+ * @return true if the key is created, false otherwise
+ */
+ @Override
+ public boolean createDirectory(String pathStr) throws IOException {
+ LOG.trace("creating dir for path: {}", pathStr);
+ incrementCounter(Statistic.OBJECTS_CREATED);
+ OFSPath ofsPath = new OFSPath(pathStr);
+ // Volume name unspecified, return failure
+ if (ofsPath.getVolumeName().length() == 0) {
+ return false;
+ }
+ // Handle where only volume is specified in pathStr
+ if (ofsPath.getBucketName().length() == 0) {
+ objectStore.createVolume(ofsPath.getVolumeName());
+ return true;
+ }
+ String keyStr = ofsPath.getKeyName();
+ try {
+ OzoneBucket bucket = getBucket(ofsPath, true);
+ // if keyStr is empty, it indicates that only volume or volume+bucket is
+ // given in pathStr, so getBucket() above should've handled the creation
+ // of volume/bucket already.
+ if (keyStr != null && keyStr.length() > 0) {
+ bucket.createDirectory(keyStr);
+ }
+ } catch (OMException e) {
+ if (e.getResult() == OMException.ResultCodes.FILE_ALREADY_EXISTS) {
+ throw new FileAlreadyExistsException(e.getMessage());
+ }
+ throw e;
+ }
+ return true;
+ }
+
+ /**
+ * Helper method to delete an object specified by key name in bucket.
+ *
+ * @param path path to a key to be deleted
+ * @return true if the key is deleted, false otherwise
+ */
+ @Override
+ public boolean deleteObject(String path) {
+ LOG.trace("issuing delete for path to key: {}", path);
+ OFSPath ofsPath = new OFSPath(path);
+ String keyName = ofsPath.getKeyName();
Review comment:
Do we support delete volume/bucket via OFS?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] smengcl commented on a change in pull request #415:
HDDS-2840. Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
smengcl commented on a change in pull request #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#discussion_r372062918
##########
File path: hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
##########
@@ -0,0 +1,694 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.ozone;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.key.KeyProvider;
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdds.client.ReplicationFactor;
+import org.apache.hadoop.hdds.client.ReplicationType;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ozone.OmUtils;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.client.OzoneKey;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
+import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
+import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenRenewer;
+
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .BUCKET_ALREADY_EXISTS;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .VOLUME_ALREADY_EXISTS;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .VOLUME_NOT_FOUND;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .BUCKET_NOT_FOUND;
+
+/**
+ * Basic Implementation of the OzoneFileSystem calls.
+ * <p>
+ * This is the minimal version which doesn't include any statistics.
+ * <p>
+ * For full featured version use OzoneClientAdapterImpl.
+ */
+public class BasicRootedOzoneClientAdapterImpl
+ implements RootedOzoneClientAdapter {
+
+ static final Logger LOG =
+ LoggerFactory.getLogger(BasicRootedOzoneClientAdapterImpl.class);
+
+ private OzoneClient ozoneClient;
+ private ClientProtocol proxy;
+ private ObjectStore objectStore;
+ private ReplicationType replicationType;
+ private ReplicationFactor replicationFactor;
+ private boolean securityEnabled;
+ private int configuredDnPort;
+
+ /**
+ * Create new OzoneClientAdapter implementation.
+ *
+ * @throws IOException In case of a problem.
+ */
+ public BasicRootedOzoneClientAdapterImpl() throws IOException {
+ this(createConf());
+ }
+
+ private static OzoneConfiguration createConf() {
+ ClassLoader contextClassLoader =
+ Thread.currentThread().getContextClassLoader();
+ Thread.currentThread().setContextClassLoader(null);
+ try {
+ return new OzoneConfiguration();
+ } finally {
+ Thread.currentThread().setContextClassLoader(contextClassLoader);
+ }
+ }
+
+ public BasicRootedOzoneClientAdapterImpl(OzoneConfiguration conf)
+ throws IOException {
+ this(null, -1, conf);
+ }
+
+ public BasicRootedOzoneClientAdapterImpl(String omHost, int omPort,
+ Configuration hadoopConf) throws IOException {
+
+ ClassLoader contextClassLoader =
+ Thread.currentThread().getContextClassLoader();
+ Thread.currentThread().setContextClassLoader(null);
+
+ try {
+ OzoneConfiguration conf = OzoneConfiguration.of(hadoopConf);
+
+ if (omHost == null && OmUtils.isServiceIdsDefined(conf)) {
+ // When the host name or service id isn't given
+ // but ozone.om.service.ids is defined, declare failure.
+
+ // This is a safety precaution that prevents the client from
+ // accidentally failing over to an unintended OM.
+ throw new IllegalArgumentException("Service ID or host name must not"
+ + " be omitted when ozone.om.service.ids is defined.");
+ }
+
+ if (omPort != -1) {
+ // When the port number is specified, perform the following check
+ if (OmUtils.isOmHAServiceId(conf, omHost)) {
+ // If omHost is a service id, it shouldn't use a port
+ throw new IllegalArgumentException("Port " + omPort +
+ " specified in URI but host '" + omHost + "' is a "
+ + "logical (HA) OzoneManager and does not use port information.");
+ }
+ } else {
+ // When port number is not specified, read it from config
+ omPort = OmUtils.getOmRpcPort(conf);
+ }
+
+ SecurityConfig secConfig = new SecurityConfig(conf);
+
+ if (secConfig.isSecurityEnabled()) {
+ this.securityEnabled = true;
+ }
+
+ String replicationTypeConf =
+ conf.get(OzoneConfigKeys.OZONE_REPLICATION_TYPE,
+ OzoneConfigKeys.OZONE_REPLICATION_TYPE_DEFAULT);
+
+ int replicationCountConf = conf.getInt(OzoneConfigKeys.OZONE_REPLICATION,
+ OzoneConfigKeys.OZONE_REPLICATION_DEFAULT);
+
+ if (OmUtils.isOmHAServiceId(conf, omHost)) {
+ // omHost is listed as one of the service ids in the config,
+ // thus we should treat omHost as omServiceId
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(omHost, conf);
+ } else if (StringUtils.isNotEmpty(omHost) && omPort != -1) {
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(omHost, omPort, conf);
+ } else {
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(conf);
+ }
+ objectStore = ozoneClient.getObjectStore();
+ proxy = objectStore.getClientProxy();
+ this.replicationType = ReplicationType.valueOf(replicationTypeConf);
+ this.replicationFactor = ReplicationFactor.valueOf(replicationCountConf);
+ this.configuredDnPort = conf.getInt(
+ OzoneConfigKeys.DFS_CONTAINER_IPC_PORT,
+ OzoneConfigKeys.DFS_CONTAINER_IPC_PORT_DEFAULT);
+ } finally {
+ Thread.currentThread().setContextClassLoader(contextClassLoader);
+ }
+ }
+
+ OzoneBucket getBucket(OFSPath ofsPath, boolean createIfNotExist)
+ throws IOException {
+
+ return getBucket(ofsPath.getVolumeName(), ofsPath.getBucketName(),
+ createIfNotExist);
+ }
+
+ /**
+ * Get OzoneBucket object to operate in.
+ * Optionally create volume and bucket if not found.
+ *
+ * @param createIfNotExist Set this to true if the caller is a write operation
+ * in order to create the volume and bucket.
+ * @throws IOException Exceptions other than OMException with result code
+ * VOLUME_NOT_FOUND or BUCKET_NOT_FOUND.
+ */
+ private OzoneBucket getBucket(String volumeStr, String bucketStr,
+ boolean createIfNotExist) throws IOException {
+
+ OzoneBucket bucket;
+ try {
+ bucket = proxy.getBucketDetails(volumeStr, bucketStr);
+ } catch (OMException ex) {
+ if (createIfNotExist) {
+ // Note: getBucketDetails always throws BUCKET_NOT_FOUND, even if
+ // the volume doesn't exist.
+ if (ex.getResult().equals(BUCKET_NOT_FOUND)) {
+ OzoneVolume volume;
+ try {
+ volume = proxy.getVolumeDetails(volumeStr);
Review comment:
@xiaoyuyao Actually this part was coded previously that it checks if the volume exists first. Then @bharatviswa504 argued that this way we would always have at least two round trips for each `getBucket()` (first `getVolumeDetails`, then `getBucketDetails`) regardless of whether the bucket already exists. This change is made in d2a01d19f0cfa2cdce5e252be2dd33fcb0d4fb6a: https://github.com/apache/hadoop-ozone/commit/d2a01d19f0cfa2cdce5e252be2dd33fcb0d4fb6a#diff-1982358804d0a7b88f575ecd1e3b3013R202
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] xiaoyuyao commented on a change in pull request
#415: HDDS-2840. Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
xiaoyuyao commented on a change in pull request #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#discussion_r372105091
##########
File path: hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
##########
@@ -0,0 +1,694 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.ozone;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.key.KeyProvider;
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdds.client.ReplicationFactor;
+import org.apache.hadoop.hdds.client.ReplicationType;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ozone.OmUtils;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.client.OzoneKey;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
+import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
+import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenRenewer;
+
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .BUCKET_ALREADY_EXISTS;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .VOLUME_ALREADY_EXISTS;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .VOLUME_NOT_FOUND;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .BUCKET_NOT_FOUND;
+
+/**
+ * Basic Implementation of the OzoneFileSystem calls.
+ * <p>
+ * This is the minimal version which doesn't include any statistics.
+ * <p>
+ * For full featured version use OzoneClientAdapterImpl.
+ */
+public class BasicRootedOzoneClientAdapterImpl
+ implements RootedOzoneClientAdapter {
+
+ static final Logger LOG =
+ LoggerFactory.getLogger(BasicRootedOzoneClientAdapterImpl.class);
+
+ private OzoneClient ozoneClient;
+ private ClientProtocol proxy;
+ private ObjectStore objectStore;
+ private ReplicationType replicationType;
+ private ReplicationFactor replicationFactor;
+ private boolean securityEnabled;
+ private int configuredDnPort;
+
+ /**
+ * Create new OzoneClientAdapter implementation.
+ *
+ * @throws IOException In case of a problem.
+ */
+ public BasicRootedOzoneClientAdapterImpl() throws IOException {
+ this(createConf());
+ }
+
+ private static OzoneConfiguration createConf() {
+ ClassLoader contextClassLoader =
+ Thread.currentThread().getContextClassLoader();
+ Thread.currentThread().setContextClassLoader(null);
+ try {
+ return new OzoneConfiguration();
+ } finally {
+ Thread.currentThread().setContextClassLoader(contextClassLoader);
+ }
+ }
+
+ public BasicRootedOzoneClientAdapterImpl(OzoneConfiguration conf)
+ throws IOException {
+ this(null, -1, conf);
+ }
+
+ public BasicRootedOzoneClientAdapterImpl(String omHost, int omPort,
+ Configuration hadoopConf) throws IOException {
+
+ ClassLoader contextClassLoader =
+ Thread.currentThread().getContextClassLoader();
+ Thread.currentThread().setContextClassLoader(null);
+
+ try {
+ OzoneConfiguration conf = OzoneConfiguration.of(hadoopConf);
+
+ if (omHost == null && OmUtils.isServiceIdsDefined(conf)) {
+ // When the host name or service id isn't given
+ // but ozone.om.service.ids is defined, declare failure.
+
+ // This is a safety precaution that prevents the client from
+ // accidentally failing over to an unintended OM.
+ throw new IllegalArgumentException("Service ID or host name must not"
+ + " be omitted when ozone.om.service.ids is defined.");
+ }
+
+ if (omPort != -1) {
+ // When the port number is specified, perform the following check
+ if (OmUtils.isOmHAServiceId(conf, omHost)) {
+ // If omHost is a service id, it shouldn't use a port
+ throw new IllegalArgumentException("Port " + omPort +
+ " specified in URI but host '" + omHost + "' is a "
+ + "logical (HA) OzoneManager and does not use port information.");
+ }
+ } else {
+ // When port number is not specified, read it from config
+ omPort = OmUtils.getOmRpcPort(conf);
+ }
+
+ SecurityConfig secConfig = new SecurityConfig(conf);
+
+ if (secConfig.isSecurityEnabled()) {
+ this.securityEnabled = true;
+ }
+
+ String replicationTypeConf =
+ conf.get(OzoneConfigKeys.OZONE_REPLICATION_TYPE,
+ OzoneConfigKeys.OZONE_REPLICATION_TYPE_DEFAULT);
+
+ int replicationCountConf = conf.getInt(OzoneConfigKeys.OZONE_REPLICATION,
+ OzoneConfigKeys.OZONE_REPLICATION_DEFAULT);
+
+ if (OmUtils.isOmHAServiceId(conf, omHost)) {
+ // omHost is listed as one of the service ids in the config,
+ // thus we should treat omHost as omServiceId
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(omHost, conf);
+ } else if (StringUtils.isNotEmpty(omHost) && omPort != -1) {
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(omHost, omPort, conf);
+ } else {
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(conf);
+ }
+ objectStore = ozoneClient.getObjectStore();
+ proxy = objectStore.getClientProxy();
+ this.replicationType = ReplicationType.valueOf(replicationTypeConf);
+ this.replicationFactor = ReplicationFactor.valueOf(replicationCountConf);
+ this.configuredDnPort = conf.getInt(
+ OzoneConfigKeys.DFS_CONTAINER_IPC_PORT,
+ OzoneConfigKeys.DFS_CONTAINER_IPC_PORT_DEFAULT);
+ } finally {
+ Thread.currentThread().setContextClassLoader(contextClassLoader);
+ }
+ }
+
+ OzoneBucket getBucket(OFSPath ofsPath, boolean createIfNotExist)
+ throws IOException {
+
+ return getBucket(ofsPath.getVolumeName(), ofsPath.getBucketName(),
+ createIfNotExist);
+ }
+
+ /**
+ * Get OzoneBucket object to operate in.
+ * Optionally create volume and bucket if not found.
+ *
+ * @param createIfNotExist Set this to true if the caller is a write operation
+ * in order to create the volume and bucket.
+ * @throws IOException Exceptions other than OMException with result code
+ * VOLUME_NOT_FOUND or BUCKET_NOT_FOUND.
+ */
+ private OzoneBucket getBucket(String volumeStr, String bucketStr,
+ boolean createIfNotExist) throws IOException {
+
+ OzoneBucket bucket;
+ try {
+ bucket = proxy.getBucketDetails(volumeStr, bucketStr);
+ } catch (OMException ex) {
+ if (createIfNotExist) {
+ // Note: getBucketDetails always throws BUCKET_NOT_FOUND, even if
+ // the volume doesn't exist.
+ if (ex.getResult().equals(BUCKET_NOT_FOUND)) {
+ OzoneVolume volume;
+ try {
+ volume = proxy.getVolumeDetails(volumeStr);
+ } catch (OMException getVolEx) {
+ if (getVolEx.getResult().equals(VOLUME_NOT_FOUND)) {
+ // Volume doesn't exist. Create it
+ try {
+ objectStore.createVolume(volumeStr);
+ } catch (OMException newVolEx) {
+ // Ignore the case where another client created the volume
+ if (!newVolEx.getResult().equals(VOLUME_ALREADY_EXISTS)) {
+ throw newVolEx;
+ }
+ }
+ } else {
+ throw getVolEx;
+ }
+ // Try get volume again
+ volume = proxy.getVolumeDetails(volumeStr);
+ }
+ // Create the bucket
+ try {
+ volume.createBucket(bucketStr);
+ } catch (OMException newBucEx) {
+ // Ignore the case where another client created the bucket
+ if (!newBucEx.getResult().equals(BUCKET_ALREADY_EXISTS)) {
+ throw newBucEx;
+ }
+ }
+ }
+ // Try get bucket again
+ bucket = proxy.getBucketDetails(volumeStr, bucketStr);
+ } else {
+ throw ex;
+ }
+ }
+
+ return bucket;
+ }
+
+ @Override
+ public void close() throws IOException {
+ ozoneClient.close();
+ }
+
+ @Override
+ public InputStream readFile(String pathStr) throws IOException {
+ incrementCounter(Statistic.OBJECTS_READ);
+ OFSPath ofsPath = new OFSPath(pathStr);
+ String key = ofsPath.getKeyName();
+ try {
+ OzoneBucket bucket = getBucket(ofsPath, false);
+ return bucket.readFile(key).getInputStream();
+ } catch (OMException ex) {
+ if (ex.getResult() == OMException.ResultCodes.FILE_NOT_FOUND
+ || ex.getResult() == OMException.ResultCodes.NOT_A_FILE) {
+ throw new FileNotFoundException(
+ ex.getResult().name() + ": " + ex.getMessage());
+ } else {
+ throw ex;
+ }
+ }
+ }
+
+ protected void incrementCounter(Statistic objectsRead) {
+ //noop: Use OzoneClientAdapterImpl which supports statistics.
+ }
+
+ @Override
+ public OzoneFSOutputStream createFile(String pathStr, boolean overWrite,
+ boolean recursive) throws IOException {
+ incrementCounter(Statistic.OBJECTS_CREATED);
+ OFSPath ofsPath = new OFSPath(pathStr);
+ String key = ofsPath.getKeyName();
+ try {
+ OzoneBucket bucket = getBucket(ofsPath, false);
+ OzoneOutputStream ozoneOutputStream = bucket.createFile(
+ key, 0, replicationType, replicationFactor, overWrite, recursive);
+ return new OzoneFSOutputStream(ozoneOutputStream.getOutputStream());
+ } catch (OMException ex) {
+ if (ex.getResult() == OMException.ResultCodes.FILE_ALREADY_EXISTS
+ || ex.getResult() == OMException.ResultCodes.NOT_A_FILE) {
+ throw new FileAlreadyExistsException(
+ ex.getResult().name() + ": " + ex.getMessage());
+ } else {
+ throw ex;
+ }
+ }
+ }
+
+ @Override
+ public void renamePath(String path, String newPath) throws IOException {
Review comment:
Let's change renamePath to rename
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] smengcl commented on issue #415: HDDS-2840.
Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
smengcl commented on issue #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#issuecomment-575399081
The race condition issue should've be resolved in d2a01d19f0cfa2cdce5e252be2dd33fcb0d4fb6a.
All `TestRootedOzoneFileSystem` tests passed. Pending pr-checks.
Quick note: We might want to add some robot tests in another jira for the new mkdir behavior. For example, `mkdir -p` inside a volume/bucket that doesn't exist, where user has or doesn't have permission.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] hanishakoneru commented on a change in pull request
#415: HDDS-2840. Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
hanishakoneru commented on a change in pull request #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#discussion_r369330201
##########
File path: hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
##########
@@ -0,0 +1,805 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.ozone;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.CreateFlag;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.Progressable;
+import org.apache.http.client.utils.URIBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.EnumSet;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+import static org.apache.hadoop.fs.ozone.Constants.LISTING_PAGE_SIZE;
+import static org.apache.hadoop.fs.ozone.Constants.OZONE_DEFAULT_USER;
+import static org.apache.hadoop.fs.ozone.Constants.OZONE_USER_DIR;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_OFS_URI_SCHEME;
+
+/**
+ * The minimal Ozone Filesystem implementation.
+ * <p>
+ * This is a basic version which doesn't extend
+ * KeyProviderTokenIssuer and doesn't include statistics. It can be used
+ * from older hadoop version. For newer hadoop version use the full featured
+ * OFileSystem.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class BasicRootedOzoneFileSystem extends FileSystem {
+ static final Logger LOG =
+ LoggerFactory.getLogger(BasicRootedOzoneFileSystem.class);
+
+ /**
+ * The Ozone client for connecting to Ozone server.
+ */
+
+ private URI uri;
+ private String userName;
+ private Path workingDir;
+// private String gOmHost;
+// private int gOmPort;
+// private Configuration gConf;
+// private boolean gIsolatedClassloader;
+
+ private RootedOzoneClientAdapter adapter;
+// private String adapterPath;
+
+ private static final String URI_EXCEPTION_TEXT =
+ "URL should be one of the following formats: " +
+ "ofs://om-service-id/ OR " +
+ "ofs://om-host.example.com/ OR " +
+ "ofs://om-host.example.com:5678/";
+
+ @Override
+ public void initialize(URI name, Configuration conf) throws IOException {
+ super.initialize(name, conf);
+ setConf(conf);
+ Objects.requireNonNull(name.getScheme(), "No scheme provided in " + name);
+ Preconditions.checkArgument(getScheme().equals(name.getScheme()),
+ "Invalid scheme provided in " + name);
+
+ String authority = name.getAuthority();
+ if (authority == null) {
+ // authority is null when fs.defaultFS is not a qualified ofs URI and
+ // ofs:/// is passed to the client. matcher will NPE if authority is null
+ throw new IllegalArgumentException(URI_EXCEPTION_TEXT);
+ }
+
+ String omHost;
Review comment:
We are assigning serviceID also to omHost variable. Can we either change the variable name to reflect this or add a comment explaining this.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] bharatviswa504 commented on issue #415: HDDS-2840.
Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
bharatviswa504 commented on issue #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#issuecomment-579027161
> > I think here we need to catch exception from createVolume, as by the time we create volume, if another client created the same volume, we get VOLUME_ALREADY_EXCEPTION, so this should be safe and ignored.
> > Same applicable for bucket creation also.
>
> Nice catch. Addressed in commit [ab64a98](https://github.com/apache/hadoop-ozone/commit/ab64a98fadd52b483136f99031d8ed9d401b36c9). @bharatviswa504 Please take a look.
Thanks for addressing it. This comment can be considered as resolved
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] smengcl commented on a change in pull request #415:
HDDS-2840. Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
smengcl commented on a change in pull request #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#discussion_r373141001
##########
File path: hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
##########
@@ -0,0 +1,482 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.TestDataUtil;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClientException;
+import org.apache.hadoop.ozone.client.OzoneKeyDetails;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.Set;
+import java.util.TreeSet;
+
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/**
+ * Ozone file system tests that are not covered by contract tests.
+ *
+ * TODO: Refactor this and TestOzoneFileSystem to eliminate most
+ * code duplication.
+ */
+public class TestRootedOzoneFileSystem {
+
+ @Rule
+ public Timeout globalTimeout = new Timeout(300_000);
+
+ private static MiniOzoneCluster cluster = null;
+
+ private static FileSystem fs;
+ private static RootedOzoneFileSystem ofs;
+
+ private static ObjectStore objectStore;
+
+ private String volumeName;
+ private String bucketName;
+
+ private String rootPath;
+
+ // Store path commonly used by tests that test functionality within a bucket
+ private String testBucketStr;
+ private Path testBucketPath;
+
+ @Before
+ public void init() throws Exception {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ cluster = MiniOzoneCluster.newBuilder(conf)
+ .setNumDatanodes(3)
+ .build();
+ cluster.waitForClusterToBeReady();
+ objectStore = cluster.getClient().getObjectStore();
+
+ // create a volume and a bucket to be used by RootedOzoneFileSystem (OFS)
+ OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(cluster);
+ volumeName = bucket.getVolumeName();
+ bucketName = bucket.getName();
+ testBucketStr = "/" + volumeName + "/" + bucketName;
+ testBucketPath = new Path(testBucketStr);
+
+ rootPath = String.format("%s://%s/", OzoneConsts.OZONE_OFS_URI_SCHEME,
+ conf.get(OZONE_OM_ADDRESS_KEY));
+
+ // Set the fs.defaultFS and start the filesystem
+ conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, rootPath);
+ // Note: FileSystem#loadFileSystems won't load OFS class due to META-INF
+ // hence this workaround.
+ conf.set("fs.ofs.impl", "org.apache.hadoop.fs.ozone.RootedOzoneFileSystem");
+ fs = FileSystem.get(conf);
+ ofs = (RootedOzoneFileSystem) fs;
+ }
+
+ @After
+ public void teardown() {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ IOUtils.closeQuietly(fs);
+ }
+
+ @Test
+ public void testOzoneFsServiceLoader() throws IOException {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ // Note: FileSystem#loadFileSystems won't load OFS class due to META-INF
+ // hence this workaround.
+ conf.set("fs.ofs.impl", "org.apache.hadoop.fs.ozone.RootedOzoneFileSystem");
+ assertEquals(
+ FileSystem.getFileSystemClass(OzoneConsts.OZONE_OFS_URI_SCHEME, conf),
+ RootedOzoneFileSystem.class);
+ }
+
+ @Test
+ public void testCreateDoesNotAddParentDirKeys() throws Exception {
+ Path grandparent = new Path(testBucketPath,
+ "testCreateDoesNotAddParentDirKeys");
+ Path parent = new Path(grandparent, "parent");
+ Path child = new Path(parent, "child");
+ ContractTestUtils.touch(fs, child);
+
+ OzoneKeyDetails key = getKey(child, false);
+ OFSPath childOFSPath = new OFSPath(child);
+ assertEquals(key.getName(), childOFSPath.getKeyName());
+
+ // Creating a child should not add parent keys to the bucket
+ try {
+ getKey(parent, true);
+ } catch (IOException ex) {
+ assertKeyNotFoundException(ex);
+ }
+
+ // List status on the parent should show the child file
+ assertEquals("List status of parent should include the 1 child file", 1L,
+ (long)fs.listStatus(parent).length);
+ assertTrue("Parent directory does not appear to be a directory",
+ fs.getFileStatus(parent).isDirectory());
+ }
+
+ @Test
+ public void testDeleteCreatesFakeParentDir() throws Exception {
+ Path grandparent = new Path(testBucketPath,
+ "testDeleteCreatesFakeParentDir");
+ Path parent = new Path(grandparent, "parent");
+ Path child = new Path(parent, "child");
+ ContractTestUtils.touch(fs, child);
+
+ // Verify that parent dir key does not exist
+ // Creating a child should not add parent keys to the bucket
+ try {
+ getKey(parent, true);
+ } catch (IOException ex) {
+ assertKeyNotFoundException(ex);
+ }
+
+ // Delete the child key
+ assertTrue(fs.delete(child, false));
+
+ // Deleting the only child should create the parent dir key if it does
+ // not exist
+ OFSPath parentOFSPath = new OFSPath(parent);
+ String parentKey = parentOFSPath.getKeyName() + "/";
+ OzoneKeyDetails parentKeyInfo = getKey(parent, true);
+ assertEquals(parentKey, parentKeyInfo.getName());
+
+ // Recursive delete with DeleteIterator
+ assertTrue(fs.delete(grandparent, true));
+ }
+
+ @Test
+ public void testListStatus() throws Exception {
+ Path parent = new Path(testBucketPath, "testListStatus");
+ Path file1 = new Path(parent, "key1");
+ Path file2 = new Path(parent, "key2");
+
+ FileStatus[] fileStatuses = ofs.listStatus(testBucketPath);
+ assertEquals("Should be empty", 0, fileStatuses.length);
+
+ ContractTestUtils.touch(fs, file1);
+ ContractTestUtils.touch(fs, file2);
+
+ fileStatuses = ofs.listStatus(testBucketPath);
+ assertEquals("Should have created parent",
+ 1, fileStatuses.length);
+ assertEquals("Parent path doesn't match",
+ fileStatuses[0].getPath().toUri().getPath(), parent.toString());
+
+ // ListStatus on a directory should return all subdirs along with
+ // files, even if there exists a file and sub-dir with the same name.
+ fileStatuses = ofs.listStatus(parent);
+ assertEquals("FileStatus did not return all children of the directory",
+ 2, fileStatuses.length);
+
+ // ListStatus should return only the immediate children of a directory.
+ Path file3 = new Path(parent, "dir1/key3");
+ Path file4 = new Path(parent, "dir1/key4");
+ ContractTestUtils.touch(fs, file3);
+ ContractTestUtils.touch(fs, file4);
+ fileStatuses = ofs.listStatus(parent);
+ assertEquals("FileStatus did not return all children of the directory",
+ 3, fileStatuses.length);
+ }
+
+ /**
+ * OFS: Helper function for tests. Return a volume name that doesn't exist.
+ */
+ private String getRandomNonExistVolumeName() throws Exception {
+ final int numDigit = 5;
+ long retriesLeft = Math.round(Math.pow(10, 5));
+ String name = null;
+ while (name == null && retriesLeft-- > 0) {
+ name = "volume-" + RandomStringUtils.randomNumeric(numDigit);
+ // Check volume existence.
+ Iterator<? extends OzoneVolume> iter =
+ objectStore.listVolumesByUser(null, name, null);
+ if (iter.hasNext()) {
+ // If there is a match, try again.
+ // Note that volume name prefix match doesn't equal volume existence
+ // but the check is sufficient for this test.
+ name = null;
+ }
+ }
+ if (retriesLeft <= 0) {
+ throw new Exception(
+ "Failed to generate random volume name that doesn't exist already.");
+ }
+ return name;
+ }
+
+ /**
+ * OFS: Test mkdir on volume, bucket and dir that doesn't exist.
+ */
+ @Test
+ public void testMkdirOnNonExistentVolumeBucketDir() throws Exception {
+ String volumeNameLocal = getRandomNonExistVolumeName();
+ String bucketNameLocal = "bucket-" + RandomStringUtils.randomNumeric(5);
+ Path root = new Path("/" + volumeNameLocal + "/" + bucketNameLocal);
+ Path dir1 = new Path(root, "dir1");
+ Path dir12 = new Path(dir1, "dir12");
+ Path dir2 = new Path(root, "dir2");
+ fs.mkdirs(dir12);
+ fs.mkdirs(dir2);
+
+ // Check volume and bucket existence, they should both be created.
+ OzoneVolume ozoneVolume = objectStore.getVolume(volumeNameLocal);
+ OzoneBucket ozoneBucket = ozoneVolume.getBucket(bucketNameLocal);
+ OFSPath ofsPathDir1 = new OFSPath(dir12);
+ String key = ofsPathDir1.getKeyName() + "/";
+ OzoneKeyDetails ozoneKeyDetails = ozoneBucket.getKey(key);
+ assertEquals(key, ozoneKeyDetails.getName());
+
+ // Verify that directories are created.
+ FileStatus[] fileStatuses = ofs.listStatus(root);
+ assertEquals(fileStatuses[0].getPath().toUri().getPath(), dir1.toString());
+ assertEquals(fileStatuses[1].getPath().toUri().getPath(), dir2.toString());
+
+ fileStatuses = ofs.listStatus(dir1);
+ assertEquals(fileStatuses[0].getPath().toUri().getPath(), dir12.toString());
+ fileStatuses = ofs.listStatus(dir12);
+ assertEquals(fileStatuses.length, 0);
+ fileStatuses = ofs.listStatus(dir2);
+ assertEquals(fileStatuses.length, 0);
+ }
+
+ /**
+ * OFS: Tests mkdir on a volume and bucket that doesn't exist.
+ */
+ @Test
+ public void testMkdirNonExistentVolumeBucket() throws Exception {
+ String volumeNameLocal = getRandomNonExistVolumeName();
+ String bucketNameLocal = "bucket-" + RandomStringUtils.randomNumeric(5);
+ Path newVolBucket = new Path(
+ "/" + volumeNameLocal + "/" + bucketNameLocal);
+ fs.mkdirs(newVolBucket);
+
+ // Verify with listVolumes and listBuckets
+ Iterator<? extends OzoneVolume> iterVol =
+ objectStore.listVolumesByUser(null, volumeNameLocal, null);
+ OzoneVolume ozoneVolume = iterVol.next();
+ assertNotNull(ozoneVolume);
+ assertEquals(volumeNameLocal, ozoneVolume.getName());
+
+ Iterator<? extends OzoneBucket> iterBuc =
+ ozoneVolume.listBuckets("bucket-");
+ OzoneBucket ozoneBucket = iterBuc.next();
+ assertNotNull(ozoneBucket);
+ assertEquals(bucketNameLocal, ozoneBucket.getName());
+
+ // TODO: Use listStatus to check volume and bucket creation in HDDS-2928.
+ }
+
+ /**
+ * OFS: Tests mkdir on a volume that doesn't exist.
+ */
+ @Test
+ public void testMkdirNonExistentVolume() throws Exception {
+ String volumeNameLocal = getRandomNonExistVolumeName();
+ Path newVolume = new Path("/" + volumeNameLocal);
+ fs.mkdirs(newVolume);
+
+ // Verify with listVolumes and listBuckets
+ Iterator<? extends OzoneVolume> iterVol =
+ objectStore.listVolumesByUser(null, volumeNameLocal, null);
+ OzoneVolume ozoneVolume = iterVol.next();
+ assertNotNull(ozoneVolume);
+ assertEquals(volumeNameLocal, ozoneVolume.getName());
+
+ // TODO: Use listStatus to check volume and bucket creation in HDDS-2928.
+ }
+
+ /**
+ * Tests listStatus operation in a bucket.
+ */
+ @Test
+ public void testListStatusOnRoot() throws Exception {
+ Path root = new Path("/" + volumeName + "/" + bucketName);
+ Path dir1 = new Path(root, "dir1");
+ Path dir12 = new Path(dir1, "dir12");
+ Path dir2 = new Path(root, "dir2");
+ fs.mkdirs(dir12);
+ fs.mkdirs(dir2);
+
+ // ListStatus on root should return dir1 (even though /dir1 key does not
+ // exist) and dir2 only. dir12 is not an immediate child of root and
+ // hence should not be listed.
+ FileStatus[] fileStatuses = ofs.listStatus(root);
+ assertEquals("FileStatus should return only the immediate children", 2,
+ fileStatuses.length);
+
+ // Verify that dir12 is not included in the result of the listStatus on root
+ String fileStatus1 = fileStatuses[0].getPath().toUri().getPath();
+ String fileStatus2 = fileStatuses[1].getPath().toUri().getPath();
+ assertFalse(fileStatus1.equals(dir12.toString()));
+ assertFalse(fileStatus2.equals(dir12.toString()));
+ }
+
+ /**
+ * Tests listStatus operation on root directory.
+ */
+ @Test
+ public void testListStatusOnLargeDirectory() throws Exception {
+ Path root = new Path("/" + volumeName + "/" + bucketName);
+ Set<String> paths = new TreeSet<>();
+ int numDirs = 5111;
Review comment:
I'm not sure why o3fs test chose this random number. But basically `numDirs` only need to be larger than `LISTING_PAGE_SIZE` so it tests pagination. And BTW I believe integration tests are disabled in PR by default now. (They are not run in PR checks.)
Reduced in 0d7100604334f06fe304b143a3b906c9cbfc88dd.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] smengcl edited a comment on issue #415: HDDS-2840.
Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
smengcl edited a comment on issue #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#issuecomment-578954095
When writing unit tests for `OFSPath`, I realized that there is one corner case for `mkdir` that I didn't handle properly: `mkdir /volume1` <- we should expect this to create a volume `volume1` only. Will push new commit to fix this in a moment.
Update: pushed https://github.com/apache/hadoop-ozone/pull/415/commits/60b4bf8e6008cbd53d95c3e0f249f33227209c1e
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] smengcl commented on a change in pull request #415:
HDDS-2840. Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
smengcl commented on a change in pull request #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#discussion_r371191441
##########
File path: hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
##########
@@ -0,0 +1,604 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.ozone;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.key.KeyProvider;
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdds.client.ReplicationFactor;
+import org.apache.hadoop.hdds.client.ReplicationType;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ozone.OmUtils;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.client.OzoneKey;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
+import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
+import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenRenewer;
+
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .VOLUME_NOT_FOUND;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .BUCKET_NOT_FOUND;
+
+/**
+ * Basic Implementation of the OzoneFileSystem calls.
+ * <p>
+ * This is the minimal version which doesn't include any statistics.
+ * <p>
+ * For full featured version use OzoneClientAdapterImpl.
+ */
+public class BasicRootedOzoneClientAdapterImpl
+ implements RootedOzoneClientAdapter {
+
+ static final Logger LOG =
+ LoggerFactory.getLogger(BasicRootedOzoneClientAdapterImpl.class);
+
+ private OzoneClient ozoneClient;
+ private ClientProtocol proxy;
+ private ObjectStore objectStore;
+ private ReplicationType replicationType;
+ private ReplicationFactor replicationFactor;
+ private boolean securityEnabled;
+ private int configuredDnPort;
+
+ /**
+ * Create new OzoneClientAdapter implementation.
+ *
+ * @throws IOException In case of a problem.
+ */
+ public BasicRootedOzoneClientAdapterImpl() throws IOException {
+ this(createConf());
+ }
+
+ private static OzoneConfiguration createConf() {
+ ClassLoader contextClassLoader =
+ Thread.currentThread().getContextClassLoader();
+ Thread.currentThread().setContextClassLoader(null);
+ try {
+ return new OzoneConfiguration();
+ } finally {
+ Thread.currentThread().setContextClassLoader(contextClassLoader);
+ }
+ }
+
+ public BasicRootedOzoneClientAdapterImpl(OzoneConfiguration conf)
+ throws IOException {
+ this(null, -1, conf);
+ }
+
+ public BasicRootedOzoneClientAdapterImpl(String omHost, int omPort,
+ Configuration hadoopConf) throws IOException {
+
+ ClassLoader contextClassLoader =
+ Thread.currentThread().getContextClassLoader();
+ Thread.currentThread().setContextClassLoader(null);
+
+ try {
+ OzoneConfiguration conf = OzoneConfiguration.of(hadoopConf);
+
+ if (omHost == null && OmUtils.isServiceIdsDefined(conf)) {
+ // When the host name or service id isn't given
+ // but ozone.om.service.ids is defined, declare failure.
+
+ // This is a safety precaution that prevents the client from
+ // accidentally failing over to an unintended OM.
+ throw new IllegalArgumentException("Service ID or host name must not"
+ + " be omitted when ozone.om.service.ids is defined.");
+ }
+
+ if (omPort != -1) {
+ // When the port number is specified, perform the following check
+ if (OmUtils.isOmHAServiceId(conf, omHost)) {
+ // If omHost is a service id, it shouldn't use a port
+ throw new IllegalArgumentException("Port " + omPort +
+ " specified in URI but host '" + omHost + "' is a "
+ + "logical (HA) OzoneManager and does not use port information.");
+ }
+ } else {
+ // When port number is not specified, read it from config
+ omPort = OmUtils.getOmRpcPort(conf);
+ }
+
+ SecurityConfig secConfig = new SecurityConfig(conf);
+
+ if (secConfig.isSecurityEnabled()) {
+ this.securityEnabled = true;
+ }
+
+ String replicationTypeConf =
+ conf.get(OzoneConfigKeys.OZONE_REPLICATION_TYPE,
+ OzoneConfigKeys.OZONE_REPLICATION_TYPE_DEFAULT);
+
+ int replicationCountConf = conf.getInt(OzoneConfigKeys.OZONE_REPLICATION,
+ OzoneConfigKeys.OZONE_REPLICATION_DEFAULT);
+
+ if (OmUtils.isOmHAServiceId(conf, omHost)) {
+ // omHost is listed as one of the service ids in the config,
+ // thus we should treat omHost as omServiceId
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(omHost, conf);
+ } else if (StringUtils.isNotEmpty(omHost) && omPort != -1) {
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(omHost, omPort, conf);
+ } else {
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(conf);
+ }
+ objectStore = ozoneClient.getObjectStore();
+ proxy = objectStore.getClientProxy();
+ this.replicationType = ReplicationType.valueOf(replicationTypeConf);
+ this.replicationFactor = ReplicationFactor.valueOf(replicationCountConf);
+ this.configuredDnPort = conf.getInt(
+ OzoneConfigKeys.DFS_CONTAINER_IPC_PORT,
+ OzoneConfigKeys.DFS_CONTAINER_IPC_PORT_DEFAULT);
+ } finally {
+ Thread.currentThread().setContextClassLoader(contextClassLoader);
+ }
+ }
+
+ private OzoneBucket getBucket(OFSPath ofsPath,
+ boolean createIfNotExist) throws IOException {
+
+ return getBucket(ofsPath.getVolumeName(), ofsPath.getBucketName(),
+ createIfNotExist);
+ }
+
+ /**
+ * Get OzoneBucket object to operate in.
+ * Optionally create volume and bucket if not found.
+ *
+ * @param createIfNotExist Set this to true if the caller is a write operation
+ * in order to create the volume and bucket.
+ * @throws IOException Exceptions other than OMException with result code
+ * VOLUME_NOT_FOUND or BUCKET_NOT_FOUND.
+ */
+ private OzoneBucket getBucket(String volumeStr, String bucketStr,
+ boolean createIfNotExist) throws IOException {
+
+ OzoneBucket bucket;
+ try {
+ bucket = proxy.getBucketDetails(volumeStr, bucketStr);
+ } catch (OMException ex) {
+ if (createIfNotExist) {
+ // Note: getBucketDetails always throws BUCKET_NOT_FOUND, even if
+ // the volume doesn't exist.
+ if (ex.getResult().equals(BUCKET_NOT_FOUND)) {
+ OzoneVolume volume;
+ try {
+ volume = proxy.getVolumeDetails(volumeStr);
+ } catch (OMException volEx) {
+ if (volEx.getResult().equals(VOLUME_NOT_FOUND)) {
+ // Volume doesn't exist. Create it
+ objectStore.createVolume(volumeStr);
+ } else {
+ throw volEx;
+ }
+ // Try get volume again
+ volume = proxy.getVolumeDetails(volumeStr);
+ }
+ // Create the bucket
+ volume.createBucket(bucketStr);
+ }
+ // Try get bucket again
+ bucket = proxy.getBucketDetails(volumeStr, bucketStr);
+ } else {
+ throw ex;
+ }
+ }
+
+ return bucket;
+ }
+
+ @Override
+ public void close() throws IOException {
+ ozoneClient.close();
+ }
+
+ @Override
+ public InputStream readFile(String pathStr) throws IOException {
+ incrementCounter(Statistic.OBJECTS_READ);
+ OFSPath ofsPath = new OFSPath(pathStr);
+ String key = ofsPath.getKeyName();
+ try {
+ OzoneBucket bucket = getBucket(ofsPath, false);
+ return bucket.readFile(key).getInputStream();
+ } catch (OMException ex) {
+ if (ex.getResult() == OMException.ResultCodes.FILE_NOT_FOUND
+ || ex.getResult() == OMException.ResultCodes.NOT_A_FILE) {
+ throw new FileNotFoundException(
+ ex.getResult().name() + ": " + ex.getMessage());
+ } else {
+ throw ex;
+ }
+ }
+ }
+
+ protected void incrementCounter(Statistic objectsRead) {
+ //noop: Use OzoneClientAdapterImpl which supports statistics.
+ }
+
+ @Override
+ public OzoneFSOutputStream createFile(String pathStr, boolean overWrite,
+ boolean recursive) throws IOException {
+ incrementCounter(Statistic.OBJECTS_CREATED);
+ OFSPath ofsPath = new OFSPath(pathStr);
+ String key = ofsPath.getKeyName();
+ try {
+ OzoneBucket bucket = getBucket(ofsPath, false);
+ OzoneOutputStream ozoneOutputStream = bucket.createFile(
+ key, 0, replicationType, replicationFactor, overWrite, recursive);
+ return new OzoneFSOutputStream(ozoneOutputStream.getOutputStream());
+ } catch (OMException ex) {
+ if (ex.getResult() == OMException.ResultCodes.FILE_ALREADY_EXISTS
+ || ex.getResult() == OMException.ResultCodes.NOT_A_FILE) {
+ throw new FileAlreadyExistsException(
+ ex.getResult().name() + ": " + ex.getMessage());
+ } else {
+ throw ex;
+ }
+ }
+ }
+
+ @Override
+ public void renamePath(String path, String newPath) throws IOException {
+ incrementCounter(Statistic.OBJECTS_RENAMED);
+ OFSPath ofsPath = new OFSPath(path);
+ OFSPath ofsNewPath = new OFSPath(newPath);
+
+ // Check path and newPathName are in the same volume and same bucket.
+ // This should have been checked in BasicRootedOzoneFileSystem#rename
+ // already via regular call path unless bypassed.
+ if (!ofsPath.isInSameBucketAs(ofsNewPath)) {
+ throw new IOException("Cannot rename a key to a different bucket");
+ }
+
+ OzoneBucket bucket = getBucket(ofsPath, false);
+ String key = ofsPath.getKeyName();
+ String newKey = ofsNewPath.getKeyName();
+ bucket.renameKey(key, newKey);
+ }
+
+ /**
+ * Helper method to create an directory specified by key name in bucket.
+ *
+ * @param pathStr path to be created as directory
+ * @return true if the key is created, false otherwise
+ */
+ @Override
+ public boolean createDirectory(String pathStr) throws IOException {
+ LOG.trace("creating dir for path: {}", pathStr);
+ incrementCounter(Statistic.OBJECTS_CREATED);
+ OFSPath ofsPath = new OFSPath(pathStr);
+ String keyStr = ofsPath.getKeyName();
+ try {
+ OzoneBucket bucket = getBucket(ofsPath, true);
+ bucket.createDirectory(keyStr);
+ } catch (OMException e) {
+ if (e.getResult() == OMException.ResultCodes.FILE_ALREADY_EXISTS) {
+ throw new FileAlreadyExistsException(e.getMessage());
+ }
+ throw e;
+ }
+ return true;
+ }
+
+ /**
+ * Helper method to delete an object specified by key name in bucket.
+ *
+ * @param path path to a key to be deleted
+ * @return true if the key is deleted, false otherwise
+ */
+ @Override
+ public boolean deleteObject(String path) {
+ LOG.trace("issuing delete for path to key: {}", path);
+ OFSPath ofsPath = new OFSPath(path);
+ String keyName = ofsPath.getKeyName();
+ try {
+ OzoneBucket bucket = getBucket(ofsPath, false);
+ incrementCounter(Statistic.OBJECTS_DELETED);
+ bucket.deleteKey(keyName);
+ return true;
+ } catch (IOException ioe) {
+ LOG.error("delete key failed " + ioe.getMessage());
+ return false;
+ }
+ }
+
+ public FileStatusAdapter getFileStatus(String path, URI uri,
+ Path qualifiedPath, String userName)
+ throws IOException {
+ OFSPath ofsPath = new OFSPath(path);
+ String key = ofsPath.getKeyName();
+ try {
+ OzoneBucket bucket = getBucket(ofsPath, false);
+ incrementCounter(Statistic.OBJECTS_QUERY);
+ OzoneFileStatus status = bucket.getFileStatus(key);
+ // Note: qualifiedPath passed in is good from
+ // BasicRootedOzoneFileSystem#getFileStatus. No need to prepend here.
+ makeQualified(status, uri, qualifiedPath, userName);
+ return toFileStatusAdapter(status);
+
+ } catch (OMException e) {
+ if (e.getResult() == OMException.ResultCodes.FILE_NOT_FOUND) {
+ throw new
+ FileNotFoundException(key + ": No such file or directory!");
+ }
+ throw e;
+ }
+ }
+
+ public void makeQualified(FileStatus status, URI uri, Path path,
+ String username) {
+ if (status instanceof OzoneFileStatus) {
+ ((OzoneFileStatus) status)
+ .makeQualified(uri, path,
+ username, username);
+ }
+ }
+
+ @Override
+ public Iterator<BasicKeyInfo> listKeys(String pathStr) throws IOException {
+ incrementCounter(Statistic.OBJECTS_LIST);
+ OFSPath ofsPath = new OFSPath(pathStr);
+ String key = ofsPath.getKeyName();
+ OzoneBucket bucket = getBucket(ofsPath, false);
+ return new IteratorAdapter(bucket.listKeys(key));
+ }
+
+ public List<FileStatusAdapter> listStatus(String pathStr, boolean recursive,
+ String startPath, long numEntries, URI uri,
+ Path workingDir, String username) throws IOException {
+
+ OFSPath ofsPath = new OFSPath(pathStr);
+ String keyName = ofsPath.getKeyName();
+ OFSPath ofsStartPath = new OFSPath(startPath);
+ String startKey = ofsStartPath.getKeyName();
Review comment:
Since I made all the input path params (`pathStr`) for `BasicRootedOzoneClientAdapterImpl` methods full path (volume+bucket+key), I believe it's best to keep `startPath` consistent. - That's why I intentionally renamed the parameter name in API definition from `startKey` to `startPath`.
Totally agree that doc is required. This should've been done in o3fs `BasicOzoneClientAdapterImpl` as well.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] hanishakoneru commented on a change in pull request
#415: HDDS-2840. Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
hanishakoneru commented on a change in pull request #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#discussion_r369821804
##########
File path: hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
##########
@@ -0,0 +1,805 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.ozone;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.CreateFlag;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.Progressable;
+import org.apache.http.client.utils.URIBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.EnumSet;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+import static org.apache.hadoop.fs.ozone.Constants.LISTING_PAGE_SIZE;
+import static org.apache.hadoop.fs.ozone.Constants.OZONE_DEFAULT_USER;
+import static org.apache.hadoop.fs.ozone.Constants.OZONE_USER_DIR;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_OFS_URI_SCHEME;
+
+/**
+ * The minimal Ozone Filesystem implementation.
+ * <p>
+ * This is a basic version which doesn't extend
+ * KeyProviderTokenIssuer and doesn't include statistics. It can be used
+ * from older hadoop version. For newer hadoop version use the full featured
+ * OFileSystem.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class BasicRootedOzoneFileSystem extends FileSystem {
+ static final Logger LOG =
+ LoggerFactory.getLogger(BasicRootedOzoneFileSystem.class);
+
+ /**
+ * The Ozone client for connecting to Ozone server.
+ */
+
+ private URI uri;
+ private String userName;
+ private Path workingDir;
+// private String gOmHost;
+// private int gOmPort;
+// private Configuration gConf;
+// private boolean gIsolatedClassloader;
+
+ private RootedOzoneClientAdapter adapter;
+// private String adapterPath;
+
+ private static final String URI_EXCEPTION_TEXT =
+ "URL should be one of the following formats: " +
+ "ofs://om-service-id/ OR " +
+ "ofs://om-host.example.com/ OR " +
+ "ofs://om-host.example.com:5678/";
+
+ @Override
+ public void initialize(URI name, Configuration conf) throws IOException {
+ super.initialize(name, conf);
+ setConf(conf);
+ Objects.requireNonNull(name.getScheme(), "No scheme provided in " + name);
+ Preconditions.checkArgument(getScheme().equals(name.getScheme()),
+ "Invalid scheme provided in " + name);
+
+ String authority = name.getAuthority();
+ if (authority == null) {
+ // authority is null when fs.defaultFS is not a qualified ofs URI and
+ // ofs:/// is passed to the client. matcher will NPE if authority is null
+ throw new IllegalArgumentException(URI_EXCEPTION_TEXT);
+ }
+
+ String omHost;
+ int omPort = -1;
+ // Parse hostname and port
+ String[] parts = authority.split(":");
+ if (parts.length > 2) {
+ throw new IllegalArgumentException(URI_EXCEPTION_TEXT);
+ }
+ omHost = parts[0];
+ if (parts.length == 2) {
+ try {
+ omPort = Integer.parseInt(parts[1]);
+ } catch (NumberFormatException e) {
+ throw new IllegalArgumentException(URI_EXCEPTION_TEXT);
+ }
+ }
+
+ try {
+ uri = new URIBuilder().setScheme(OZONE_OFS_URI_SCHEME)
+ .setHost(authority)
+ .build();
+ LOG.trace("Ozone URI for OFS initialization is " + uri);
+
+ //isolated is the default for ozonefs-lib-legacy which includes the
+ // /ozonefs.txt, otherwise the default is false. It could be overridden.
+ boolean defaultValue =
+ BasicRootedOzoneFileSystem.class.getClassLoader()
+ .getResource("ozonefs.txt") != null;
+
+ //Use string here instead of the constant as constant may not be available
+ //on the classpath of a hadoop 2.7
+ boolean isolatedClassloader =
+ conf.getBoolean("ozone.fs.isolated-classloader", defaultValue);
+
+ // adapter should be initialized in operations.
+ this.adapter = createAdapter(conf, omHost, omPort, isolatedClassloader);
+
+ try {
+ this.userName =
+ UserGroupInformation.getCurrentUser().getShortUserName();
+ } catch (IOException e) {
+ this.userName = OZONE_DEFAULT_USER;
+ }
+ this.workingDir = new Path(OZONE_USER_DIR, this.userName)
+ .makeQualified(this.uri, this.workingDir);
+ } catch (URISyntaxException ue) {
+ final String msg = "Invalid Ozone endpoint " + name;
+ LOG.error(msg, ue);
+ throw new IOException(msg, ue);
+ }
+ }
+
+ protected RootedOzoneClientAdapter createAdapter(Configuration conf,
+ String omHost, int omPort, boolean isolatedClassloader)
+ throws IOException {
+
+ if (isolatedClassloader) {
+ // TODO: Check how this code path need to be changed, for legacy Hadoop?
+ return RootedOzoneClientAdapterFactory.createAdapter();
+ } else {
+ return new BasicRootedOzoneClientAdapterImpl(omHost, omPort, conf);
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ try {
+ adapter.close();
+ } finally {
+ super.close();
+ }
+ }
+
+ @Override
+ public URI getUri() {
+ return uri;
+ }
+
+ @Override
+ public String getScheme() {
+ return OZONE_OFS_URI_SCHEME;
+ }
+
+ @Override
+ public FSDataInputStream open(Path path, int bufferSize) throws IOException {
+ incrementCounter(Statistic.INVOCATION_OPEN);
+ statistics.incrementReadOps(1);
+ LOG.trace("open() path: {}", path);
+ final String key = pathToKey(path);
+ return new FSDataInputStream(
+ new OzoneFSInputStream(adapter.readFile(key), statistics));
+ }
+
+ protected void incrementCounter(Statistic statistic) {
+ //don't do anything in this default implementation.
+ }
+
+ @Override
+ public FSDataOutputStream create(Path f, FsPermission permission,
+ boolean overwrite, int bufferSize,
+ short replication, long blockSize,
+ Progressable progress) throws IOException {
+ LOG.trace("create() path:{}", f);
+ incrementCounter(Statistic.INVOCATION_CREATE);
+ statistics.incrementWriteOps(1);
+ final String key = pathToKey(f);
+ return createOutputStream(key, overwrite, true);
+ }
+
+ @Override
+ public FSDataOutputStream createNonRecursive(Path path,
+ FsPermission permission,
+ EnumSet<CreateFlag> flags,
+ int bufferSize,
+ short replication,
+ long blockSize,
+ Progressable progress) throws IOException {
+ incrementCounter(Statistic.INVOCATION_CREATE_NON_RECURSIVE);
+ statistics.incrementWriteOps(1);
+ final String key = pathToKey(path);
+ return createOutputStream(key, flags.contains(CreateFlag.OVERWRITE), false);
+ }
+
+ private FSDataOutputStream createOutputStream(String key, boolean overwrite,
+ boolean recursive) throws IOException {
+ return new FSDataOutputStream(adapter.createFile(key, overwrite, recursive),
+ statistics);
+ }
+
+ @Override
+ public FSDataOutputStream append(Path f, int bufferSize,
+ Progressable progress) throws IOException {
+ throw new UnsupportedOperationException("append() Not implemented by the "
+ + getClass().getSimpleName() + " FileSystem implementation");
+ }
+
+ private class RenameIterator extends OzoneListingIterator {
+ private final String srcPath;
+ private final String dstPath;
+
+ RenameIterator(Path srcPath, Path dstPath)
+ throws IOException {
+ super(srcPath);
+ this.srcPath = pathToKey(srcPath);
+ this.dstPath = pathToKey(dstPath);
+ LOG.trace("rename from:{} to:{}", this.srcPath, this.dstPath);
+ }
+
+ @Override
+ boolean processKeyPath(String keyPath) throws IOException {
+ String newPath = dstPath.concat(keyPath.substring(srcPath.length()));
+ adapter.renamePath(keyPath, newPath);
+ return true;
+ }
+ }
+
+ /**
+ * Check whether the source and destination path are valid and then perform
+ * rename from source path to destination path.
+ * <p>
+ * The rename operation is performed by renaming the keys with src as prefix.
+ * For such keys the prefix is changed from src to dst.
+ *
+ * @param src source path for rename
+ * @param dst destination path for rename
+ * @return true if rename operation succeeded or
+ * if the src and dst have the same path and are of the same type
+ * @throws IOException on I/O errors or if the src/dst paths are invalid.
+ */
+ @Override
+ public boolean rename(Path src, Path dst) throws IOException {
+ incrementCounter(Statistic.INVOCATION_RENAME);
+ statistics.incrementWriteOps(1);
+ if (src.equals(dst)) {
+ return true;
+ }
+
+ LOG.trace("rename() from: {} to: {}", src, dst);
+ if (src.isRoot()) {
+ // Cannot rename root of file system
+ LOG.trace("Cannot rename the root of a filesystem");
+ return false;
+ }
+
+ // src and dst should be in the same bucket
+ OFSPath ofsSrc = new OFSPath(src);
+ OFSPath ofsDst = new OFSPath(dst);
+ if (!ofsSrc.isInSameBucketAs(ofsDst)) {
+ throw new IOException("Cannot rename a key to a different bucket");
+ }
+
+ // Cannot rename a directory to its own subdirectory
+ Path dstParent = dst.getParent();
+ while (dstParent != null && !src.equals(dstParent)) {
+ dstParent = dstParent.getParent();
+ }
+ Preconditions.checkArgument(dstParent == null,
+ "Cannot rename a directory to its own subdirectory");
+ // Check if the source exists
+ FileStatus srcStatus;
+ try {
+ srcStatus = getFileStatus(src);
+ } catch (FileNotFoundException fnfe) {
+ // source doesn't exist, return
+ return false;
+ }
+
+ // Check if the destination exists
+ FileStatus dstStatus;
+ try {
+ dstStatus = getFileStatus(dst);
+ } catch (FileNotFoundException fnde) {
+ dstStatus = null;
+ }
+
+ if (dstStatus == null) {
+ // If dst doesn't exist, check whether dst parent dir exists or not
+ // if the parent exists, the source can still be renamed to dst path
+ dstStatus = getFileStatus(dst.getParent());
+ if (!dstStatus.isDirectory()) {
+ throw new IOException(String.format(
+ "Failed to rename %s to %s, %s is a file", src, dst,
+ dst.getParent()));
+ }
+ } else {
+ // if dst exists and source and destination are same,
+ // check both the src and dst are of same type
+ if (srcStatus.getPath().equals(dstStatus.getPath())) {
+ return !srcStatus.isDirectory();
+ } else if (dstStatus.isDirectory()) {
+ // If dst is a directory, rename source as subpath of it.
+ // for example rename /source to /dst will lead to /dst/source
+ dst = new Path(dst, src.getName());
+ FileStatus[] statuses;
+ try {
+ statuses = listStatus(dst);
+ } catch (FileNotFoundException fnde) {
+ statuses = null;
+ }
+
+ if (statuses != null && statuses.length > 0) {
+ // If dst exists and not a directory not empty
+ throw new FileAlreadyExistsException(String.format(
+ "Failed to rename %s to %s, file already exists or not empty!",
+ src, dst));
+ }
+ } else {
+ // If dst is not a directory
+ throw new FileAlreadyExistsException(String.format(
+ "Failed to rename %s to %s, file already exists!", src, dst));
+ }
+ }
+
+ if (srcStatus.isDirectory()) {
+ if (dst.toString().startsWith(src.toString() + OZONE_URI_DELIMITER)) {
+ LOG.trace("Cannot rename a directory to a subdirectory of self");
+ return false;
+ }
+ }
+ RenameIterator iterator = new RenameIterator(src, dst);
+ boolean result = iterator.iterate();
+ if (result) {
+ createFakeParentDirectory(src);
+ }
+ return result;
+ }
+
+ private class DeleteIterator extends OzoneListingIterator {
+ private boolean recursive;
+
+ DeleteIterator(Path f, boolean recursive)
+ throws IOException {
+ super(f);
+ this.recursive = recursive;
+ if (getStatus().isDirectory()
+ && !this.recursive
+ && listStatus(f).length != 0) {
+ throw new PathIsNotEmptyDirectoryException(f.toString());
+ }
+ }
+
+ @Override
+ boolean processKeyPath(String keyPath) throws IOException {
+ if (keyPath.equals("")) {
+ LOG.trace("Skipping deleting root directory");
+ return true;
+ } else {
+ LOG.trace("deleting key path:" + keyPath);
+ boolean succeed = adapter.deleteObject(keyPath);
Review comment:
When deleting a directory, we iterate through all the keys in that directory. For each key, we are going to call adapter.deleteObject(). This function would in turn get BucketInfo from OM each time. We should optimize this to avoid the extra getBucketInfo calls.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] smengcl commented on a change in pull request #415:
HDDS-2840. Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
smengcl commented on a change in pull request #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#discussion_r372556019
##########
File path: hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
##########
@@ -0,0 +1,694 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.ozone;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.key.KeyProvider;
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdds.client.ReplicationFactor;
+import org.apache.hadoop.hdds.client.ReplicationType;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ozone.OmUtils;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.client.OzoneKey;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
+import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
+import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenRenewer;
+
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .BUCKET_ALREADY_EXISTS;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .VOLUME_ALREADY_EXISTS;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .VOLUME_NOT_FOUND;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .BUCKET_NOT_FOUND;
+
+/**
+ * Basic Implementation of the OzoneFileSystem calls.
+ * <p>
+ * This is the minimal version which doesn't include any statistics.
+ * <p>
+ * For full featured version use OzoneClientAdapterImpl.
+ */
+public class BasicRootedOzoneClientAdapterImpl
+ implements RootedOzoneClientAdapter {
+
+ static final Logger LOG =
+ LoggerFactory.getLogger(BasicRootedOzoneClientAdapterImpl.class);
+
+ private OzoneClient ozoneClient;
+ private ClientProtocol proxy;
+ private ObjectStore objectStore;
+ private ReplicationType replicationType;
+ private ReplicationFactor replicationFactor;
+ private boolean securityEnabled;
+ private int configuredDnPort;
+
+ /**
+ * Create new OzoneClientAdapter implementation.
+ *
+ * @throws IOException In case of a problem.
+ */
+ public BasicRootedOzoneClientAdapterImpl() throws IOException {
+ this(createConf());
+ }
+
+ private static OzoneConfiguration createConf() {
+ ClassLoader contextClassLoader =
+ Thread.currentThread().getContextClassLoader();
+ Thread.currentThread().setContextClassLoader(null);
+ try {
+ return new OzoneConfiguration();
+ } finally {
+ Thread.currentThread().setContextClassLoader(contextClassLoader);
+ }
+ }
+
+ public BasicRootedOzoneClientAdapterImpl(OzoneConfiguration conf)
+ throws IOException {
+ this(null, -1, conf);
+ }
+
+ public BasicRootedOzoneClientAdapterImpl(String omHost, int omPort,
+ Configuration hadoopConf) throws IOException {
+
+ ClassLoader contextClassLoader =
+ Thread.currentThread().getContextClassLoader();
+ Thread.currentThread().setContextClassLoader(null);
+
+ try {
+ OzoneConfiguration conf = OzoneConfiguration.of(hadoopConf);
+
+ if (omHost == null && OmUtils.isServiceIdsDefined(conf)) {
+ // When the host name or service id isn't given
+ // but ozone.om.service.ids is defined, declare failure.
+
+ // This is a safety precaution that prevents the client from
+ // accidentally failing over to an unintended OM.
+ throw new IllegalArgumentException("Service ID or host name must not"
+ + " be omitted when ozone.om.service.ids is defined.");
+ }
+
+ if (omPort != -1) {
+ // When the port number is specified, perform the following check
+ if (OmUtils.isOmHAServiceId(conf, omHost)) {
+ // If omHost is a service id, it shouldn't use a port
+ throw new IllegalArgumentException("Port " + omPort +
+ " specified in URI but host '" + omHost + "' is a "
+ + "logical (HA) OzoneManager and does not use port information.");
+ }
+ } else {
+ // When port number is not specified, read it from config
+ omPort = OmUtils.getOmRpcPort(conf);
+ }
+
+ SecurityConfig secConfig = new SecurityConfig(conf);
+
+ if (secConfig.isSecurityEnabled()) {
+ this.securityEnabled = true;
+ }
+
+ String replicationTypeConf =
+ conf.get(OzoneConfigKeys.OZONE_REPLICATION_TYPE,
+ OzoneConfigKeys.OZONE_REPLICATION_TYPE_DEFAULT);
+
+ int replicationCountConf = conf.getInt(OzoneConfigKeys.OZONE_REPLICATION,
+ OzoneConfigKeys.OZONE_REPLICATION_DEFAULT);
+
+ if (OmUtils.isOmHAServiceId(conf, omHost)) {
+ // omHost is listed as one of the service ids in the config,
+ // thus we should treat omHost as omServiceId
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(omHost, conf);
+ } else if (StringUtils.isNotEmpty(omHost) && omPort != -1) {
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(omHost, omPort, conf);
+ } else {
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(conf);
+ }
+ objectStore = ozoneClient.getObjectStore();
+ proxy = objectStore.getClientProxy();
+ this.replicationType = ReplicationType.valueOf(replicationTypeConf);
+ this.replicationFactor = ReplicationFactor.valueOf(replicationCountConf);
+ this.configuredDnPort = conf.getInt(
+ OzoneConfigKeys.DFS_CONTAINER_IPC_PORT,
+ OzoneConfigKeys.DFS_CONTAINER_IPC_PORT_DEFAULT);
+ } finally {
+ Thread.currentThread().setContextClassLoader(contextClassLoader);
+ }
+ }
+
+ OzoneBucket getBucket(OFSPath ofsPath, boolean createIfNotExist)
+ throws IOException {
+
+ return getBucket(ofsPath.getVolumeName(), ofsPath.getBucketName(),
+ createIfNotExist);
+ }
+
+ /**
+ * Get OzoneBucket object to operate in.
+ * Optionally create volume and bucket if not found.
+ *
+ * @param createIfNotExist Set this to true if the caller is a write operation
+ * in order to create the volume and bucket.
+ * @throws IOException Exceptions other than OMException with result code
+ * VOLUME_NOT_FOUND or BUCKET_NOT_FOUND.
+ */
+ private OzoneBucket getBucket(String volumeStr, String bucketStr,
+ boolean createIfNotExist) throws IOException {
+
+ OzoneBucket bucket;
+ try {
+ bucket = proxy.getBucketDetails(volumeStr, bucketStr);
+ } catch (OMException ex) {
+ if (createIfNotExist) {
+ // Note: getBucketDetails always throws BUCKET_NOT_FOUND, even if
+ // the volume doesn't exist.
+ if (ex.getResult().equals(BUCKET_NOT_FOUND)) {
+ OzoneVolume volume;
+ try {
+ volume = proxy.getVolumeDetails(volumeStr);
+ } catch (OMException getVolEx) {
+ if (getVolEx.getResult().equals(VOLUME_NOT_FOUND)) {
+ // Volume doesn't exist. Create it
+ try {
+ objectStore.createVolume(volumeStr);
+ } catch (OMException newVolEx) {
+ // Ignore the case where another client created the volume
+ if (!newVolEx.getResult().equals(VOLUME_ALREADY_EXISTS)) {
+ throw newVolEx;
+ }
+ }
+ } else {
+ throw getVolEx;
+ }
+ // Try get volume again
+ volume = proxy.getVolumeDetails(volumeStr);
+ }
+ // Create the bucket
+ try {
+ volume.createBucket(bucketStr);
+ } catch (OMException newBucEx) {
+ // Ignore the case where another client created the bucket
+ if (!newBucEx.getResult().equals(BUCKET_ALREADY_EXISTS)) {
+ throw newBucEx;
+ }
+ }
+ }
+ // Try get bucket again
+ bucket = proxy.getBucketDetails(volumeStr, bucketStr);
+ } else {
+ throw ex;
+ }
+ }
+
+ return bucket;
+ }
+
+ @Override
+ public void close() throws IOException {
+ ozoneClient.close();
+ }
+
+ @Override
+ public InputStream readFile(String pathStr) throws IOException {
+ incrementCounter(Statistic.OBJECTS_READ);
+ OFSPath ofsPath = new OFSPath(pathStr);
+ String key = ofsPath.getKeyName();
+ try {
+ OzoneBucket bucket = getBucket(ofsPath, false);
+ return bucket.readFile(key).getInputStream();
+ } catch (OMException ex) {
+ if (ex.getResult() == OMException.ResultCodes.FILE_NOT_FOUND
+ || ex.getResult() == OMException.ResultCodes.NOT_A_FILE) {
+ throw new FileNotFoundException(
+ ex.getResult().name() + ": " + ex.getMessage());
+ } else {
+ throw ex;
+ }
+ }
+ }
+
+ protected void incrementCounter(Statistic objectsRead) {
+ //noop: Use OzoneClientAdapterImpl which supports statistics.
+ }
+
+ @Override
+ public OzoneFSOutputStream createFile(String pathStr, boolean overWrite,
+ boolean recursive) throws IOException {
+ incrementCounter(Statistic.OBJECTS_CREATED);
+ OFSPath ofsPath = new OFSPath(pathStr);
+ String key = ofsPath.getKeyName();
+ try {
+ OzoneBucket bucket = getBucket(ofsPath, false);
+ OzoneOutputStream ozoneOutputStream = bucket.createFile(
+ key, 0, replicationType, replicationFactor, overWrite, recursive);
+ return new OzoneFSOutputStream(ozoneOutputStream.getOutputStream());
+ } catch (OMException ex) {
+ if (ex.getResult() == OMException.ResultCodes.FILE_ALREADY_EXISTS
+ || ex.getResult() == OMException.ResultCodes.NOT_A_FILE) {
+ throw new FileAlreadyExistsException(
+ ex.getResult().name() + ": " + ex.getMessage());
+ } else {
+ throw ex;
+ }
+ }
+ }
+
+ @Override
+ public void renamePath(String path, String newPath) throws IOException {
+ incrementCounter(Statistic.OBJECTS_RENAMED);
+ OFSPath ofsPath = new OFSPath(path);
+ OFSPath ofsNewPath = new OFSPath(newPath);
+
+ // Check path and newPathName are in the same volume and same bucket.
+ // This should have been checked in BasicRootedOzoneFileSystem#rename
+ // already via regular call path unless bypassed.
+ if (!ofsPath.isInSameBucketAs(ofsNewPath)) {
+ throw new IOException("Cannot rename a key to a different bucket");
+ }
+
+ OzoneBucket bucket = getBucket(ofsPath, false);
+ String key = ofsPath.getKeyName();
+ String newKey = ofsNewPath.getKeyName();
+ bucket.renameKey(key, newKey);
+ }
+
+ /**
+ * Package-private helper function to reduce calls to getBucket().
+ * @param bucket Bucket to operate in.
+ * @param path Existing key path.
+ * @param newPath New key path.
+ * @throws IOException IOException from bucket.renameKey().
+ */
+ void renamePath(OzoneBucket bucket, String path, String newPath)
+ throws IOException {
+ incrementCounter(Statistic.OBJECTS_RENAMED);
+ OFSPath ofsPath = new OFSPath(path);
+ OFSPath ofsNewPath = new OFSPath(newPath);
+ // No same-bucket policy check here since this call path is controlled
+ String key = ofsPath.getKeyName();
+ String newKey = ofsNewPath.getKeyName();
+ bucket.renameKey(key, newKey);
+ }
+
+ /**
+ * Helper method to create an directory specified by key name in bucket.
+ *
+ * @param pathStr path to be created as directory
+ * @return true if the key is created, false otherwise
+ */
+ @Override
+ public boolean createDirectory(String pathStr) throws IOException {
+ LOG.trace("creating dir for path: {}", pathStr);
+ incrementCounter(Statistic.OBJECTS_CREATED);
+ OFSPath ofsPath = new OFSPath(pathStr);
+ // Volume name unspecified, return failure
+ if (ofsPath.getVolumeName().length() == 0) {
+ return false;
+ }
+ // Handle where only volume is specified in pathStr
+ if (ofsPath.getBucketName().length() == 0) {
+ objectStore.createVolume(ofsPath.getVolumeName());
+ return true;
+ }
+ String keyStr = ofsPath.getKeyName();
+ try {
+ OzoneBucket bucket = getBucket(ofsPath, true);
+ // if keyStr is empty, it indicates that only volume or volume+bucket is
+ // given in pathStr, so getBucket() above should've handled the creation
+ // of volume/bucket already.
+ if (keyStr != null && keyStr.length() > 0) {
+ bucket.createDirectory(keyStr);
+ }
+ } catch (OMException e) {
+ if (e.getResult() == OMException.ResultCodes.FILE_ALREADY_EXISTS) {
+ throw new FileAlreadyExistsException(e.getMessage());
+ }
+ throw e;
+ }
+ return true;
+ }
+
+ /**
+ * Helper method to delete an object specified by key name in bucket.
+ *
+ * @param path path to a key to be deleted
+ * @return true if the key is deleted, false otherwise
+ */
+ @Override
+ public boolean deleteObject(String path) {
+ LOG.trace("issuing delete for path to key: {}", path);
+ OFSPath ofsPath = new OFSPath(path);
+ String keyName = ofsPath.getKeyName();
Review comment:
On supporting delete volume/bucket though, this is a good question. I'd argue it is weird to support delete volume/bucket via `rm` interface but from a flat filesystem point of view it should be a feature. What's you opinion? @arp7
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] hanishakoneru commented on a change in pull request
#415: HDDS-2840. Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
hanishakoneru commented on a change in pull request #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#discussion_r369304095
##########
File path: hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
##########
@@ -0,0 +1,395 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.TestDataUtil;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClientException;
+import org.apache.hadoop.ozone.client.OzoneKeyDetails;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.Set;
+import java.util.TreeSet;
+
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Ozone file system tests that are not covered by contract tests.
+ */
+public class TestRootedOzoneFileSystem {
+
+ @Rule
+ public Timeout globalTimeout = new Timeout(300_000);
+
+ private static MiniOzoneCluster cluster = null;
+
+ private static FileSystem fs;
+ private static RootedOzoneFileSystem ofs;
+
+ private static ObjectStore objectStore;
+
+ private String volumeName;
+ private String bucketName;
+
+ private String rootPath;
+
+ @Before
+ public void init() throws Exception {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ cluster = MiniOzoneCluster.newBuilder(conf)
+ .setNumDatanodes(3)
+ .build();
+ cluster.waitForClusterToBeReady();
+ objectStore = cluster.getClient().getObjectStore();
+
+ // create a volume and a bucket to be used by OFileSystem
+ OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(cluster);
+ volumeName = bucket.getVolumeName();
+ bucketName = bucket.getName();
+
+ // For now:
+ rootPath = String.format("%s://%s/", OzoneConsts.OZONE_OFS_URI_SCHEME,
+ conf.get(OZONE_OM_ADDRESS_KEY));
+
+ // Set the fs.defaultFS and start the filesystem
+ conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, rootPath);
+ // TODO: FileSystem#loadFileSystems is not loading ofs:// class by default
+ // hence this workaround. Might need to add some config in hadoop source.
+ conf.set("fs.ofs.impl", "org.apache.hadoop.fs.ozone.RootedOzoneFileSystem");
+ fs = FileSystem.get(conf);
+ ofs = (RootedOzoneFileSystem) fs;
+ }
+
+ @After
+ public void teardown() throws IOException {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ IOUtils.closeQuietly(fs);
+ }
+
+ @Test
+ public void testOzoneFsServiceLoader() throws IOException {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ // TODO: Address this properly.
+ conf.set("fs.ofs.impl", "org.apache.hadoop.fs.ozone.RootedOzoneFileSystem");
+ assertEquals(
+ FileSystem.getFileSystemClass(OzoneConsts.OZONE_OFS_URI_SCHEME,
+ conf), RootedOzoneFileSystem.class);
+ }
+
+ @Test
+ public void testCreateDoesNotAddParentDirKeys() throws Exception {
+ Path rootBucket = new Path("/" + volumeName + "/" + bucketName);
+ Path grandparent = new Path(rootBucket,
+ "testCreateDoesNotAddParentDirKeys");
+ Path parent = new Path(grandparent, "parent");
+ Path child = new Path(parent, "child");
+ ContractTestUtils.touch(fs, child);
+
+ OzoneKeyDetails key = getKeyInBucket(child, false);
+ OFSPath childOFSPath = new OFSPath(child);
+ assertEquals(key.getName(), childOFSPath.getKeyName());
+
+ // Creating a child should not add parent keys to the bucket
+ try {
+ getKeyInBucket(parent, true);
+ } catch (IOException ex) {
+ assertKeyNotFoundException(ex);
+ }
+
+ // List status on the parent should show the child file
+ assertEquals("List status of parent should include the 1 child file", 1L,
+ (long)fs.listStatus(parent).length);
+ assertTrue("Parent directory does not appear to be a directory",
+ fs.getFileStatus(parent).isDirectory());
+ }
+
+ @Test
+ public void testDeleteCreatesFakeParentDir() throws Exception {
+ Path rootBucket = new Path("/" + volumeName + "/" + bucketName);
+ Path grandparent = new Path(rootBucket, "testDeleteCreatesFakeParentDir");
+ Path parent = new Path(grandparent, "parent");
+ Path child = new Path(parent, "child");
+ ContractTestUtils.touch(fs, child);
+
+ // Verify that parent dir key does not exist
+ // Creating a child should not add parent keys to the bucket
+ try {
+ getKeyInBucket(parent, true);
+ } catch (IOException ex) {
+ assertKeyNotFoundException(ex);
+ }
+
+ // Delete the child key
+ assertTrue(fs.delete(child, false));
+
+ // Deleting the only child should create the parent dir key if it does
+ // not exist
+ OFSPath parentOFSPath = new OFSPath(parent);
+ String parentKey = parentOFSPath.getKeyName() + "/";
+ OzoneKeyDetails parentKeyInfo = getKeyInBucket(parent, true);
+ assertEquals(parentKey, parentKeyInfo.getName());
+
+ // Recursive delete with DeleteIterator
+ assertTrue(fs.delete(grandparent, true));
+ }
+
+ @Test
+ public void testListStatus() throws Exception {
+ Path rootBucket = new Path("/" + volumeName + "/" + bucketName);
+ Path parent = new Path(rootBucket, "testListStatus");
+ Path file1 = new Path(parent, "key1");
+ Path file2 = new Path(parent, "key2");
+
+ FileStatus[] fileStatuses = ofs.listStatus(rootBucket);
+ assertEquals("Should be empty", 0, fileStatuses.length);
Review comment:
We can add one more check later that listStatus(rootBucket) returns 1 fileStatus object - parent.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] smengcl commented on a change in pull request #415:
HDDS-2840. Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
smengcl commented on a change in pull request #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#discussion_r371205593
##########
File path: hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
##########
@@ -0,0 +1,805 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.ozone;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.CreateFlag;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.Progressable;
+import org.apache.http.client.utils.URIBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.EnumSet;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+import static org.apache.hadoop.fs.ozone.Constants.LISTING_PAGE_SIZE;
+import static org.apache.hadoop.fs.ozone.Constants.OZONE_DEFAULT_USER;
+import static org.apache.hadoop.fs.ozone.Constants.OZONE_USER_DIR;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_OFS_URI_SCHEME;
+
+/**
+ * The minimal Ozone Filesystem implementation.
+ * <p>
+ * This is a basic version which doesn't extend
+ * KeyProviderTokenIssuer and doesn't include statistics. It can be used
+ * from older hadoop version. For newer hadoop version use the full featured
+ * OFileSystem.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class BasicRootedOzoneFileSystem extends FileSystem {
+ static final Logger LOG =
+ LoggerFactory.getLogger(BasicRootedOzoneFileSystem.class);
+
+ /**
+ * The Ozone client for connecting to Ozone server.
+ */
+
+ private URI uri;
+ private String userName;
+ private Path workingDir;
+// private String gOmHost;
+// private int gOmPort;
+// private Configuration gConf;
+// private boolean gIsolatedClassloader;
+
+ private RootedOzoneClientAdapter adapter;
+// private String adapterPath;
+
+ private static final String URI_EXCEPTION_TEXT =
+ "URL should be one of the following formats: " +
+ "ofs://om-service-id/ OR " +
+ "ofs://om-host.example.com/ OR " +
+ "ofs://om-host.example.com:5678/";
+
+ @Override
+ public void initialize(URI name, Configuration conf) throws IOException {
+ super.initialize(name, conf);
+ setConf(conf);
+ Objects.requireNonNull(name.getScheme(), "No scheme provided in " + name);
+ Preconditions.checkArgument(getScheme().equals(name.getScheme()),
+ "Invalid scheme provided in " + name);
+
+ String authority = name.getAuthority();
+ if (authority == null) {
+ // authority is null when fs.defaultFS is not a qualified ofs URI and
+ // ofs:/// is passed to the client. matcher will NPE if authority is null
+ throw new IllegalArgumentException(URI_EXCEPTION_TEXT);
+ }
+
+ String omHost;
+ int omPort = -1;
+ // Parse hostname and port
+ String[] parts = authority.split(":");
+ if (parts.length > 2) {
+ throw new IllegalArgumentException(URI_EXCEPTION_TEXT);
+ }
+ omHost = parts[0];
+ if (parts.length == 2) {
+ try {
+ omPort = Integer.parseInt(parts[1]);
+ } catch (NumberFormatException e) {
+ throw new IllegalArgumentException(URI_EXCEPTION_TEXT);
+ }
+ }
+
+ try {
+ uri = new URIBuilder().setScheme(OZONE_OFS_URI_SCHEME)
+ .setHost(authority)
+ .build();
+ LOG.trace("Ozone URI for OFS initialization is " + uri);
+
+ //isolated is the default for ozonefs-lib-legacy which includes the
+ // /ozonefs.txt, otherwise the default is false. It could be overridden.
+ boolean defaultValue =
+ BasicRootedOzoneFileSystem.class.getClassLoader()
+ .getResource("ozonefs.txt") != null;
+
+ //Use string here instead of the constant as constant may not be available
+ //on the classpath of a hadoop 2.7
+ boolean isolatedClassloader =
+ conf.getBoolean("ozone.fs.isolated-classloader", defaultValue);
+
+ // adapter should be initialized in operations.
+ this.adapter = createAdapter(conf, omHost, omPort, isolatedClassloader);
+
+ try {
+ this.userName =
+ UserGroupInformation.getCurrentUser().getShortUserName();
+ } catch (IOException e) {
+ this.userName = OZONE_DEFAULT_USER;
+ }
+ this.workingDir = new Path(OZONE_USER_DIR, this.userName)
+ .makeQualified(this.uri, this.workingDir);
+ } catch (URISyntaxException ue) {
+ final String msg = "Invalid Ozone endpoint " + name;
+ LOG.error(msg, ue);
+ throw new IOException(msg, ue);
+ }
+ }
+
+ protected RootedOzoneClientAdapter createAdapter(Configuration conf,
+ String omHost, int omPort, boolean isolatedClassloader)
+ throws IOException {
+
+ if (isolatedClassloader) {
+ // TODO: Check how this code path need to be changed, for legacy Hadoop?
+ return RootedOzoneClientAdapterFactory.createAdapter();
+ } else {
+ return new BasicRootedOzoneClientAdapterImpl(omHost, omPort, conf);
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ try {
+ adapter.close();
+ } finally {
+ super.close();
+ }
+ }
+
+ @Override
+ public URI getUri() {
+ return uri;
+ }
+
+ @Override
+ public String getScheme() {
+ return OZONE_OFS_URI_SCHEME;
+ }
+
+ @Override
+ public FSDataInputStream open(Path path, int bufferSize) throws IOException {
+ incrementCounter(Statistic.INVOCATION_OPEN);
+ statistics.incrementReadOps(1);
+ LOG.trace("open() path: {}", path);
+ final String key = pathToKey(path);
+ return new FSDataInputStream(
+ new OzoneFSInputStream(adapter.readFile(key), statistics));
+ }
+
+ protected void incrementCounter(Statistic statistic) {
+ //don't do anything in this default implementation.
+ }
+
+ @Override
+ public FSDataOutputStream create(Path f, FsPermission permission,
+ boolean overwrite, int bufferSize,
+ short replication, long blockSize,
+ Progressable progress) throws IOException {
+ LOG.trace("create() path:{}", f);
+ incrementCounter(Statistic.INVOCATION_CREATE);
+ statistics.incrementWriteOps(1);
+ final String key = pathToKey(f);
+ return createOutputStream(key, overwrite, true);
+ }
+
+ @Override
+ public FSDataOutputStream createNonRecursive(Path path,
+ FsPermission permission,
+ EnumSet<CreateFlag> flags,
+ int bufferSize,
+ short replication,
+ long blockSize,
+ Progressable progress) throws IOException {
+ incrementCounter(Statistic.INVOCATION_CREATE_NON_RECURSIVE);
+ statistics.incrementWriteOps(1);
+ final String key = pathToKey(path);
+ return createOutputStream(key, flags.contains(CreateFlag.OVERWRITE), false);
+ }
+
+ private FSDataOutputStream createOutputStream(String key, boolean overwrite,
+ boolean recursive) throws IOException {
+ return new FSDataOutputStream(adapter.createFile(key, overwrite, recursive),
+ statistics);
+ }
+
+ @Override
+ public FSDataOutputStream append(Path f, int bufferSize,
+ Progressable progress) throws IOException {
+ throw new UnsupportedOperationException("append() Not implemented by the "
+ + getClass().getSimpleName() + " FileSystem implementation");
+ }
+
+ private class RenameIterator extends OzoneListingIterator {
+ private final String srcPath;
+ private final String dstPath;
+
+ RenameIterator(Path srcPath, Path dstPath)
+ throws IOException {
+ super(srcPath);
+ this.srcPath = pathToKey(srcPath);
+ this.dstPath = pathToKey(dstPath);
+ LOG.trace("rename from:{} to:{}", this.srcPath, this.dstPath);
+ }
+
+ @Override
+ boolean processKeyPath(String keyPath) throws IOException {
+ String newPath = dstPath.concat(keyPath.substring(srcPath.length()));
+ adapter.renamePath(keyPath, newPath);
+ return true;
+ }
+ }
+
+ /**
+ * Check whether the source and destination path are valid and then perform
+ * rename from source path to destination path.
+ * <p>
+ * The rename operation is performed by renaming the keys with src as prefix.
+ * For such keys the prefix is changed from src to dst.
+ *
+ * @param src source path for rename
+ * @param dst destination path for rename
+ * @return true if rename operation succeeded or
+ * if the src and dst have the same path and are of the same type
+ * @throws IOException on I/O errors or if the src/dst paths are invalid.
+ */
+ @Override
+ public boolean rename(Path src, Path dst) throws IOException {
+ incrementCounter(Statistic.INVOCATION_RENAME);
+ statistics.incrementWriteOps(1);
+ if (src.equals(dst)) {
+ return true;
+ }
+
+ LOG.trace("rename() from: {} to: {}", src, dst);
+ if (src.isRoot()) {
+ // Cannot rename root of file system
+ LOG.trace("Cannot rename the root of a filesystem");
+ return false;
+ }
+
+ // src and dst should be in the same bucket
+ OFSPath ofsSrc = new OFSPath(src);
+ OFSPath ofsDst = new OFSPath(dst);
+ if (!ofsSrc.isInSameBucketAs(ofsDst)) {
+ throw new IOException("Cannot rename a key to a different bucket");
+ }
+
+ // Cannot rename a directory to its own subdirectory
+ Path dstParent = dst.getParent();
+ while (dstParent != null && !src.equals(dstParent)) {
+ dstParent = dstParent.getParent();
+ }
+ Preconditions.checkArgument(dstParent == null,
+ "Cannot rename a directory to its own subdirectory");
+ // Check if the source exists
+ FileStatus srcStatus;
+ try {
+ srcStatus = getFileStatus(src);
+ } catch (FileNotFoundException fnfe) {
+ // source doesn't exist, return
+ return false;
+ }
+
+ // Check if the destination exists
+ FileStatus dstStatus;
+ try {
+ dstStatus = getFileStatus(dst);
+ } catch (FileNotFoundException fnde) {
+ dstStatus = null;
+ }
+
+ if (dstStatus == null) {
+ // If dst doesn't exist, check whether dst parent dir exists or not
+ // if the parent exists, the source can still be renamed to dst path
+ dstStatus = getFileStatus(dst.getParent());
+ if (!dstStatus.isDirectory()) {
+ throw new IOException(String.format(
+ "Failed to rename %s to %s, %s is a file", src, dst,
+ dst.getParent()));
+ }
+ } else {
+ // if dst exists and source and destination are same,
+ // check both the src and dst are of same type
+ if (srcStatus.getPath().equals(dstStatus.getPath())) {
+ return !srcStatus.isDirectory();
+ } else if (dstStatus.isDirectory()) {
+ // If dst is a directory, rename source as subpath of it.
+ // for example rename /source to /dst will lead to /dst/source
+ dst = new Path(dst, src.getName());
+ FileStatus[] statuses;
+ try {
+ statuses = listStatus(dst);
+ } catch (FileNotFoundException fnde) {
+ statuses = null;
+ }
+
+ if (statuses != null && statuses.length > 0) {
+ // If dst exists and not a directory not empty
+ throw new FileAlreadyExistsException(String.format(
+ "Failed to rename %s to %s, file already exists or not empty!",
+ src, dst));
+ }
+ } else {
+ // If dst is not a directory
+ throw new FileAlreadyExistsException(String.format(
+ "Failed to rename %s to %s, file already exists!", src, dst));
+ }
+ }
+
+ if (srcStatus.isDirectory()) {
+ if (dst.toString().startsWith(src.toString() + OZONE_URI_DELIMITER)) {
+ LOG.trace("Cannot rename a directory to a subdirectory of self");
+ return false;
+ }
+ }
+ RenameIterator iterator = new RenameIterator(src, dst);
+ boolean result = iterator.iterate();
+ if (result) {
+ createFakeParentDirectory(src);
+ }
+ return result;
+ }
+
+ private class DeleteIterator extends OzoneListingIterator {
+ private boolean recursive;
+
+ DeleteIterator(Path f, boolean recursive)
+ throws IOException {
+ super(f);
+ this.recursive = recursive;
+ if (getStatus().isDirectory()
+ && !this.recursive
+ && listStatus(f).length != 0) {
+ throw new PathIsNotEmptyDirectoryException(f.toString());
+ }
+ }
+
+ @Override
+ boolean processKeyPath(String keyPath) throws IOException {
+ if (keyPath.equals("")) {
+ LOG.trace("Skipping deleting root directory");
+ return true;
+ } else {
+ LOG.trace("deleting key path:" + keyPath);
+ boolean succeed = adapter.deleteObject(keyPath);
+ // if recursive delete is requested ignore the return value of
+ // deleteObject and issue deletes for other keys.
+ return recursive || succeed;
+ }
+ }
+ }
+
+ /**
+ * Deletes the children of the input dir path by iterating though the
+ * DeleteIterator.
+ *
+ * @param f directory path to be deleted
+ * @return true if successfully deletes all required keys, false otherwise
+ * @throws IOException
+ */
+ private boolean innerDelete(Path f, boolean recursive) throws IOException {
+ LOG.trace("delete() path:{} recursive:{}", f, recursive);
+ try {
+ DeleteIterator iterator = new DeleteIterator(f, recursive);
+ return iterator.iterate();
+ } catch (FileNotFoundException e) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Couldn't delete {} - does not exist", f);
+ }
+ return false;
+ }
+ }
+
+ @Override
+ public boolean delete(Path f, boolean recursive) throws IOException {
+ incrementCounter(Statistic.INVOCATION_DELETE);
+ statistics.incrementWriteOps(1);
+ LOG.debug("Delete path {} - recursive {}", f, recursive);
+ FileStatus status;
+ try {
+ status = getFileStatus(f);
+ } catch (FileNotFoundException ex) {
+ LOG.warn("delete: Path does not exist: {}", f);
+ return false;
+ }
+
+ String key = pathToKey(f);
+ boolean result;
+
+ if (status.isDirectory()) {
+ LOG.debug("delete: Path is a directory: {}", f);
+ key = addTrailingSlashIfNeeded(key);
+
+ if (key.equals("/")) {
+ LOG.warn("Cannot delete root directory.");
+ return false;
+ }
+
+ result = innerDelete(f, recursive);
+ } else {
+ LOG.debug("delete: Path is a file: {}", f);
+ result = adapter.deleteObject(key);
+ }
+
+ if (result) {
+ // If this delete operation removes all files/directories from the
+ // parent direcotry, then an empty parent directory must be created.
+ createFakeParentDirectory(f);
+ }
+
+ return result;
+ }
+
+ /**
+ * Create a fake parent directory key if it does not already exist and no
+ * other child of this parent directory exists.
+ *
+ * @param f path to the fake parent directory
+ * @throws IOException
+ */
+ private void createFakeParentDirectory(Path f) throws IOException {
+ Path parent = f.getParent();
+ if (parent != null && !parent.isRoot()) {
+ createFakeDirectoryIfNecessary(parent);
+ }
+ }
+
+ /**
+ * Create a fake directory key if it does not already exist.
+ *
+ * @param f path to the fake directory
+ * @throws IOException
+ */
+ private void createFakeDirectoryIfNecessary(Path f) throws IOException {
+ String key = pathToKey(f);
+ if (!key.isEmpty() && !o3Exists(f)) {
+ LOG.debug("Creating new fake directory at {}", f);
+ String dirKey = addTrailingSlashIfNeeded(key);
+ adapter.createDirectory(dirKey);
+ }
+ }
+
+ /**
+ * Check if a file or directory exists corresponding to given path.
+ *
+ * @param f path to file/directory.
+ * @return true if it exists, false otherwise.
+ * @throws IOException
+ */
+ private boolean o3Exists(final Path f) throws IOException {
+ Path path = makeQualified(f);
+ try {
+ getFileStatus(path);
+ return true;
+ } catch (FileNotFoundException ex) {
+ return false;
+ }
+ }
+
+ @Override
+ public FileStatus[] listStatus(Path f) throws IOException {
+ incrementCounter(Statistic.INVOCATION_LIST_STATUS);
+ statistics.incrementReadOps(1);
+ LOG.trace("listStatus() path:{}", f);
+ int numEntries = LISTING_PAGE_SIZE;
+ LinkedList<FileStatus> statuses = new LinkedList<>();
+ List<FileStatus> tmpStatusList;
+ String startPath = "";
+
+ do {
+ tmpStatusList =
+ adapter.listStatus(pathToKey(f), false, startPath,
+ numEntries, uri, workingDir, getUsername())
+ .stream()
+ .map(this::convertFileStatus)
+ .collect(Collectors.toList());
+
+ if (!tmpStatusList.isEmpty()) {
+ if (startPath.isEmpty()) {
+ statuses.addAll(tmpStatusList);
+ } else {
+ statuses.addAll(tmpStatusList.subList(1, tmpStatusList.size()));
+ }
+ startPath = pathToKey(statuses.getLast().getPath());
+ }
+ // listStatus returns entries numEntries in size if available.
+ // Any lesser number of entries indicate that the required entries have
+ // exhausted.
+ } while (tmpStatusList.size() == numEntries);
+
+ return statuses.toArray(new FileStatus[0]);
+ }
+
+ @Override
+ public void setWorkingDirectory(Path newDir) {
+ workingDir = newDir;
+ }
+
+ @Override
+ public Path getWorkingDirectory() {
+ return workingDir;
+ }
+
+ @Override
+ public Token<?> getDelegationToken(String renewer) throws IOException {
+ return adapter.getDelegationToken(renewer);
+ }
+
+ /**
+ * Get a canonical service name for this file system. If the URI is logical,
+ * the hostname part of the URI will be returned.
+ *
+ * @return a service string that uniquely identifies this file system.
+ */
+ @Override
+ public String getCanonicalServiceName() {
+ return adapter.getCanonicalServiceName();
+ }
+
+ /**
+ * Get the username of the FS.
+ *
+ * @return the short name of the user who instantiated the FS
+ */
+ public String getUsername() {
+ return userName;
+ }
+
+ /**
+ * Creates a directory. Directory is represented using a key with no value.
+ *
+ * @param path directory path to be created
+ * @return true if directory exists or created successfully.
+ * @throws IOException
+ */
+ private boolean mkdir(Path path) throws IOException {
+ return adapter.createDirectory(pathToKey(path));
+ }
+
+ @Override
+ public boolean mkdirs(Path f, FsPermission permission) throws IOException {
+ LOG.trace("mkdir() path:{} ", f);
+ String key = pathToKey(f);
+ if (isEmpty(key)) {
+ return false;
+ }
+ return mkdir(f);
+ }
+
+ @Override
+ public FileStatus getFileStatus(Path f) throws IOException {
+ incrementCounter(Statistic.INVOCATION_GET_FILE_STATUS);
+ statistics.incrementReadOps(1);
+ LOG.trace("getFileStatus() path:{}", f);
+ Path qualifiedPath = f.makeQualified(uri, workingDir);
+ String key = pathToKey(qualifiedPath);
+ FileStatus fileStatus = null;
+ try {
+ fileStatus = convertFileStatus(
+ adapter.getFileStatus(key, uri, qualifiedPath, getUsername()));
+ } catch (OMException ex) {
+ if (ex.getResult().equals(OMException.ResultCodes.KEY_NOT_FOUND)) {
+ throw new FileNotFoundException("File not found. path:" + f);
+ }
+ }
+ return fileStatus;
+ }
+
+ @Override
+ public BlockLocation[] getFileBlockLocations(FileStatus fileStatus,
+ long start, long len)
+ throws IOException {
+ if (fileStatus instanceof LocatedFileStatus) {
+ return ((LocatedFileStatus) fileStatus).getBlockLocations();
+ } else {
+ return super.getFileBlockLocations(fileStatus, start, len);
+ }
+ }
+
+ /**
+ * Turn a path (relative or otherwise) into an Ozone key.
+ *
+ * @param path the path of the file.
+ * @return the key of the object that represents the file.
+ */
+ public String pathToKey(Path path) {
+ Objects.requireNonNull(path, "Path can't be null!");
Review comment:
This `pathToKey` is added in HDDS-177 by Anu. Honestly what I did here was fixing the typo.
Your understanding is right. It should convert a relative path into an absolute one (if not). Then remove leading `/`.
But I think defaulting volume to `user` and bucket name to `<USERNAME>` in this case is exactly the way we intend it to be? So when user run `ozone fs -put textfile.txt doc/` should automatically put the file under `doc/` in user's home directory. (`workingDir` should be renamed to `homeDir` IMO) CMIIW.
Yes we are assuming that admin would create a volume named `user` and created a bucket for each user beforehand. That is indeed the implication. Let's discuss it.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] hanishakoneru commented on a change in pull request
#415: HDDS-2840. Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
hanishakoneru commented on a change in pull request #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#discussion_r369325484
##########
File path: hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
##########
@@ -0,0 +1,604 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.ozone;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.key.KeyProvider;
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdds.client.ReplicationFactor;
+import org.apache.hadoop.hdds.client.ReplicationType;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ozone.OmUtils;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.client.OzoneKey;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
+import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
+import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenRenewer;
+
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .VOLUME_NOT_FOUND;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .BUCKET_NOT_FOUND;
+
+/**
+ * Basic Implementation of the OzoneFileSystem calls.
+ * <p>
+ * This is the minimal version which doesn't include any statistics.
+ * <p>
+ * For full featured version use OzoneClientAdapterImpl.
+ */
+public class BasicRootedOzoneClientAdapterImpl
+ implements RootedOzoneClientAdapter {
+
+ static final Logger LOG =
+ LoggerFactory.getLogger(BasicRootedOzoneClientAdapterImpl.class);
+
+ private OzoneClient ozoneClient;
+ private ClientProtocol proxy;
+ private ObjectStore objectStore;
+ private ReplicationType replicationType;
+ private ReplicationFactor replicationFactor;
+ private boolean securityEnabled;
+ private int configuredDnPort;
+
+ /**
+ * Create new OzoneClientAdapter implementation.
+ *
+ * @throws IOException In case of a problem.
+ */
+ public BasicRootedOzoneClientAdapterImpl() throws IOException {
+ this(createConf());
+ }
+
+ private static OzoneConfiguration createConf() {
+ ClassLoader contextClassLoader =
+ Thread.currentThread().getContextClassLoader();
+ Thread.currentThread().setContextClassLoader(null);
+ try {
+ return new OzoneConfiguration();
+ } finally {
+ Thread.currentThread().setContextClassLoader(contextClassLoader);
+ }
+ }
+
+ public BasicRootedOzoneClientAdapterImpl(OzoneConfiguration conf)
+ throws IOException {
+ this(null, -1, conf);
+ }
+
+ public BasicRootedOzoneClientAdapterImpl(String omHost, int omPort,
+ Configuration hadoopConf) throws IOException {
+
+ ClassLoader contextClassLoader =
+ Thread.currentThread().getContextClassLoader();
+ Thread.currentThread().setContextClassLoader(null);
+
+ try {
+ OzoneConfiguration conf = OzoneConfiguration.of(hadoopConf);
+
+ if (omHost == null && OmUtils.isServiceIdsDefined(conf)) {
+ // When the host name or service id isn't given
+ // but ozone.om.service.ids is defined, declare failure.
+
+ // This is a safety precaution that prevents the client from
+ // accidentally failing over to an unintended OM.
+ throw new IllegalArgumentException("Service ID or host name must not"
+ + " be omitted when ozone.om.service.ids is defined.");
+ }
+
+ if (omPort != -1) {
+ // When the port number is specified, perform the following check
+ if (OmUtils.isOmHAServiceId(conf, omHost)) {
+ // If omHost is a service id, it shouldn't use a port
+ throw new IllegalArgumentException("Port " + omPort +
+ " specified in URI but host '" + omHost + "' is a "
+ + "logical (HA) OzoneManager and does not use port information.");
+ }
+ } else {
+ // When port number is not specified, read it from config
+ omPort = OmUtils.getOmRpcPort(conf);
+ }
+
+ SecurityConfig secConfig = new SecurityConfig(conf);
+
+ if (secConfig.isSecurityEnabled()) {
+ this.securityEnabled = true;
+ }
+
+ String replicationTypeConf =
+ conf.get(OzoneConfigKeys.OZONE_REPLICATION_TYPE,
+ OzoneConfigKeys.OZONE_REPLICATION_TYPE_DEFAULT);
+
+ int replicationCountConf = conf.getInt(OzoneConfigKeys.OZONE_REPLICATION,
+ OzoneConfigKeys.OZONE_REPLICATION_DEFAULT);
+
+ if (OmUtils.isOmHAServiceId(conf, omHost)) {
+ // omHost is listed as one of the service ids in the config,
+ // thus we should treat omHost as omServiceId
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(omHost, conf);
+ } else if (StringUtils.isNotEmpty(omHost) && omPort != -1) {
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(omHost, omPort, conf);
+ } else {
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(conf);
+ }
+ objectStore = ozoneClient.getObjectStore();
+ proxy = objectStore.getClientProxy();
+ this.replicationType = ReplicationType.valueOf(replicationTypeConf);
+ this.replicationFactor = ReplicationFactor.valueOf(replicationCountConf);
+ this.configuredDnPort = conf.getInt(
+ OzoneConfigKeys.DFS_CONTAINER_IPC_PORT,
+ OzoneConfigKeys.DFS_CONTAINER_IPC_PORT_DEFAULT);
+ } finally {
+ Thread.currentThread().setContextClassLoader(contextClassLoader);
+ }
+ }
+
+ private OzoneBucket getBucket(OFSPath ofsPath,
+ boolean createIfNotExist) throws IOException {
+
+ return getBucket(ofsPath.getVolumeName(), ofsPath.getBucketName(),
+ createIfNotExist);
+ }
+
+ /**
+ * Get OzoneBucket object to operate in.
+ * Optionally create volume and bucket if not found.
+ *
+ * @param createIfNotExist Set this to true if the caller is a write operation
+ * in order to create the volume and bucket.
+ * @throws IOException Exceptions other than OMException with result code
+ * VOLUME_NOT_FOUND or BUCKET_NOT_FOUND.
+ */
+ private OzoneBucket getBucket(String volumeStr, String bucketStr,
+ boolean createIfNotExist) throws IOException {
+
+ OzoneBucket bucket;
+ try {
+ bucket = proxy.getBucketDetails(volumeStr, bucketStr);
+ } catch (OMException ex) {
+ if (createIfNotExist) {
+ // Note: getBucketDetails always throws BUCKET_NOT_FOUND, even if
+ // the volume doesn't exist.
+ if (ex.getResult().equals(BUCKET_NOT_FOUND)) {
+ OzoneVolume volume;
+ try {
+ volume = proxy.getVolumeDetails(volumeStr);
+ } catch (OMException volEx) {
+ if (volEx.getResult().equals(VOLUME_NOT_FOUND)) {
+ // Volume doesn't exist. Create it
+ objectStore.createVolume(volumeStr);
+ } else {
+ throw volEx;
+ }
+ // Try get volume again
+ volume = proxy.getVolumeDetails(volumeStr);
+ }
+ // Create the bucket
+ volume.createBucket(bucketStr);
+ }
+ // Try get bucket again
+ bucket = proxy.getBucketDetails(volumeStr, bucketStr);
+ } else {
+ throw ex;
+ }
+ }
+
+ return bucket;
+ }
+
+ @Override
+ public void close() throws IOException {
+ ozoneClient.close();
+ }
+
+ @Override
+ public InputStream readFile(String pathStr) throws IOException {
+ incrementCounter(Statistic.OBJECTS_READ);
+ OFSPath ofsPath = new OFSPath(pathStr);
+ String key = ofsPath.getKeyName();
+ try {
+ OzoneBucket bucket = getBucket(ofsPath, false);
+ return bucket.readFile(key).getInputStream();
+ } catch (OMException ex) {
+ if (ex.getResult() == OMException.ResultCodes.FILE_NOT_FOUND
+ || ex.getResult() == OMException.ResultCodes.NOT_A_FILE) {
+ throw new FileNotFoundException(
+ ex.getResult().name() + ": " + ex.getMessage());
+ } else {
+ throw ex;
+ }
+ }
+ }
+
+ protected void incrementCounter(Statistic objectsRead) {
+ //noop: Use OzoneClientAdapterImpl which supports statistics.
+ }
+
+ @Override
+ public OzoneFSOutputStream createFile(String pathStr, boolean overWrite,
+ boolean recursive) throws IOException {
+ incrementCounter(Statistic.OBJECTS_CREATED);
+ OFSPath ofsPath = new OFSPath(pathStr);
+ String key = ofsPath.getKeyName();
+ try {
+ OzoneBucket bucket = getBucket(ofsPath, false);
+ OzoneOutputStream ozoneOutputStream = bucket.createFile(
+ key, 0, replicationType, replicationFactor, overWrite, recursive);
+ return new OzoneFSOutputStream(ozoneOutputStream.getOutputStream());
+ } catch (OMException ex) {
+ if (ex.getResult() == OMException.ResultCodes.FILE_ALREADY_EXISTS
+ || ex.getResult() == OMException.ResultCodes.NOT_A_FILE) {
+ throw new FileAlreadyExistsException(
+ ex.getResult().name() + ": " + ex.getMessage());
+ } else {
+ throw ex;
+ }
+ }
+ }
+
+ @Override
+ public void renamePath(String path, String newPath) throws IOException {
+ incrementCounter(Statistic.OBJECTS_RENAMED);
+ OFSPath ofsPath = new OFSPath(path);
+ OFSPath ofsNewPath = new OFSPath(newPath);
+
+ // Check path and newPathName are in the same volume and same bucket.
+ // This should have been checked in BasicRootedOzoneFileSystem#rename
+ // already via regular call path unless bypassed.
+ if (!ofsPath.isInSameBucketAs(ofsNewPath)) {
+ throw new IOException("Cannot rename a key to a different bucket");
+ }
+
+ OzoneBucket bucket = getBucket(ofsPath, false);
+ String key = ofsPath.getKeyName();
+ String newKey = ofsNewPath.getKeyName();
+ bucket.renameKey(key, newKey);
+ }
+
+ /**
+ * Helper method to create an directory specified by key name in bucket.
+ *
+ * @param pathStr path to be created as directory
+ * @return true if the key is created, false otherwise
+ */
+ @Override
+ public boolean createDirectory(String pathStr) throws IOException {
+ LOG.trace("creating dir for path: {}", pathStr);
+ incrementCounter(Statistic.OBJECTS_CREATED);
+ OFSPath ofsPath = new OFSPath(pathStr);
+ String keyStr = ofsPath.getKeyName();
+ try {
+ OzoneBucket bucket = getBucket(ofsPath, true);
+ bucket.createDirectory(keyStr);
+ } catch (OMException e) {
+ if (e.getResult() == OMException.ResultCodes.FILE_ALREADY_EXISTS) {
+ throw new FileAlreadyExistsException(e.getMessage());
+ }
+ throw e;
+ }
+ return true;
+ }
+
+ /**
+ * Helper method to delete an object specified by key name in bucket.
+ *
+ * @param path path to a key to be deleted
+ * @return true if the key is deleted, false otherwise
+ */
+ @Override
+ public boolean deleteObject(String path) {
+ LOG.trace("issuing delete for path to key: {}", path);
+ OFSPath ofsPath = new OFSPath(path);
+ String keyName = ofsPath.getKeyName();
+ try {
+ OzoneBucket bucket = getBucket(ofsPath, false);
+ incrementCounter(Statistic.OBJECTS_DELETED);
+ bucket.deleteKey(keyName);
+ return true;
+ } catch (IOException ioe) {
+ LOG.error("delete key failed " + ioe.getMessage());
+ return false;
+ }
+ }
+
+ public FileStatusAdapter getFileStatus(String path, URI uri,
+ Path qualifiedPath, String userName)
+ throws IOException {
+ OFSPath ofsPath = new OFSPath(path);
+ String key = ofsPath.getKeyName();
+ try {
+ OzoneBucket bucket = getBucket(ofsPath, false);
+ incrementCounter(Statistic.OBJECTS_QUERY);
+ OzoneFileStatus status = bucket.getFileStatus(key);
+ // Note: qualifiedPath passed in is good from
+ // BasicRootedOzoneFileSystem#getFileStatus. No need to prepend here.
+ makeQualified(status, uri, qualifiedPath, userName);
+ return toFileStatusAdapter(status);
+
+ } catch (OMException e) {
+ if (e.getResult() == OMException.ResultCodes.FILE_NOT_FOUND) {
+ throw new
+ FileNotFoundException(key + ": No such file or directory!");
+ }
+ throw e;
+ }
+ }
+
+ public void makeQualified(FileStatus status, URI uri, Path path,
+ String username) {
+ if (status instanceof OzoneFileStatus) {
+ ((OzoneFileStatus) status)
+ .makeQualified(uri, path,
+ username, username);
+ }
+ }
+
+ @Override
+ public Iterator<BasicKeyInfo> listKeys(String pathStr) throws IOException {
+ incrementCounter(Statistic.OBJECTS_LIST);
+ OFSPath ofsPath = new OFSPath(pathStr);
+ String key = ofsPath.getKeyName();
+ OzoneBucket bucket = getBucket(ofsPath, false);
+ return new IteratorAdapter(bucket.listKeys(key));
+ }
+
+ public List<FileStatusAdapter> listStatus(String pathStr, boolean recursive,
+ String startPath, long numEntries, URI uri,
+ Path workingDir, String username) throws IOException {
+
+ OFSPath ofsPath = new OFSPath(pathStr);
+ String keyName = ofsPath.getKeyName();
+ OFSPath ofsStartPath = new OFSPath(startPath);
+ String startKey = ofsStartPath.getKeyName();
Review comment:
Do we want to convert startPath also to ofsPath here? Most likely, the startPath would only have the keyName prefix as the full path with volume and bucket is already given in pathStr.
I am ok either way but we should add JavaDocs to explain this. Otherwise, user may assume it to be keyName prefix but we take it as volume/bucket/key or vice versa.
It would be good to add JavaDoc to all the methods. Atleast all the methods in OzoneClientAdapter should have a JavaDoc as they are client facing.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] smengcl commented on a change in pull request #415:
HDDS-2840. Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
smengcl commented on a change in pull request #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#discussion_r372549616
##########
File path: hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
##########
@@ -0,0 +1,694 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.ozone;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.key.KeyProvider;
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdds.client.ReplicationFactor;
+import org.apache.hadoop.hdds.client.ReplicationType;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ozone.OmUtils;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.client.OzoneKey;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
+import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
+import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenRenewer;
+
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .BUCKET_ALREADY_EXISTS;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .VOLUME_ALREADY_EXISTS;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .VOLUME_NOT_FOUND;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .BUCKET_NOT_FOUND;
+
+/**
+ * Basic Implementation of the OzoneFileSystem calls.
+ * <p>
+ * This is the minimal version which doesn't include any statistics.
+ * <p>
+ * For full featured version use OzoneClientAdapterImpl.
+ */
+public class BasicRootedOzoneClientAdapterImpl
+ implements RootedOzoneClientAdapter {
+
+ static final Logger LOG =
+ LoggerFactory.getLogger(BasicRootedOzoneClientAdapterImpl.class);
+
+ private OzoneClient ozoneClient;
+ private ClientProtocol proxy;
+ private ObjectStore objectStore;
+ private ReplicationType replicationType;
+ private ReplicationFactor replicationFactor;
+ private boolean securityEnabled;
+ private int configuredDnPort;
+
+ /**
+ * Create new OzoneClientAdapter implementation.
+ *
+ * @throws IOException In case of a problem.
+ */
+ public BasicRootedOzoneClientAdapterImpl() throws IOException {
+ this(createConf());
+ }
+
+ private static OzoneConfiguration createConf() {
+ ClassLoader contextClassLoader =
+ Thread.currentThread().getContextClassLoader();
+ Thread.currentThread().setContextClassLoader(null);
+ try {
+ return new OzoneConfiguration();
+ } finally {
+ Thread.currentThread().setContextClassLoader(contextClassLoader);
+ }
+ }
+
+ public BasicRootedOzoneClientAdapterImpl(OzoneConfiguration conf)
+ throws IOException {
+ this(null, -1, conf);
+ }
+
+ public BasicRootedOzoneClientAdapterImpl(String omHost, int omPort,
+ Configuration hadoopConf) throws IOException {
+
+ ClassLoader contextClassLoader =
+ Thread.currentThread().getContextClassLoader();
+ Thread.currentThread().setContextClassLoader(null);
+
+ try {
+ OzoneConfiguration conf = OzoneConfiguration.of(hadoopConf);
+
+ if (omHost == null && OmUtils.isServiceIdsDefined(conf)) {
+ // When the host name or service id isn't given
+ // but ozone.om.service.ids is defined, declare failure.
+
+ // This is a safety precaution that prevents the client from
+ // accidentally failing over to an unintended OM.
+ throw new IllegalArgumentException("Service ID or host name must not"
+ + " be omitted when ozone.om.service.ids is defined.");
+ }
+
+ if (omPort != -1) {
+ // When the port number is specified, perform the following check
+ if (OmUtils.isOmHAServiceId(conf, omHost)) {
+ // If omHost is a service id, it shouldn't use a port
+ throw new IllegalArgumentException("Port " + omPort +
+ " specified in URI but host '" + omHost + "' is a "
+ + "logical (HA) OzoneManager and does not use port information.");
+ }
+ } else {
+ // When port number is not specified, read it from config
+ omPort = OmUtils.getOmRpcPort(conf);
+ }
+
+ SecurityConfig secConfig = new SecurityConfig(conf);
+
+ if (secConfig.isSecurityEnabled()) {
+ this.securityEnabled = true;
+ }
+
+ String replicationTypeConf =
+ conf.get(OzoneConfigKeys.OZONE_REPLICATION_TYPE,
+ OzoneConfigKeys.OZONE_REPLICATION_TYPE_DEFAULT);
+
+ int replicationCountConf = conf.getInt(OzoneConfigKeys.OZONE_REPLICATION,
+ OzoneConfigKeys.OZONE_REPLICATION_DEFAULT);
+
+ if (OmUtils.isOmHAServiceId(conf, omHost)) {
+ // omHost is listed as one of the service ids in the config,
+ // thus we should treat omHost as omServiceId
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(omHost, conf);
+ } else if (StringUtils.isNotEmpty(omHost) && omPort != -1) {
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(omHost, omPort, conf);
+ } else {
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(conf);
+ }
+ objectStore = ozoneClient.getObjectStore();
+ proxy = objectStore.getClientProxy();
+ this.replicationType = ReplicationType.valueOf(replicationTypeConf);
+ this.replicationFactor = ReplicationFactor.valueOf(replicationCountConf);
+ this.configuredDnPort = conf.getInt(
+ OzoneConfigKeys.DFS_CONTAINER_IPC_PORT,
+ OzoneConfigKeys.DFS_CONTAINER_IPC_PORT_DEFAULT);
+ } finally {
+ Thread.currentThread().setContextClassLoader(contextClassLoader);
+ }
+ }
+
+ OzoneBucket getBucket(OFSPath ofsPath, boolean createIfNotExist)
+ throws IOException {
+
+ return getBucket(ofsPath.getVolumeName(), ofsPath.getBucketName(),
+ createIfNotExist);
+ }
+
+ /**
+ * Get OzoneBucket object to operate in.
+ * Optionally create volume and bucket if not found.
+ *
+ * @param createIfNotExist Set this to true if the caller is a write operation
+ * in order to create the volume and bucket.
+ * @throws IOException Exceptions other than OMException with result code
+ * VOLUME_NOT_FOUND or BUCKET_NOT_FOUND.
+ */
+ private OzoneBucket getBucket(String volumeStr, String bucketStr,
+ boolean createIfNotExist) throws IOException {
+
+ OzoneBucket bucket;
+ try {
+ bucket = proxy.getBucketDetails(volumeStr, bucketStr);
+ } catch (OMException ex) {
+ if (createIfNotExist) {
+ // Note: getBucketDetails always throws BUCKET_NOT_FOUND, even if
+ // the volume doesn't exist.
+ if (ex.getResult().equals(BUCKET_NOT_FOUND)) {
+ OzoneVolume volume;
+ try {
+ volume = proxy.getVolumeDetails(volumeStr);
+ } catch (OMException getVolEx) {
+ if (getVolEx.getResult().equals(VOLUME_NOT_FOUND)) {
+ // Volume doesn't exist. Create it
+ try {
+ objectStore.createVolume(volumeStr);
+ } catch (OMException newVolEx) {
+ // Ignore the case where another client created the volume
+ if (!newVolEx.getResult().equals(VOLUME_ALREADY_EXISTS)) {
+ throw newVolEx;
+ }
+ }
+ } else {
+ throw getVolEx;
+ }
+ // Try get volume again
+ volume = proxy.getVolumeDetails(volumeStr);
+ }
+ // Create the bucket
+ try {
+ volume.createBucket(bucketStr);
+ } catch (OMException newBucEx) {
+ // Ignore the case where another client created the bucket
+ if (!newBucEx.getResult().equals(BUCKET_ALREADY_EXISTS)) {
+ throw newBucEx;
+ }
+ }
+ }
+ // Try get bucket again
+ bucket = proxy.getBucketDetails(volumeStr, bucketStr);
+ } else {
+ throw ex;
+ }
+ }
+
+ return bucket;
+ }
+
+ @Override
+ public void close() throws IOException {
+ ozoneClient.close();
+ }
+
+ @Override
+ public InputStream readFile(String pathStr) throws IOException {
+ incrementCounter(Statistic.OBJECTS_READ);
+ OFSPath ofsPath = new OFSPath(pathStr);
+ String key = ofsPath.getKeyName();
+ try {
+ OzoneBucket bucket = getBucket(ofsPath, false);
+ return bucket.readFile(key).getInputStream();
+ } catch (OMException ex) {
+ if (ex.getResult() == OMException.ResultCodes.FILE_NOT_FOUND
+ || ex.getResult() == OMException.ResultCodes.NOT_A_FILE) {
+ throw new FileNotFoundException(
+ ex.getResult().name() + ": " + ex.getMessage());
+ } else {
+ throw ex;
+ }
+ }
+ }
+
+ protected void incrementCounter(Statistic objectsRead) {
+ //noop: Use OzoneClientAdapterImpl which supports statistics.
+ }
+
+ @Override
+ public OzoneFSOutputStream createFile(String pathStr, boolean overWrite,
+ boolean recursive) throws IOException {
+ incrementCounter(Statistic.OBJECTS_CREATED);
+ OFSPath ofsPath = new OFSPath(pathStr);
+ String key = ofsPath.getKeyName();
+ try {
+ OzoneBucket bucket = getBucket(ofsPath, false);
+ OzoneOutputStream ozoneOutputStream = bucket.createFile(
+ key, 0, replicationType, replicationFactor, overWrite, recursive);
+ return new OzoneFSOutputStream(ozoneOutputStream.getOutputStream());
+ } catch (OMException ex) {
+ if (ex.getResult() == OMException.ResultCodes.FILE_ALREADY_EXISTS
+ || ex.getResult() == OMException.ResultCodes.NOT_A_FILE) {
+ throw new FileAlreadyExistsException(
+ ex.getResult().name() + ": " + ex.getMessage());
+ } else {
+ throw ex;
+ }
+ }
+ }
+
+ @Override
+ public void renamePath(String path, String newPath) throws IOException {
+ incrementCounter(Statistic.OBJECTS_RENAMED);
+ OFSPath ofsPath = new OFSPath(path);
+ OFSPath ofsNewPath = new OFSPath(newPath);
+
+ // Check path and newPathName are in the same volume and same bucket.
+ // This should have been checked in BasicRootedOzoneFileSystem#rename
+ // already via regular call path unless bypassed.
+ if (!ofsPath.isInSameBucketAs(ofsNewPath)) {
+ throw new IOException("Cannot rename a key to a different bucket");
+ }
+
+ OzoneBucket bucket = getBucket(ofsPath, false);
+ String key = ofsPath.getKeyName();
+ String newKey = ofsNewPath.getKeyName();
+ bucket.renameKey(key, newKey);
+ }
+
+ /**
+ * Package-private helper function to reduce calls to getBucket().
+ * @param bucket Bucket to operate in.
+ * @param path Existing key path.
+ * @param newPath New key path.
+ * @throws IOException IOException from bucket.renameKey().
+ */
+ void renamePath(OzoneBucket bucket, String path, String newPath)
+ throws IOException {
+ incrementCounter(Statistic.OBJECTS_RENAMED);
+ OFSPath ofsPath = new OFSPath(path);
+ OFSPath ofsNewPath = new OFSPath(newPath);
+ // No same-bucket policy check here since this call path is controlled
+ String key = ofsPath.getKeyName();
+ String newKey = ofsNewPath.getKeyName();
+ bucket.renameKey(key, newKey);
+ }
+
+ /**
+ * Helper method to create an directory specified by key name in bucket.
+ *
+ * @param pathStr path to be created as directory
+ * @return true if the key is created, false otherwise
+ */
+ @Override
+ public boolean createDirectory(String pathStr) throws IOException {
+ LOG.trace("creating dir for path: {}", pathStr);
+ incrementCounter(Statistic.OBJECTS_CREATED);
+ OFSPath ofsPath = new OFSPath(pathStr);
+ // Volume name unspecified, return failure
+ if (ofsPath.getVolumeName().length() == 0) {
+ return false;
+ }
+ // Handle where only volume is specified in pathStr
+ if (ofsPath.getBucketName().length() == 0) {
Review comment:
Done in e16a52ee7629ce89a185be00ab71742f56f1b7ce.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] smengcl commented on a change in pull request #415:
HDDS-2840. Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
smengcl commented on a change in pull request #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#discussion_r372094342
##########
File path: hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OFSPath.java
##########
@@ -0,0 +1,143 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.ozone;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.yetus.audience.InterfaceAudience;
+import org.apache.yetus.audience.InterfaceStability;
+import java.util.StringTokenizer;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
+
+/**
+ * Utility class for Rooted Ozone Filesystem (OFS) path processing.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+class OFSPath {
+ /**
+ * Here is a table illustrating what each name variable is given an input path
+ * Assuming /tmp is mounted to /tempVol/tempBucket
Review comment:
This is undecided. Yes we could give admin a CLI command to do the initialization. But we might want to leave this for jira HDDS-2929.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] hanishakoneru commented on a change in pull request
#415: HDDS-2840. Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
hanishakoneru commented on a change in pull request #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#discussion_r369299000
##########
File path: hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
##########
@@ -0,0 +1,395 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.TestDataUtil;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClientException;
+import org.apache.hadoop.ozone.client.OzoneKeyDetails;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.Set;
+import java.util.TreeSet;
+
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Ozone file system tests that are not covered by contract tests.
+ */
+public class TestRootedOzoneFileSystem {
+
+ @Rule
+ public Timeout globalTimeout = new Timeout(300_000);
+
+ private static MiniOzoneCluster cluster = null;
+
+ private static FileSystem fs;
+ private static RootedOzoneFileSystem ofs;
+
+ private static ObjectStore objectStore;
+
+ private String volumeName;
+ private String bucketName;
+
+ private String rootPath;
+
+ @Before
+ public void init() throws Exception {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ cluster = MiniOzoneCluster.newBuilder(conf)
+ .setNumDatanodes(3)
+ .build();
+ cluster.waitForClusterToBeReady();
+ objectStore = cluster.getClient().getObjectStore();
+
+ // create a volume and a bucket to be used by OFileSystem
Review comment:
OFileSystem -> RootedOzoneFileSystem
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] smengcl commented on a change in pull request #415:
HDDS-2840. Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
smengcl commented on a change in pull request #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#discussion_r371197517
##########
File path: hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
##########
@@ -0,0 +1,805 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.ozone;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.CreateFlag;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.Progressable;
+import org.apache.http.client.utils.URIBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.EnumSet;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+import static org.apache.hadoop.fs.ozone.Constants.LISTING_PAGE_SIZE;
+import static org.apache.hadoop.fs.ozone.Constants.OZONE_DEFAULT_USER;
+import static org.apache.hadoop.fs.ozone.Constants.OZONE_USER_DIR;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_OFS_URI_SCHEME;
+
+/**
+ * The minimal Ozone Filesystem implementation.
+ * <p>
+ * This is a basic version which doesn't extend
+ * KeyProviderTokenIssuer and doesn't include statistics. It can be used
+ * from older hadoop version. For newer hadoop version use the full featured
+ * OFileSystem.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class BasicRootedOzoneFileSystem extends FileSystem {
+ static final Logger LOG =
+ LoggerFactory.getLogger(BasicRootedOzoneFileSystem.class);
+
+ /**
+ * The Ozone client for connecting to Ozone server.
+ */
+
+ private URI uri;
+ private String userName;
+ private Path workingDir;
+// private String gOmHost;
+// private int gOmPort;
+// private Configuration gConf;
+// private boolean gIsolatedClassloader;
+
+ private RootedOzoneClientAdapter adapter;
+// private String adapterPath;
+
+ private static final String URI_EXCEPTION_TEXT =
+ "URL should be one of the following formats: " +
+ "ofs://om-service-id/ OR " +
+ "ofs://om-host.example.com/ OR " +
+ "ofs://om-host.example.com:5678/";
+
+ @Override
+ public void initialize(URI name, Configuration conf) throws IOException {
+ super.initialize(name, conf);
+ setConf(conf);
+ Objects.requireNonNull(name.getScheme(), "No scheme provided in " + name);
+ Preconditions.checkArgument(getScheme().equals(name.getScheme()),
+ "Invalid scheme provided in " + name);
+
+ String authority = name.getAuthority();
+ if (authority == null) {
+ // authority is null when fs.defaultFS is not a qualified ofs URI and
+ // ofs:/// is passed to the client. matcher will NPE if authority is null
+ throw new IllegalArgumentException(URI_EXCEPTION_TEXT);
+ }
+
+ String omHost;
Review comment:
Done, `omHost` -> `omHostOrServiceId`: 8e34c5f6718680c35b683ca4b0969286e975e318
BTW this should be applied to o3fs as well.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] smengcl commented on a change in pull request #415:
HDDS-2840. Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
smengcl commented on a change in pull request #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#discussion_r371262656
##########
File path: hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OFSPath.java
##########
@@ -0,0 +1,137 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.ozone;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.yetus.audience.InterfaceAudience;
+import org.apache.yetus.audience.InterfaceStability;
+import java.util.StringTokenizer;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
+
+/**
+ * Utility class for Rooted Ozone Filesystem (OFS) path processing.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+class OFSPath {
+ private String volumeName;
+ private String bucketName;
+ private String mountName;
+ private String keyName;
Review comment:
I added a table for illustration: 4bb7ed19674dcb959703fe633e940ab4bf17873e
Note the table might change in the future. This just shows what it does at the moment.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] smengcl commented on a change in pull request #415:
HDDS-2840. Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
smengcl commented on a change in pull request #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#discussion_r371245236
##########
File path: hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
##########
@@ -0,0 +1,805 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.ozone;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.CreateFlag;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.Progressable;
+import org.apache.http.client.utils.URIBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.EnumSet;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+import static org.apache.hadoop.fs.ozone.Constants.LISTING_PAGE_SIZE;
+import static org.apache.hadoop.fs.ozone.Constants.OZONE_DEFAULT_USER;
+import static org.apache.hadoop.fs.ozone.Constants.OZONE_USER_DIR;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_OFS_URI_SCHEME;
+
+/**
+ * The minimal Ozone Filesystem implementation.
+ * <p>
+ * This is a basic version which doesn't extend
+ * KeyProviderTokenIssuer and doesn't include statistics. It can be used
+ * from older hadoop version. For newer hadoop version use the full featured
+ * OFileSystem.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class BasicRootedOzoneFileSystem extends FileSystem {
+ static final Logger LOG =
+ LoggerFactory.getLogger(BasicRootedOzoneFileSystem.class);
+
+ /**
+ * The Ozone client for connecting to Ozone server.
+ */
+
+ private URI uri;
+ private String userName;
+ private Path workingDir;
+// private String gOmHost;
+// private int gOmPort;
+// private Configuration gConf;
+// private boolean gIsolatedClassloader;
+
+ private RootedOzoneClientAdapter adapter;
+// private String adapterPath;
+
+ private static final String URI_EXCEPTION_TEXT =
+ "URL should be one of the following formats: " +
+ "ofs://om-service-id/ OR " +
+ "ofs://om-host.example.com/ OR " +
+ "ofs://om-host.example.com:5678/";
+
+ @Override
+ public void initialize(URI name, Configuration conf) throws IOException {
+ super.initialize(name, conf);
+ setConf(conf);
+ Objects.requireNonNull(name.getScheme(), "No scheme provided in " + name);
+ Preconditions.checkArgument(getScheme().equals(name.getScheme()),
+ "Invalid scheme provided in " + name);
+
+ String authority = name.getAuthority();
+ if (authority == null) {
+ // authority is null when fs.defaultFS is not a qualified ofs URI and
+ // ofs:/// is passed to the client. matcher will NPE if authority is null
+ throw new IllegalArgumentException(URI_EXCEPTION_TEXT);
+ }
+
+ String omHost;
+ int omPort = -1;
+ // Parse hostname and port
+ String[] parts = authority.split(":");
+ if (parts.length > 2) {
+ throw new IllegalArgumentException(URI_EXCEPTION_TEXT);
+ }
+ omHost = parts[0];
+ if (parts.length == 2) {
+ try {
+ omPort = Integer.parseInt(parts[1]);
+ } catch (NumberFormatException e) {
+ throw new IllegalArgumentException(URI_EXCEPTION_TEXT);
+ }
+ }
+
+ try {
+ uri = new URIBuilder().setScheme(OZONE_OFS_URI_SCHEME)
+ .setHost(authority)
+ .build();
+ LOG.trace("Ozone URI for OFS initialization is " + uri);
+
+ //isolated is the default for ozonefs-lib-legacy which includes the
+ // /ozonefs.txt, otherwise the default is false. It could be overridden.
+ boolean defaultValue =
+ BasicRootedOzoneFileSystem.class.getClassLoader()
+ .getResource("ozonefs.txt") != null;
+
+ //Use string here instead of the constant as constant may not be available
+ //on the classpath of a hadoop 2.7
+ boolean isolatedClassloader =
+ conf.getBoolean("ozone.fs.isolated-classloader", defaultValue);
+
+ // adapter should be initialized in operations.
+ this.adapter = createAdapter(conf, omHost, omPort, isolatedClassloader);
+
+ try {
+ this.userName =
+ UserGroupInformation.getCurrentUser().getShortUserName();
+ } catch (IOException e) {
+ this.userName = OZONE_DEFAULT_USER;
+ }
+ this.workingDir = new Path(OZONE_USER_DIR, this.userName)
+ .makeQualified(this.uri, this.workingDir);
+ } catch (URISyntaxException ue) {
+ final String msg = "Invalid Ozone endpoint " + name;
+ LOG.error(msg, ue);
+ throw new IOException(msg, ue);
+ }
+ }
+
+ protected RootedOzoneClientAdapter createAdapter(Configuration conf,
+ String omHost, int omPort, boolean isolatedClassloader)
+ throws IOException {
+
+ if (isolatedClassloader) {
+ // TODO: Check how this code path need to be changed, for legacy Hadoop?
+ return RootedOzoneClientAdapterFactory.createAdapter();
+ } else {
+ return new BasicRootedOzoneClientAdapterImpl(omHost, omPort, conf);
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ try {
+ adapter.close();
+ } finally {
+ super.close();
+ }
+ }
+
+ @Override
+ public URI getUri() {
+ return uri;
+ }
+
+ @Override
+ public String getScheme() {
+ return OZONE_OFS_URI_SCHEME;
+ }
+
+ @Override
+ public FSDataInputStream open(Path path, int bufferSize) throws IOException {
+ incrementCounter(Statistic.INVOCATION_OPEN);
+ statistics.incrementReadOps(1);
+ LOG.trace("open() path: {}", path);
+ final String key = pathToKey(path);
+ return new FSDataInputStream(
+ new OzoneFSInputStream(adapter.readFile(key), statistics));
+ }
+
+ protected void incrementCounter(Statistic statistic) {
+ //don't do anything in this default implementation.
+ }
+
+ @Override
+ public FSDataOutputStream create(Path f, FsPermission permission,
+ boolean overwrite, int bufferSize,
+ short replication, long blockSize,
+ Progressable progress) throws IOException {
+ LOG.trace("create() path:{}", f);
+ incrementCounter(Statistic.INVOCATION_CREATE);
+ statistics.incrementWriteOps(1);
+ final String key = pathToKey(f);
+ return createOutputStream(key, overwrite, true);
+ }
+
+ @Override
+ public FSDataOutputStream createNonRecursive(Path path,
+ FsPermission permission,
+ EnumSet<CreateFlag> flags,
+ int bufferSize,
+ short replication,
+ long blockSize,
+ Progressable progress) throws IOException {
+ incrementCounter(Statistic.INVOCATION_CREATE_NON_RECURSIVE);
+ statistics.incrementWriteOps(1);
+ final String key = pathToKey(path);
+ return createOutputStream(key, flags.contains(CreateFlag.OVERWRITE), false);
+ }
+
+ private FSDataOutputStream createOutputStream(String key, boolean overwrite,
+ boolean recursive) throws IOException {
+ return new FSDataOutputStream(adapter.createFile(key, overwrite, recursive),
+ statistics);
+ }
+
+ @Override
+ public FSDataOutputStream append(Path f, int bufferSize,
+ Progressable progress) throws IOException {
+ throw new UnsupportedOperationException("append() Not implemented by the "
+ + getClass().getSimpleName() + " FileSystem implementation");
+ }
+
+ private class RenameIterator extends OzoneListingIterator {
+ private final String srcPath;
+ private final String dstPath;
+
+ RenameIterator(Path srcPath, Path dstPath)
+ throws IOException {
+ super(srcPath);
+ this.srcPath = pathToKey(srcPath);
+ this.dstPath = pathToKey(dstPath);
+ LOG.trace("rename from:{} to:{}", this.srcPath, this.dstPath);
+ }
+
+ @Override
+ boolean processKeyPath(String keyPath) throws IOException {
+ String newPath = dstPath.concat(keyPath.substring(srcPath.length()));
+ adapter.renamePath(keyPath, newPath);
Review comment:
Yep this is a problem.
I attempted to address it in commit dfc50cc8b0f9d35aa5d9dfcea3e4beedb4930341.
I do realize my solution still have 2 minor issues:
1. code duplication, e.g. overloaded `renamePath`/`deleteObject` shares mostly the same code;
2. using key in `processKeyPath` would be more efficient -- but this would break my intention of using ONLY full path outside of `AdapterImpl`.
Those two issues above could be solved with some degree of refactoring. I'm doing it this way just to show one of the ways to solve the RPC call problem. Comments?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] xiaoyuyao commented on a change in pull request
#415: HDDS-2840. Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
xiaoyuyao commented on a change in pull request #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#discussion_r372017127
##########
File path: hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
##########
@@ -0,0 +1,694 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.ozone;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.key.KeyProvider;
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdds.client.ReplicationFactor;
+import org.apache.hadoop.hdds.client.ReplicationType;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ozone.OmUtils;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.client.OzoneKey;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
+import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
+import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenRenewer;
+
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .BUCKET_ALREADY_EXISTS;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .VOLUME_ALREADY_EXISTS;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .VOLUME_NOT_FOUND;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .BUCKET_NOT_FOUND;
+
+/**
+ * Basic Implementation of the OzoneFileSystem calls.
+ * <p>
+ * This is the minimal version which doesn't include any statistics.
+ * <p>
+ * For full featured version use OzoneClientAdapterImpl.
+ */
+public class BasicRootedOzoneClientAdapterImpl
+ implements RootedOzoneClientAdapter {
+
+ static final Logger LOG =
+ LoggerFactory.getLogger(BasicRootedOzoneClientAdapterImpl.class);
+
+ private OzoneClient ozoneClient;
+ private ClientProtocol proxy;
+ private ObjectStore objectStore;
+ private ReplicationType replicationType;
+ private ReplicationFactor replicationFactor;
+ private boolean securityEnabled;
+ private int configuredDnPort;
+
+ /**
+ * Create new OzoneClientAdapter implementation.
+ *
+ * @throws IOException In case of a problem.
+ */
+ public BasicRootedOzoneClientAdapterImpl() throws IOException {
+ this(createConf());
+ }
+
+ private static OzoneConfiguration createConf() {
+ ClassLoader contextClassLoader =
+ Thread.currentThread().getContextClassLoader();
+ Thread.currentThread().setContextClassLoader(null);
+ try {
+ return new OzoneConfiguration();
+ } finally {
+ Thread.currentThread().setContextClassLoader(contextClassLoader);
+ }
+ }
+
+ public BasicRootedOzoneClientAdapterImpl(OzoneConfiguration conf)
+ throws IOException {
+ this(null, -1, conf);
+ }
+
+ public BasicRootedOzoneClientAdapterImpl(String omHost, int omPort,
+ Configuration hadoopConf) throws IOException {
+
+ ClassLoader contextClassLoader =
+ Thread.currentThread().getContextClassLoader();
+ Thread.currentThread().setContextClassLoader(null);
+
+ try {
+ OzoneConfiguration conf = OzoneConfiguration.of(hadoopConf);
+
+ if (omHost == null && OmUtils.isServiceIdsDefined(conf)) {
+ // When the host name or service id isn't given
+ // but ozone.om.service.ids is defined, declare failure.
+
+ // This is a safety precaution that prevents the client from
+ // accidentally failing over to an unintended OM.
+ throw new IllegalArgumentException("Service ID or host name must not"
+ + " be omitted when ozone.om.service.ids is defined.");
+ }
+
+ if (omPort != -1) {
+ // When the port number is specified, perform the following check
+ if (OmUtils.isOmHAServiceId(conf, omHost)) {
+ // If omHost is a service id, it shouldn't use a port
+ throw new IllegalArgumentException("Port " + omPort +
+ " specified in URI but host '" + omHost + "' is a "
+ + "logical (HA) OzoneManager and does not use port information.");
+ }
+ } else {
+ // When port number is not specified, read it from config
+ omPort = OmUtils.getOmRpcPort(conf);
+ }
+
+ SecurityConfig secConfig = new SecurityConfig(conf);
+
+ if (secConfig.isSecurityEnabled()) {
+ this.securityEnabled = true;
+ }
+
+ String replicationTypeConf =
+ conf.get(OzoneConfigKeys.OZONE_REPLICATION_TYPE,
+ OzoneConfigKeys.OZONE_REPLICATION_TYPE_DEFAULT);
+
+ int replicationCountConf = conf.getInt(OzoneConfigKeys.OZONE_REPLICATION,
+ OzoneConfigKeys.OZONE_REPLICATION_DEFAULT);
+
+ if (OmUtils.isOmHAServiceId(conf, omHost)) {
+ // omHost is listed as one of the service ids in the config,
+ // thus we should treat omHost as omServiceId
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(omHost, conf);
+ } else if (StringUtils.isNotEmpty(omHost) && omPort != -1) {
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(omHost, omPort, conf);
+ } else {
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(conf);
+ }
+ objectStore = ozoneClient.getObjectStore();
+ proxy = objectStore.getClientProxy();
+ this.replicationType = ReplicationType.valueOf(replicationTypeConf);
+ this.replicationFactor = ReplicationFactor.valueOf(replicationCountConf);
+ this.configuredDnPort = conf.getInt(
+ OzoneConfigKeys.DFS_CONTAINER_IPC_PORT,
+ OzoneConfigKeys.DFS_CONTAINER_IPC_PORT_DEFAULT);
+ } finally {
+ Thread.currentThread().setContextClassLoader(contextClassLoader);
+ }
+ }
+
+ OzoneBucket getBucket(OFSPath ofsPath, boolean createIfNotExist)
+ throws IOException {
+
+ return getBucket(ofsPath.getVolumeName(), ofsPath.getBucketName(),
+ createIfNotExist);
+ }
+
+ /**
+ * Get OzoneBucket object to operate in.
+ * Optionally create volume and bucket if not found.
+ *
+ * @param createIfNotExist Set this to true if the caller is a write operation
+ * in order to create the volume and bucket.
+ * @throws IOException Exceptions other than OMException with result code
+ * VOLUME_NOT_FOUND or BUCKET_NOT_FOUND.
+ */
+ private OzoneBucket getBucket(String volumeStr, String bucketStr,
+ boolean createIfNotExist) throws IOException {
+
+ OzoneBucket bucket;
+ try {
+ bucket = proxy.getBucketDetails(volumeStr, bucketStr);
+ } catch (OMException ex) {
+ if (createIfNotExist) {
+ // Note: getBucketDetails always throws BUCKET_NOT_FOUND, even if
+ // the volume doesn't exist.
+ if (ex.getResult().equals(BUCKET_NOT_FOUND)) {
+ OzoneVolume volume;
+ try {
+ volume = proxy.getVolumeDetails(volumeStr);
+ } catch (OMException getVolEx) {
+ if (getVolEx.getResult().equals(VOLUME_NOT_FOUND)) {
+ // Volume doesn't exist. Create it
+ try {
+ objectStore.createVolume(volumeStr);
+ } catch (OMException newVolEx) {
+ // Ignore the case where another client created the volume
+ if (!newVolEx.getResult().equals(VOLUME_ALREADY_EXISTS)) {
Review comment:
Even the volume creation succeed, the bucket creation might fail if the ACLs are not set properly when security and acls are enabled. With o3fs, this is OK because the volume/bucket are provisioned and acls setup before it is mounted. With ofs, this may not work without ACL changes.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] hanishakoneru commented on a change in pull request
#415: HDDS-2840. Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
hanishakoneru commented on a change in pull request #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#discussion_r369328310
##########
File path: hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
##########
@@ -0,0 +1,805 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.ozone;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.CreateFlag;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.Progressable;
+import org.apache.http.client.utils.URIBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.EnumSet;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+import static org.apache.hadoop.fs.ozone.Constants.LISTING_PAGE_SIZE;
+import static org.apache.hadoop.fs.ozone.Constants.OZONE_DEFAULT_USER;
+import static org.apache.hadoop.fs.ozone.Constants.OZONE_USER_DIR;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_OFS_URI_SCHEME;
+
+/**
+ * The minimal Ozone Filesystem implementation.
+ * <p>
+ * This is a basic version which doesn't extend
+ * KeyProviderTokenIssuer and doesn't include statistics. It can be used
+ * from older hadoop version. For newer hadoop version use the full featured
+ * OFileSystem.
+ */
Review comment:
OFileSystem -> RootedOzoneFileSystem
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] hanishakoneru commented on a change in pull request
#415: HDDS-2840. Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
hanishakoneru commented on a change in pull request #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#discussion_r369299929
##########
File path: hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
##########
@@ -0,0 +1,395 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.TestDataUtil;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClientException;
+import org.apache.hadoop.ozone.client.OzoneKeyDetails;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.Set;
+import java.util.TreeSet;
+
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Ozone file system tests that are not covered by contract tests.
+ */
+public class TestRootedOzoneFileSystem {
+
+ @Rule
+ public Timeout globalTimeout = new Timeout(300_000);
+
+ private static MiniOzoneCluster cluster = null;
+
+ private static FileSystem fs;
+ private static RootedOzoneFileSystem ofs;
+
+ private static ObjectStore objectStore;
+
+ private String volumeName;
+ private String bucketName;
+
+ private String rootPath;
+
+ @Before
+ public void init() throws Exception {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ cluster = MiniOzoneCluster.newBuilder(conf)
+ .setNumDatanodes(3)
+ .build();
+ cluster.waitForClusterToBeReady();
+ objectStore = cluster.getClient().getObjectStore();
+
+ // create a volume and a bucket to be used by OFileSystem
+ OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(cluster);
+ volumeName = bucket.getVolumeName();
+ bucketName = bucket.getName();
+
+ // For now:
Review comment:
If this is going to change, can we please add a TODO here to keep track.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] smengcl edited a comment on issue #415: HDDS-2840.
Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
smengcl edited a comment on issue #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#issuecomment-577933488
> I think here we need to catch exception from createVolume, as by the time we create volume, if another client created the same volume, we get VOLUME_ALREADY_EXCEPTION, so this should be safe and ignored.
>
> Same applicable for bucket creation also.
Nice catch. Addressed in commit ab64a98fadd52b483136f99031d8ed9d401b36c9. @bharatviswa504 Please take a look.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] smengcl commented on a change in pull request #415:
HDDS-2840. Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
smengcl commented on a change in pull request #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#discussion_r373138181
##########
File path: hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
##########
@@ -0,0 +1,694 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.ozone;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.key.KeyProvider;
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdds.client.ReplicationFactor;
+import org.apache.hadoop.hdds.client.ReplicationType;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ozone.OmUtils;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.client.OzoneKey;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
+import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
+import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenRenewer;
+
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .BUCKET_ALREADY_EXISTS;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .VOLUME_ALREADY_EXISTS;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .VOLUME_NOT_FOUND;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .BUCKET_NOT_FOUND;
+
+/**
+ * Basic Implementation of the OzoneFileSystem calls.
+ * <p>
+ * This is the minimal version which doesn't include any statistics.
+ * <p>
+ * For full featured version use OzoneClientAdapterImpl.
+ */
+public class BasicRootedOzoneClientAdapterImpl
+ implements RootedOzoneClientAdapter {
+
+ static final Logger LOG =
+ LoggerFactory.getLogger(BasicRootedOzoneClientAdapterImpl.class);
+
+ private OzoneClient ozoneClient;
+ private ClientProtocol proxy;
+ private ObjectStore objectStore;
+ private ReplicationType replicationType;
+ private ReplicationFactor replicationFactor;
+ private boolean securityEnabled;
+ private int configuredDnPort;
+
+ /**
+ * Create new OzoneClientAdapter implementation.
+ *
+ * @throws IOException In case of a problem.
+ */
+ public BasicRootedOzoneClientAdapterImpl() throws IOException {
+ this(createConf());
+ }
+
+ private static OzoneConfiguration createConf() {
+ ClassLoader contextClassLoader =
+ Thread.currentThread().getContextClassLoader();
+ Thread.currentThread().setContextClassLoader(null);
+ try {
+ return new OzoneConfiguration();
+ } finally {
+ Thread.currentThread().setContextClassLoader(contextClassLoader);
+ }
+ }
+
+ public BasicRootedOzoneClientAdapterImpl(OzoneConfiguration conf)
+ throws IOException {
+ this(null, -1, conf);
+ }
+
+ public BasicRootedOzoneClientAdapterImpl(String omHost, int omPort,
+ Configuration hadoopConf) throws IOException {
+
+ ClassLoader contextClassLoader =
+ Thread.currentThread().getContextClassLoader();
+ Thread.currentThread().setContextClassLoader(null);
+
+ try {
+ OzoneConfiguration conf = OzoneConfiguration.of(hadoopConf);
+
+ if (omHost == null && OmUtils.isServiceIdsDefined(conf)) {
+ // When the host name or service id isn't given
+ // but ozone.om.service.ids is defined, declare failure.
+
+ // This is a safety precaution that prevents the client from
+ // accidentally failing over to an unintended OM.
+ throw new IllegalArgumentException("Service ID or host name must not"
+ + " be omitted when ozone.om.service.ids is defined.");
+ }
+
+ if (omPort != -1) {
+ // When the port number is specified, perform the following check
+ if (OmUtils.isOmHAServiceId(conf, omHost)) {
+ // If omHost is a service id, it shouldn't use a port
+ throw new IllegalArgumentException("Port " + omPort +
+ " specified in URI but host '" + omHost + "' is a "
+ + "logical (HA) OzoneManager and does not use port information.");
+ }
+ } else {
+ // When port number is not specified, read it from config
+ omPort = OmUtils.getOmRpcPort(conf);
+ }
+
+ SecurityConfig secConfig = new SecurityConfig(conf);
+
+ if (secConfig.isSecurityEnabled()) {
+ this.securityEnabled = true;
+ }
+
+ String replicationTypeConf =
+ conf.get(OzoneConfigKeys.OZONE_REPLICATION_TYPE,
+ OzoneConfigKeys.OZONE_REPLICATION_TYPE_DEFAULT);
+
+ int replicationCountConf = conf.getInt(OzoneConfigKeys.OZONE_REPLICATION,
+ OzoneConfigKeys.OZONE_REPLICATION_DEFAULT);
+
+ if (OmUtils.isOmHAServiceId(conf, omHost)) {
+ // omHost is listed as one of the service ids in the config,
+ // thus we should treat omHost as omServiceId
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(omHost, conf);
+ } else if (StringUtils.isNotEmpty(omHost) && omPort != -1) {
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(omHost, omPort, conf);
+ } else {
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(conf);
+ }
+ objectStore = ozoneClient.getObjectStore();
+ proxy = objectStore.getClientProxy();
+ this.replicationType = ReplicationType.valueOf(replicationTypeConf);
+ this.replicationFactor = ReplicationFactor.valueOf(replicationCountConf);
+ this.configuredDnPort = conf.getInt(
+ OzoneConfigKeys.DFS_CONTAINER_IPC_PORT,
+ OzoneConfigKeys.DFS_CONTAINER_IPC_PORT_DEFAULT);
+ } finally {
+ Thread.currentThread().setContextClassLoader(contextClassLoader);
+ }
+ }
+
+ OzoneBucket getBucket(OFSPath ofsPath, boolean createIfNotExist)
+ throws IOException {
+
+ return getBucket(ofsPath.getVolumeName(), ofsPath.getBucketName(),
+ createIfNotExist);
+ }
+
+ /**
+ * Get OzoneBucket object to operate in.
+ * Optionally create volume and bucket if not found.
+ *
+ * @param createIfNotExist Set this to true if the caller is a write operation
+ * in order to create the volume and bucket.
+ * @throws IOException Exceptions other than OMException with result code
+ * VOLUME_NOT_FOUND or BUCKET_NOT_FOUND.
+ */
+ private OzoneBucket getBucket(String volumeStr, String bucketStr,
+ boolean createIfNotExist) throws IOException {
+
+ OzoneBucket bucket;
+ try {
+ bucket = proxy.getBucketDetails(volumeStr, bucketStr);
+ } catch (OMException ex) {
+ if (createIfNotExist) {
+ // Note: getBucketDetails always throws BUCKET_NOT_FOUND, even if
+ // the volume doesn't exist.
+ if (ex.getResult().equals(BUCKET_NOT_FOUND)) {
+ OzoneVolume volume;
+ try {
+ volume = proxy.getVolumeDetails(volumeStr);
+ } catch (OMException getVolEx) {
+ if (getVolEx.getResult().equals(VOLUME_NOT_FOUND)) {
+ // Volume doesn't exist. Create it
+ try {
+ objectStore.createVolume(volumeStr);
+ } catch (OMException newVolEx) {
+ // Ignore the case where another client created the volume
+ if (!newVolEx.getResult().equals(VOLUME_ALREADY_EXISTS)) {
+ throw newVolEx;
+ }
+ }
+ } else {
+ throw getVolEx;
+ }
+ // Try get volume again
+ volume = proxy.getVolumeDetails(volumeStr);
+ }
+ // Create the bucket
+ try {
+ volume.createBucket(bucketStr);
+ } catch (OMException newBucEx) {
+ // Ignore the case where another client created the bucket
+ if (!newBucEx.getResult().equals(BUCKET_ALREADY_EXISTS)) {
+ throw newBucEx;
+ }
+ }
+ }
+ // Try get bucket again
+ bucket = proxy.getBucketDetails(volumeStr, bucketStr);
+ } else {
+ throw ex;
+ }
+ }
+
+ return bucket;
+ }
+
+ @Override
+ public void close() throws IOException {
+ ozoneClient.close();
+ }
+
+ @Override
+ public InputStream readFile(String pathStr) throws IOException {
+ incrementCounter(Statistic.OBJECTS_READ);
+ OFSPath ofsPath = new OFSPath(pathStr);
+ String key = ofsPath.getKeyName();
+ try {
+ OzoneBucket bucket = getBucket(ofsPath, false);
+ return bucket.readFile(key).getInputStream();
+ } catch (OMException ex) {
+ if (ex.getResult() == OMException.ResultCodes.FILE_NOT_FOUND
+ || ex.getResult() == OMException.ResultCodes.NOT_A_FILE) {
+ throw new FileNotFoundException(
+ ex.getResult().name() + ": " + ex.getMessage());
+ } else {
+ throw ex;
+ }
+ }
+ }
+
+ protected void incrementCounter(Statistic objectsRead) {
+ //noop: Use OzoneClientAdapterImpl which supports statistics.
+ }
+
+ @Override
+ public OzoneFSOutputStream createFile(String pathStr, boolean overWrite,
+ boolean recursive) throws IOException {
+ incrementCounter(Statistic.OBJECTS_CREATED);
+ OFSPath ofsPath = new OFSPath(pathStr);
+ String key = ofsPath.getKeyName();
+ try {
+ OzoneBucket bucket = getBucket(ofsPath, false);
+ OzoneOutputStream ozoneOutputStream = bucket.createFile(
+ key, 0, replicationType, replicationFactor, overWrite, recursive);
+ return new OzoneFSOutputStream(ozoneOutputStream.getOutputStream());
+ } catch (OMException ex) {
+ if (ex.getResult() == OMException.ResultCodes.FILE_ALREADY_EXISTS
+ || ex.getResult() == OMException.ResultCodes.NOT_A_FILE) {
+ throw new FileAlreadyExistsException(
+ ex.getResult().name() + ": " + ex.getMessage());
+ } else {
+ throw ex;
+ }
+ }
+ }
+
+ @Override
+ public void renamePath(String path, String newPath) throws IOException {
+ incrementCounter(Statistic.OBJECTS_RENAMED);
+ OFSPath ofsPath = new OFSPath(path);
+ OFSPath ofsNewPath = new OFSPath(newPath);
+
+ // Check path and newPathName are in the same volume and same bucket.
+ // This should have been checked in BasicRootedOzoneFileSystem#rename
+ // already via regular call path unless bypassed.
+ if (!ofsPath.isInSameBucketAs(ofsNewPath)) {
+ throw new IOException("Cannot rename a key to a different bucket");
+ }
+
+ OzoneBucket bucket = getBucket(ofsPath, false);
+ String key = ofsPath.getKeyName();
+ String newKey = ofsNewPath.getKeyName();
+ bucket.renameKey(key, newKey);
+ }
+
+ /**
+ * Package-private helper function to reduce calls to getBucket().
+ * @param bucket Bucket to operate in.
+ * @param path Existing key path.
+ * @param newPath New key path.
+ * @throws IOException IOException from bucket.renameKey().
+ */
+ void renamePath(OzoneBucket bucket, String path, String newPath)
+ throws IOException {
+ incrementCounter(Statistic.OBJECTS_RENAMED);
+ OFSPath ofsPath = new OFSPath(path);
+ OFSPath ofsNewPath = new OFSPath(newPath);
+ // No same-bucket policy check here since this call path is controlled
+ String key = ofsPath.getKeyName();
+ String newKey = ofsNewPath.getKeyName();
+ bucket.renameKey(key, newKey);
+ }
+
+ /**
+ * Helper method to create an directory specified by key name in bucket.
+ *
+ * @param pathStr path to be created as directory
+ * @return true if the key is created, false otherwise
+ */
+ @Override
+ public boolean createDirectory(String pathStr) throws IOException {
+ LOG.trace("creating dir for path: {}", pathStr);
+ incrementCounter(Statistic.OBJECTS_CREATED);
+ OFSPath ofsPath = new OFSPath(pathStr);
+ // Volume name unspecified, return failure
+ if (ofsPath.getVolumeName().length() == 0) {
+ return false;
+ }
+ // Handle where only volume is specified in pathStr
+ if (ofsPath.getBucketName().length() == 0) {
+ objectStore.createVolume(ofsPath.getVolumeName());
+ return true;
+ }
+ String keyStr = ofsPath.getKeyName();
+ try {
+ OzoneBucket bucket = getBucket(ofsPath, true);
+ // if keyStr is empty, it indicates that only volume or volume+bucket is
+ // given in pathStr, so getBucket() above should've handled the creation
+ // of volume/bucket already.
+ if (keyStr != null && keyStr.length() > 0) {
+ bucket.createDirectory(keyStr);
+ }
+ } catch (OMException e) {
+ if (e.getResult() == OMException.ResultCodes.FILE_ALREADY_EXISTS) {
+ throw new FileAlreadyExistsException(e.getMessage());
+ }
+ throw e;
+ }
+ return true;
+ }
+
+ /**
+ * Helper method to delete an object specified by key name in bucket.
+ *
+ * @param path path to a key to be deleted
+ * @return true if the key is deleted, false otherwise
+ */
+ @Override
+ public boolean deleteObject(String path) {
+ LOG.trace("issuing delete for path to key: {}", path);
+ OFSPath ofsPath = new OFSPath(path);
+ String keyName = ofsPath.getKeyName();
Review comment:
Agree
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] xiaoyuyao merged pull request #415: HDDS-2840.
Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
xiaoyuyao merged pull request #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] smengcl commented on a change in pull request #415:
HDDS-2840. Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
smengcl commented on a change in pull request #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#discussion_r367205152
##########
File path: hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
##########
@@ -0,0 +1,602 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.ozone;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.key.KeyProvider;
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdds.client.ReplicationFactor;
+import org.apache.hadoop.hdds.client.ReplicationType;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ozone.OmUtils;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.client.OzoneKey;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
+import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
+import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenRenewer;
+
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Basic Implementation of the OzoneFileSystem calls.
+ * <p>
+ * This is the minimal version which doesn't include any statistics.
+ * <p>
+ * For full featured version use OzoneClientAdapterImpl.
+ */
+public class BasicRootedOzoneClientAdapterImpl
+ implements RootedOzoneClientAdapter {
+
+ static final Logger LOG =
+ LoggerFactory.getLogger(BasicRootedOzoneClientAdapterImpl.class);
+
+ private OzoneClient ozoneClient;
+ private ObjectStore objectStore;
+ private OzoneVolume volume;
+ private OzoneBucket bucket;
+ private ReplicationType replicationType;
+ private ReplicationFactor replicationFactor;
+ private boolean securityEnabled;
+ private int configuredDnPort;
+
+ /**
+ * Create new OzoneClientAdapter implementation.
+ *
+ * @throws IOException In case of a problem.
+ */
+ public BasicRootedOzoneClientAdapterImpl() throws IOException {
+ this(createConf());
+ }
+
+ private static OzoneConfiguration createConf() {
+ ClassLoader contextClassLoader =
+ Thread.currentThread().getContextClassLoader();
+ Thread.currentThread().setContextClassLoader(null);
+ try {
+ return new OzoneConfiguration();
+ } finally {
+ Thread.currentThread().setContextClassLoader(contextClassLoader);
+ }
+ }
+
+ public BasicRootedOzoneClientAdapterImpl(OzoneConfiguration conf)
+ throws IOException {
+ this(null, -1, conf);
+ }
+
+ public BasicRootedOzoneClientAdapterImpl(String omHost, int omPort,
+ Configuration hadoopConf) throws IOException {
+
+ ClassLoader contextClassLoader =
+ Thread.currentThread().getContextClassLoader();
+ Thread.currentThread().setContextClassLoader(null);
+
+ try {
+ OzoneConfiguration conf = OzoneConfiguration.of(hadoopConf);
+
+ if (omHost == null && OmUtils.isServiceIdsDefined(conf)) {
+ // When the host name or service id isn't given
+ // but ozone.om.service.ids is defined, declare failure.
+
+ // This is a safety precaution that prevents the client from
+ // accidentally failing over to an unintended OM.
+ throw new IllegalArgumentException("Service ID or host name must not"
+ + " be omitted when ozone.om.service.ids is defined.");
+ }
+
+ if (omPort != -1) {
+ // When the port number is specified, perform the following check
+ if (OmUtils.isOmHAServiceId(conf, omHost)) {
+ // If omHost is a service id, it shouldn't use a port
+ throw new IllegalArgumentException("Port " + omPort +
+ " specified in URI but host '" + omHost + "' is a "
+ + "logical (HA) OzoneManager and does not use port information.");
+ }
+ } else {
+ // When port number is not specified, read it from config
+ omPort = OmUtils.getOmRpcPort(conf);
+ }
+
+ SecurityConfig secConfig = new SecurityConfig(conf);
+
+ if (secConfig.isSecurityEnabled()) {
+ this.securityEnabled = true;
+ }
+
+ String replicationTypeConf =
+ conf.get(OzoneConfigKeys.OZONE_REPLICATION_TYPE,
+ OzoneConfigKeys.OZONE_REPLICATION_TYPE_DEFAULT);
+
+ int replicationCountConf = conf.getInt(OzoneConfigKeys.OZONE_REPLICATION,
+ OzoneConfigKeys.OZONE_REPLICATION_DEFAULT);
+
+ if (OmUtils.isOmHAServiceId(conf, omHost)) {
+ // omHost is listed as one of the service ids in the config,
+ // thus we should treat omHost as omServiceId
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(omHost, conf);
+ } else if (StringUtils.isNotEmpty(omHost) && omPort != -1) {
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(omHost, omPort, conf);
+ } else {
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(conf);
+ }
+ objectStore = ozoneClient.getObjectStore();
+ this.replicationType = ReplicationType.valueOf(replicationTypeConf);
+ this.replicationFactor = ReplicationFactor.valueOf(replicationCountConf);
+ this.configuredDnPort = conf.getInt(
+ OzoneConfigKeys.DFS_CONTAINER_IPC_PORT,
+ OzoneConfigKeys.DFS_CONTAINER_IPC_PORT_DEFAULT);
+ } finally {
+ Thread.currentThread().setContextClassLoader(contextClassLoader);
+ }
+ }
+
+ public void setVolume(String volumeString) throws IOException {
+ this.volume = objectStore.getVolume(volumeString);
+ }
+
+ public void setBucket(String bucketString) throws IOException {
+ this.bucket = volume.getBucket(bucketString);
+ }
+
+ private void getVolumeAndBucket(OFSPath ofsPath,
+ boolean createIfNotExist) throws IOException {
+ getVolumeAndBucket(ofsPath.getVolumeName(), ofsPath.getBucketName(),
+ createIfNotExist);
+ }
+
+ /**
+ * Apply volumeStr and bucketStr stored in the object instance before
+ * executing a FileSystem operation.
+ *
+ * @param createIfNotExist Set this to true if the caller is a write operation
+ * in order to create the volume and bucket.
+ * @throws IOException
+ */
+ private void getVolumeAndBucket(String volumeStr, String bucketStr,
Review comment:
done
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] hanishakoneru commented on a change in pull request
#415: HDDS-2840. Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
hanishakoneru commented on a change in pull request #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#discussion_r367196530
##########
File path: hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
##########
@@ -0,0 +1,602 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.ozone;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.key.KeyProvider;
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdds.client.ReplicationFactor;
+import org.apache.hadoop.hdds.client.ReplicationType;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ozone.OmUtils;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.client.OzoneKey;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
+import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
+import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenRenewer;
+
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Basic Implementation of the OzoneFileSystem calls.
+ * <p>
+ * This is the minimal version which doesn't include any statistics.
+ * <p>
+ * For full featured version use OzoneClientAdapterImpl.
+ */
+public class BasicRootedOzoneClientAdapterImpl
+ implements RootedOzoneClientAdapter {
+
+ static final Logger LOG =
+ LoggerFactory.getLogger(BasicRootedOzoneClientAdapterImpl.class);
+
+ private OzoneClient ozoneClient;
+ private ObjectStore objectStore;
+ private OzoneVolume volume;
+ private OzoneBucket bucket;
+ private ReplicationType replicationType;
+ private ReplicationFactor replicationFactor;
+ private boolean securityEnabled;
+ private int configuredDnPort;
+
+ /**
+ * Create new OzoneClientAdapter implementation.
+ *
+ * @throws IOException In case of a problem.
+ */
+ public BasicRootedOzoneClientAdapterImpl() throws IOException {
+ this(createConf());
+ }
+
+ private static OzoneConfiguration createConf() {
+ ClassLoader contextClassLoader =
+ Thread.currentThread().getContextClassLoader();
+ Thread.currentThread().setContextClassLoader(null);
+ try {
+ return new OzoneConfiguration();
+ } finally {
+ Thread.currentThread().setContextClassLoader(contextClassLoader);
+ }
+ }
+
+ public BasicRootedOzoneClientAdapterImpl(OzoneConfiguration conf)
+ throws IOException {
+ this(null, -1, conf);
+ }
+
+ public BasicRootedOzoneClientAdapterImpl(String omHost, int omPort,
+ Configuration hadoopConf) throws IOException {
+
+ ClassLoader contextClassLoader =
+ Thread.currentThread().getContextClassLoader();
+ Thread.currentThread().setContextClassLoader(null);
+
+ try {
+ OzoneConfiguration conf = OzoneConfiguration.of(hadoopConf);
+
+ if (omHost == null && OmUtils.isServiceIdsDefined(conf)) {
+ // When the host name or service id isn't given
+ // but ozone.om.service.ids is defined, declare failure.
+
+ // This is a safety precaution that prevents the client from
+ // accidentally failing over to an unintended OM.
+ throw new IllegalArgumentException("Service ID or host name must not"
+ + " be omitted when ozone.om.service.ids is defined.");
+ }
+
+ if (omPort != -1) {
+ // When the port number is specified, perform the following check
+ if (OmUtils.isOmHAServiceId(conf, omHost)) {
+ // If omHost is a service id, it shouldn't use a port
+ throw new IllegalArgumentException("Port " + omPort +
+ " specified in URI but host '" + omHost + "' is a "
+ + "logical (HA) OzoneManager and does not use port information.");
+ }
+ } else {
+ // When port number is not specified, read it from config
+ omPort = OmUtils.getOmRpcPort(conf);
+ }
+
+ SecurityConfig secConfig = new SecurityConfig(conf);
+
+ if (secConfig.isSecurityEnabled()) {
+ this.securityEnabled = true;
+ }
+
+ String replicationTypeConf =
+ conf.get(OzoneConfigKeys.OZONE_REPLICATION_TYPE,
+ OzoneConfigKeys.OZONE_REPLICATION_TYPE_DEFAULT);
+
+ int replicationCountConf = conf.getInt(OzoneConfigKeys.OZONE_REPLICATION,
+ OzoneConfigKeys.OZONE_REPLICATION_DEFAULT);
+
+ if (OmUtils.isOmHAServiceId(conf, omHost)) {
+ // omHost is listed as one of the service ids in the config,
+ // thus we should treat omHost as omServiceId
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(omHost, conf);
+ } else if (StringUtils.isNotEmpty(omHost) && omPort != -1) {
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(omHost, omPort, conf);
+ } else {
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(conf);
+ }
+ objectStore = ozoneClient.getObjectStore();
+ this.replicationType = ReplicationType.valueOf(replicationTypeConf);
+ this.replicationFactor = ReplicationFactor.valueOf(replicationCountConf);
+ this.configuredDnPort = conf.getInt(
+ OzoneConfigKeys.DFS_CONTAINER_IPC_PORT,
+ OzoneConfigKeys.DFS_CONTAINER_IPC_PORT_DEFAULT);
+ } finally {
+ Thread.currentThread().setContextClassLoader(contextClassLoader);
+ }
+ }
+
+ public void setVolume(String volumeString) throws IOException {
+ this.volume = objectStore.getVolume(volumeString);
+ }
+
+ public void setBucket(String bucketString) throws IOException {
+ this.bucket = volume.getBucket(bucketString);
+ }
+
+ private void getVolumeAndBucket(OFSPath ofsPath,
+ boolean createIfNotExist) throws IOException {
+ getVolumeAndBucket(ofsPath.getVolumeName(), ofsPath.getBucketName(),
+ createIfNotExist);
+ }
+
+ /**
+ * Apply volumeStr and bucketStr stored in the object instance before
+ * executing a FileSystem operation.
+ *
+ * @param createIfNotExist Set this to true if the caller is a write operation
+ * in order to create the volume and bucket.
+ * @throws IOException
+ */
+ private void getVolumeAndBucket(String volumeStr, String bucketStr,
Review comment:
Can we rename this method to setVolumeAndBucket as we are setting the values here.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] smengcl commented on issue #415: HDDS-2840.
Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
smengcl commented on issue #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#issuecomment-573229933
Status update:
1. This PR is ready for review.
2. This diff should help for the review, the Base is the commit after class copying (except RootedOzoneClientAdapterFactory): https://github.com/smengcl/hadoop-ozone/compare/0bee28acadfb2c358c6f008173e7eca6ed7fa23f...smengcl:HDDS-2840
3. I have also applied HDDS-2188 to affected OFS classes in order for code to compile.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] xiaoyuyao commented on a change in pull request
#415: HDDS-2840. Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
xiaoyuyao commented on a change in pull request #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#discussion_r373120426
##########
File path: hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
##########
@@ -0,0 +1,482 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.TestDataUtil;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClientException;
+import org.apache.hadoop.ozone.client.OzoneKeyDetails;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.Set;
+import java.util.TreeSet;
+
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/**
+ * Ozone file system tests that are not covered by contract tests.
+ *
+ * TODO: Refactor this and TestOzoneFileSystem to eliminate most
+ * code duplication.
+ */
+public class TestRootedOzoneFileSystem {
+
+ @Rule
+ public Timeout globalTimeout = new Timeout(300_000);
+
+ private static MiniOzoneCluster cluster = null;
+
+ private static FileSystem fs;
+ private static RootedOzoneFileSystem ofs;
+
+ private static ObjectStore objectStore;
+
+ private String volumeName;
+ private String bucketName;
+
+ private String rootPath;
+
+ // Store path commonly used by tests that test functionality within a bucket
+ private String testBucketStr;
+ private Path testBucketPath;
+
+ @Before
+ public void init() throws Exception {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ cluster = MiniOzoneCluster.newBuilder(conf)
+ .setNumDatanodes(3)
+ .build();
+ cluster.waitForClusterToBeReady();
+ objectStore = cluster.getClient().getObjectStore();
+
+ // create a volume and a bucket to be used by RootedOzoneFileSystem (OFS)
+ OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(cluster);
+ volumeName = bucket.getVolumeName();
+ bucketName = bucket.getName();
+ testBucketStr = "/" + volumeName + "/" + bucketName;
+ testBucketPath = new Path(testBucketStr);
+
+ rootPath = String.format("%s://%s/", OzoneConsts.OZONE_OFS_URI_SCHEME,
+ conf.get(OZONE_OM_ADDRESS_KEY));
+
+ // Set the fs.defaultFS and start the filesystem
+ conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, rootPath);
+ // Note: FileSystem#loadFileSystems won't load OFS class due to META-INF
+ // hence this workaround.
+ conf.set("fs.ofs.impl", "org.apache.hadoop.fs.ozone.RootedOzoneFileSystem");
+ fs = FileSystem.get(conf);
+ ofs = (RootedOzoneFileSystem) fs;
+ }
+
+ @After
+ public void teardown() {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ IOUtils.closeQuietly(fs);
+ }
+
+ @Test
+ public void testOzoneFsServiceLoader() throws IOException {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ // Note: FileSystem#loadFileSystems won't load OFS class due to META-INF
+ // hence this workaround.
+ conf.set("fs.ofs.impl", "org.apache.hadoop.fs.ozone.RootedOzoneFileSystem");
+ assertEquals(
+ FileSystem.getFileSystemClass(OzoneConsts.OZONE_OFS_URI_SCHEME, conf),
+ RootedOzoneFileSystem.class);
+ }
+
+ @Test
+ public void testCreateDoesNotAddParentDirKeys() throws Exception {
+ Path grandparent = new Path(testBucketPath,
+ "testCreateDoesNotAddParentDirKeys");
+ Path parent = new Path(grandparent, "parent");
+ Path child = new Path(parent, "child");
+ ContractTestUtils.touch(fs, child);
+
+ OzoneKeyDetails key = getKey(child, false);
+ OFSPath childOFSPath = new OFSPath(child);
+ assertEquals(key.getName(), childOFSPath.getKeyName());
+
+ // Creating a child should not add parent keys to the bucket
+ try {
+ getKey(parent, true);
+ } catch (IOException ex) {
+ assertKeyNotFoundException(ex);
+ }
+
+ // List status on the parent should show the child file
+ assertEquals("List status of parent should include the 1 child file", 1L,
+ (long)fs.listStatus(parent).length);
+ assertTrue("Parent directory does not appear to be a directory",
+ fs.getFileStatus(parent).isDirectory());
+ }
+
+ @Test
+ public void testDeleteCreatesFakeParentDir() throws Exception {
+ Path grandparent = new Path(testBucketPath,
+ "testDeleteCreatesFakeParentDir");
+ Path parent = new Path(grandparent, "parent");
+ Path child = new Path(parent, "child");
+ ContractTestUtils.touch(fs, child);
+
+ // Verify that parent dir key does not exist
+ // Creating a child should not add parent keys to the bucket
+ try {
+ getKey(parent, true);
+ } catch (IOException ex) {
+ assertKeyNotFoundException(ex);
+ }
+
+ // Delete the child key
+ assertTrue(fs.delete(child, false));
+
+ // Deleting the only child should create the parent dir key if it does
+ // not exist
+ OFSPath parentOFSPath = new OFSPath(parent);
+ String parentKey = parentOFSPath.getKeyName() + "/";
+ OzoneKeyDetails parentKeyInfo = getKey(parent, true);
+ assertEquals(parentKey, parentKeyInfo.getName());
+
+ // Recursive delete with DeleteIterator
+ assertTrue(fs.delete(grandparent, true));
+ }
+
+ @Test
+ public void testListStatus() throws Exception {
+ Path parent = new Path(testBucketPath, "testListStatus");
+ Path file1 = new Path(parent, "key1");
+ Path file2 = new Path(parent, "key2");
+
+ FileStatus[] fileStatuses = ofs.listStatus(testBucketPath);
+ assertEquals("Should be empty", 0, fileStatuses.length);
+
+ ContractTestUtils.touch(fs, file1);
+ ContractTestUtils.touch(fs, file2);
+
+ fileStatuses = ofs.listStatus(testBucketPath);
+ assertEquals("Should have created parent",
+ 1, fileStatuses.length);
+ assertEquals("Parent path doesn't match",
+ fileStatuses[0].getPath().toUri().getPath(), parent.toString());
+
+ // ListStatus on a directory should return all subdirs along with
+ // files, even if there exists a file and sub-dir with the same name.
+ fileStatuses = ofs.listStatus(parent);
+ assertEquals("FileStatus did not return all children of the directory",
+ 2, fileStatuses.length);
+
+ // ListStatus should return only the immediate children of a directory.
+ Path file3 = new Path(parent, "dir1/key3");
+ Path file4 = new Path(parent, "dir1/key4");
+ ContractTestUtils.touch(fs, file3);
+ ContractTestUtils.touch(fs, file4);
+ fileStatuses = ofs.listStatus(parent);
+ assertEquals("FileStatus did not return all children of the directory",
+ 3, fileStatuses.length);
+ }
+
+ /**
+ * OFS: Helper function for tests. Return a volume name that doesn't exist.
+ */
+ private String getRandomNonExistVolumeName() throws Exception {
+ final int numDigit = 5;
+ long retriesLeft = Math.round(Math.pow(10, 5));
+ String name = null;
+ while (name == null && retriesLeft-- > 0) {
+ name = "volume-" + RandomStringUtils.randomNumeric(numDigit);
+ // Check volume existence.
+ Iterator<? extends OzoneVolume> iter =
+ objectStore.listVolumesByUser(null, name, null);
+ if (iter.hasNext()) {
+ // If there is a match, try again.
+ // Note that volume name prefix match doesn't equal volume existence
+ // but the check is sufficient for this test.
+ name = null;
+ }
+ }
+ if (retriesLeft <= 0) {
+ throw new Exception(
+ "Failed to generate random volume name that doesn't exist already.");
+ }
+ return name;
+ }
+
+ /**
+ * OFS: Test mkdir on volume, bucket and dir that doesn't exist.
+ */
+ @Test
+ public void testMkdirOnNonExistentVolumeBucketDir() throws Exception {
+ String volumeNameLocal = getRandomNonExistVolumeName();
+ String bucketNameLocal = "bucket-" + RandomStringUtils.randomNumeric(5);
+ Path root = new Path("/" + volumeNameLocal + "/" + bucketNameLocal);
+ Path dir1 = new Path(root, "dir1");
+ Path dir12 = new Path(dir1, "dir12");
+ Path dir2 = new Path(root, "dir2");
+ fs.mkdirs(dir12);
+ fs.mkdirs(dir2);
+
+ // Check volume and bucket existence, they should both be created.
+ OzoneVolume ozoneVolume = objectStore.getVolume(volumeNameLocal);
+ OzoneBucket ozoneBucket = ozoneVolume.getBucket(bucketNameLocal);
+ OFSPath ofsPathDir1 = new OFSPath(dir12);
+ String key = ofsPathDir1.getKeyName() + "/";
+ OzoneKeyDetails ozoneKeyDetails = ozoneBucket.getKey(key);
+ assertEquals(key, ozoneKeyDetails.getName());
+
+ // Verify that directories are created.
+ FileStatus[] fileStatuses = ofs.listStatus(root);
+ assertEquals(fileStatuses[0].getPath().toUri().getPath(), dir1.toString());
+ assertEquals(fileStatuses[1].getPath().toUri().getPath(), dir2.toString());
+
+ fileStatuses = ofs.listStatus(dir1);
+ assertEquals(fileStatuses[0].getPath().toUri().getPath(), dir12.toString());
+ fileStatuses = ofs.listStatus(dir12);
+ assertEquals(fileStatuses.length, 0);
+ fileStatuses = ofs.listStatus(dir2);
+ assertEquals(fileStatuses.length, 0);
+ }
+
+ /**
+ * OFS: Tests mkdir on a volume and bucket that doesn't exist.
+ */
+ @Test
+ public void testMkdirNonExistentVolumeBucket() throws Exception {
+ String volumeNameLocal = getRandomNonExistVolumeName();
+ String bucketNameLocal = "bucket-" + RandomStringUtils.randomNumeric(5);
+ Path newVolBucket = new Path(
+ "/" + volumeNameLocal + "/" + bucketNameLocal);
+ fs.mkdirs(newVolBucket);
+
+ // Verify with listVolumes and listBuckets
+ Iterator<? extends OzoneVolume> iterVol =
+ objectStore.listVolumesByUser(null, volumeNameLocal, null);
+ OzoneVolume ozoneVolume = iterVol.next();
+ assertNotNull(ozoneVolume);
+ assertEquals(volumeNameLocal, ozoneVolume.getName());
+
+ Iterator<? extends OzoneBucket> iterBuc =
+ ozoneVolume.listBuckets("bucket-");
+ OzoneBucket ozoneBucket = iterBuc.next();
+ assertNotNull(ozoneBucket);
+ assertEquals(bucketNameLocal, ozoneBucket.getName());
+
+ // TODO: Use listStatus to check volume and bucket creation in HDDS-2928.
+ }
+
+ /**
+ * OFS: Tests mkdir on a volume that doesn't exist.
+ */
+ @Test
+ public void testMkdirNonExistentVolume() throws Exception {
+ String volumeNameLocal = getRandomNonExistVolumeName();
+ Path newVolume = new Path("/" + volumeNameLocal);
+ fs.mkdirs(newVolume);
+
+ // Verify with listVolumes and listBuckets
+ Iterator<? extends OzoneVolume> iterVol =
+ objectStore.listVolumesByUser(null, volumeNameLocal, null);
+ OzoneVolume ozoneVolume = iterVol.next();
+ assertNotNull(ozoneVolume);
+ assertEquals(volumeNameLocal, ozoneVolume.getName());
+
+ // TODO: Use listStatus to check volume and bucket creation in HDDS-2928.
+ }
+
+ /**
+ * Tests listStatus operation in a bucket.
+ */
+ @Test
+ public void testListStatusOnRoot() throws Exception {
+ Path root = new Path("/" + volumeName + "/" + bucketName);
+ Path dir1 = new Path(root, "dir1");
+ Path dir12 = new Path(dir1, "dir12");
+ Path dir2 = new Path(root, "dir2");
+ fs.mkdirs(dir12);
+ fs.mkdirs(dir2);
+
+ // ListStatus on root should return dir1 (even though /dir1 key does not
+ // exist) and dir2 only. dir12 is not an immediate child of root and
+ // hence should not be listed.
+ FileStatus[] fileStatuses = ofs.listStatus(root);
+ assertEquals("FileStatus should return only the immediate children", 2,
+ fileStatuses.length);
+
+ // Verify that dir12 is not included in the result of the listStatus on root
+ String fileStatus1 = fileStatuses[0].getPath().toUri().getPath();
+ String fileStatus2 = fileStatuses[1].getPath().toUri().getPath();
+ assertFalse(fileStatus1.equals(dir12.toString()));
+ assertFalse(fileStatus2.equals(dir12.toString()));
+ }
+
+ /**
+ * Tests listStatus operation on root directory.
+ */
+ @Test
+ public void testListStatusOnLargeDirectory() throws Exception {
+ Path root = new Path("/" + volumeName + "/" + bucketName);
+ Set<String> paths = new TreeSet<>();
+ int numDirs = 5111;
Review comment:
Consider reducing the numDirs as it will unnecessarily slow down the PR.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] smengcl commented on a change in pull request #415:
HDDS-2840. Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
smengcl commented on a change in pull request #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#discussion_r372559181
##########
File path: hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
##########
@@ -0,0 +1,694 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.ozone;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.key.KeyProvider;
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdds.client.ReplicationFactor;
+import org.apache.hadoop.hdds.client.ReplicationType;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ozone.OmUtils;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.client.OzoneKey;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
+import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
+import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenRenewer;
+
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .BUCKET_ALREADY_EXISTS;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .VOLUME_ALREADY_EXISTS;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .VOLUME_NOT_FOUND;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .BUCKET_NOT_FOUND;
+
+/**
+ * Basic Implementation of the OzoneFileSystem calls.
+ * <p>
+ * This is the minimal version which doesn't include any statistics.
+ * <p>
+ * For full featured version use OzoneClientAdapterImpl.
+ */
+public class BasicRootedOzoneClientAdapterImpl
+ implements RootedOzoneClientAdapter {
+
+ static final Logger LOG =
+ LoggerFactory.getLogger(BasicRootedOzoneClientAdapterImpl.class);
+
+ private OzoneClient ozoneClient;
+ private ClientProtocol proxy;
+ private ObjectStore objectStore;
+ private ReplicationType replicationType;
+ private ReplicationFactor replicationFactor;
+ private boolean securityEnabled;
+ private int configuredDnPort;
+
+ /**
+ * Create new OzoneClientAdapter implementation.
+ *
+ * @throws IOException In case of a problem.
+ */
+ public BasicRootedOzoneClientAdapterImpl() throws IOException {
+ this(createConf());
+ }
+
+ private static OzoneConfiguration createConf() {
+ ClassLoader contextClassLoader =
+ Thread.currentThread().getContextClassLoader();
+ Thread.currentThread().setContextClassLoader(null);
+ try {
+ return new OzoneConfiguration();
+ } finally {
+ Thread.currentThread().setContextClassLoader(contextClassLoader);
+ }
+ }
+
+ public BasicRootedOzoneClientAdapterImpl(OzoneConfiguration conf)
+ throws IOException {
+ this(null, -1, conf);
+ }
+
+ public BasicRootedOzoneClientAdapterImpl(String omHost, int omPort,
+ Configuration hadoopConf) throws IOException {
+
+ ClassLoader contextClassLoader =
+ Thread.currentThread().getContextClassLoader();
+ Thread.currentThread().setContextClassLoader(null);
+
+ try {
+ OzoneConfiguration conf = OzoneConfiguration.of(hadoopConf);
+
+ if (omHost == null && OmUtils.isServiceIdsDefined(conf)) {
+ // When the host name or service id isn't given
+ // but ozone.om.service.ids is defined, declare failure.
+
+ // This is a safety precaution that prevents the client from
+ // accidentally failing over to an unintended OM.
+ throw new IllegalArgumentException("Service ID or host name must not"
+ + " be omitted when ozone.om.service.ids is defined.");
+ }
+
+ if (omPort != -1) {
+ // When the port number is specified, perform the following check
+ if (OmUtils.isOmHAServiceId(conf, omHost)) {
+ // If omHost is a service id, it shouldn't use a port
+ throw new IllegalArgumentException("Port " + omPort +
+ " specified in URI but host '" + omHost + "' is a "
+ + "logical (HA) OzoneManager and does not use port information.");
+ }
+ } else {
+ // When port number is not specified, read it from config
+ omPort = OmUtils.getOmRpcPort(conf);
+ }
+
+ SecurityConfig secConfig = new SecurityConfig(conf);
+
+ if (secConfig.isSecurityEnabled()) {
+ this.securityEnabled = true;
+ }
+
+ String replicationTypeConf =
+ conf.get(OzoneConfigKeys.OZONE_REPLICATION_TYPE,
+ OzoneConfigKeys.OZONE_REPLICATION_TYPE_DEFAULT);
+
+ int replicationCountConf = conf.getInt(OzoneConfigKeys.OZONE_REPLICATION,
+ OzoneConfigKeys.OZONE_REPLICATION_DEFAULT);
+
+ if (OmUtils.isOmHAServiceId(conf, omHost)) {
+ // omHost is listed as one of the service ids in the config,
+ // thus we should treat omHost as omServiceId
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(omHost, conf);
+ } else if (StringUtils.isNotEmpty(omHost) && omPort != -1) {
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(omHost, omPort, conf);
+ } else {
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(conf);
+ }
+ objectStore = ozoneClient.getObjectStore();
+ proxy = objectStore.getClientProxy();
+ this.replicationType = ReplicationType.valueOf(replicationTypeConf);
+ this.replicationFactor = ReplicationFactor.valueOf(replicationCountConf);
+ this.configuredDnPort = conf.getInt(
+ OzoneConfigKeys.DFS_CONTAINER_IPC_PORT,
+ OzoneConfigKeys.DFS_CONTAINER_IPC_PORT_DEFAULT);
+ } finally {
+ Thread.currentThread().setContextClassLoader(contextClassLoader);
+ }
+ }
+
+ OzoneBucket getBucket(OFSPath ofsPath, boolean createIfNotExist)
+ throws IOException {
+
+ return getBucket(ofsPath.getVolumeName(), ofsPath.getBucketName(),
+ createIfNotExist);
+ }
+
+ /**
+ * Get OzoneBucket object to operate in.
+ * Optionally create volume and bucket if not found.
+ *
+ * @param createIfNotExist Set this to true if the caller is a write operation
+ * in order to create the volume and bucket.
+ * @throws IOException Exceptions other than OMException with result code
+ * VOLUME_NOT_FOUND or BUCKET_NOT_FOUND.
+ */
+ private OzoneBucket getBucket(String volumeStr, String bucketStr,
+ boolean createIfNotExist) throws IOException {
+
+ OzoneBucket bucket;
+ try {
+ bucket = proxy.getBucketDetails(volumeStr, bucketStr);
+ } catch (OMException ex) {
+ if (createIfNotExist) {
+ // Note: getBucketDetails always throws BUCKET_NOT_FOUND, even if
+ // the volume doesn't exist.
+ if (ex.getResult().equals(BUCKET_NOT_FOUND)) {
+ OzoneVolume volume;
+ try {
+ volume = proxy.getVolumeDetails(volumeStr);
+ } catch (OMException getVolEx) {
+ if (getVolEx.getResult().equals(VOLUME_NOT_FOUND)) {
+ // Volume doesn't exist. Create it
+ try {
+ objectStore.createVolume(volumeStr);
+ } catch (OMException newVolEx) {
+ // Ignore the case where another client created the volume
+ if (!newVolEx.getResult().equals(VOLUME_ALREADY_EXISTS)) {
+ throw newVolEx;
+ }
+ }
+ } else {
+ throw getVolEx;
+ }
+ // Try get volume again
+ volume = proxy.getVolumeDetails(volumeStr);
+ }
+ // Create the bucket
+ try {
+ volume.createBucket(bucketStr);
+ } catch (OMException newBucEx) {
+ // Ignore the case where another client created the bucket
+ if (!newBucEx.getResult().equals(BUCKET_ALREADY_EXISTS)) {
+ throw newBucEx;
+ }
+ }
+ }
+ // Try get bucket again
+ bucket = proxy.getBucketDetails(volumeStr, bucketStr);
+ } else {
+ throw ex;
+ }
+ }
+
+ return bucket;
+ }
+
+ @Override
+ public void close() throws IOException {
+ ozoneClient.close();
+ }
+
+ @Override
+ public InputStream readFile(String pathStr) throws IOException {
+ incrementCounter(Statistic.OBJECTS_READ);
+ OFSPath ofsPath = new OFSPath(pathStr);
+ String key = ofsPath.getKeyName();
+ try {
+ OzoneBucket bucket = getBucket(ofsPath, false);
+ return bucket.readFile(key).getInputStream();
+ } catch (OMException ex) {
+ if (ex.getResult() == OMException.ResultCodes.FILE_NOT_FOUND
+ || ex.getResult() == OMException.ResultCodes.NOT_A_FILE) {
+ throw new FileNotFoundException(
+ ex.getResult().name() + ": " + ex.getMessage());
+ } else {
+ throw ex;
+ }
+ }
+ }
+
+ protected void incrementCounter(Statistic objectsRead) {
+ //noop: Use OzoneClientAdapterImpl which supports statistics.
+ }
+
+ @Override
+ public OzoneFSOutputStream createFile(String pathStr, boolean overWrite,
+ boolean recursive) throws IOException {
+ incrementCounter(Statistic.OBJECTS_CREATED);
+ OFSPath ofsPath = new OFSPath(pathStr);
+ String key = ofsPath.getKeyName();
+ try {
+ OzoneBucket bucket = getBucket(ofsPath, false);
+ OzoneOutputStream ozoneOutputStream = bucket.createFile(
+ key, 0, replicationType, replicationFactor, overWrite, recursive);
+ return new OzoneFSOutputStream(ozoneOutputStream.getOutputStream());
+ } catch (OMException ex) {
+ if (ex.getResult() == OMException.ResultCodes.FILE_ALREADY_EXISTS
+ || ex.getResult() == OMException.ResultCodes.NOT_A_FILE) {
+ throw new FileAlreadyExistsException(
+ ex.getResult().name() + ": " + ex.getMessage());
+ } else {
+ throw ex;
+ }
+ }
+ }
+
+ @Override
+ public void renamePath(String path, String newPath) throws IOException {
Review comment:
Done in 5c1c08178a4d230ded059c8f11e82f19a375a862
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] hanishakoneru commented on a change in pull request
#415: HDDS-2840. Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
hanishakoneru commented on a change in pull request #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#discussion_r369308830
##########
File path: hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
##########
@@ -0,0 +1,408 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.TestDataUtil;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClientException;
+import org.apache.hadoop.ozone.client.OzoneKeyDetails;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.Set;
+import java.util.TreeSet;
+
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/**
+ * Ozone file system tests that are not covered by contract tests.
+ */
+public class TestRootedOzoneFileSystem {
+
+ @Rule
+ public Timeout globalTimeout = new Timeout(300_000);
+
+ private static MiniOzoneCluster cluster = null;
+
+ private static FileSystem fs;
+ private static RootedOzoneFileSystem ofs;
+
+ private static ObjectStore objectStore;
+
+ private String volumeName;
+ private String bucketName;
+
+ private String rootPath;
+
+ @Before
+ public void init() throws Exception {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ cluster = MiniOzoneCluster.newBuilder(conf)
+ .setNumDatanodes(3)
+ .build();
+ cluster.waitForClusterToBeReady();
+ objectStore = cluster.getClient().getObjectStore();
+
+ // create a volume and a bucket to be used by OFileSystem
+ OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(cluster);
+ volumeName = bucket.getVolumeName();
+ bucketName = bucket.getName();
+
+ // For now:
+ rootPath = String.format("%s://%s/", OzoneConsts.OZONE_OFS_URI_SCHEME,
+ conf.get(OZONE_OM_ADDRESS_KEY));
+
+ // Set the fs.defaultFS and start the filesystem
+ conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, rootPath);
+ // Note: FileSystem#loadFileSystems won't load OFS class due to META-INF
+ // hence this workaround.
+ conf.set("fs.ofs.impl", "org.apache.hadoop.fs.ozone.RootedOzoneFileSystem");
+ fs = FileSystem.get(conf);
+ ofs = (RootedOzoneFileSystem) fs;
+ }
+
+ @After
+ public void teardown() throws IOException {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ IOUtils.closeQuietly(fs);
+ }
+
+ @Test
+ public void testOzoneFsServiceLoader() throws IOException {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ // Note: FileSystem#loadFileSystems won't load OFS class due to META-INF
+ // hence this workaround.
+ conf.set("fs.ofs.impl", "org.apache.hadoop.fs.ozone.RootedOzoneFileSystem");
+ assertEquals(
+ FileSystem.getFileSystemClass(OzoneConsts.OZONE_OFS_URI_SCHEME, conf),
+ RootedOzoneFileSystem.class);
+ }
+
+ @Test
+ public void testCreateDoesNotAddParentDirKeys() throws Exception {
+ Path rootBucket = new Path("/" + volumeName + "/" + bucketName);
+ Path grandparent = new Path(rootBucket,
+ "testCreateDoesNotAddParentDirKeys");
+ Path parent = new Path(grandparent, "parent");
+ Path child = new Path(parent, "child");
+ ContractTestUtils.touch(fs, child);
+
+ OzoneKeyDetails key = getKeyInBucket(child, false);
+ OFSPath childOFSPath = new OFSPath(child);
+ assertEquals(key.getName(), childOFSPath.getKeyName());
+
+ // Creating a child should not add parent keys to the bucket
+ try {
+ getKeyInBucket(parent, true);
+ } catch (IOException ex) {
+ assertKeyNotFoundException(ex);
+ }
+
+ // List status on the parent should show the child file
+ assertEquals("List status of parent should include the 1 child file", 1L,
+ (long)fs.listStatus(parent).length);
+ assertTrue("Parent directory does not appear to be a directory",
+ fs.getFileStatus(parent).isDirectory());
+ }
+
+ @Test
+ public void testDeleteCreatesFakeParentDir() throws Exception {
+ Path rootBucket = new Path("/" + volumeName + "/" + bucketName);
+ Path grandparent = new Path(rootBucket, "testDeleteCreatesFakeParentDir");
+ Path parent = new Path(grandparent, "parent");
+ Path child = new Path(parent, "child");
+ ContractTestUtils.touch(fs, child);
+
+ // Verify that parent dir key does not exist
+ // Creating a child should not add parent keys to the bucket
+ try {
+ getKeyInBucket(parent, true);
+ } catch (IOException ex) {
+ assertKeyNotFoundException(ex);
+ }
+
+ // Delete the child key
+ assertTrue(fs.delete(child, false));
+
+ // Deleting the only child should create the parent dir key if it does
+ // not exist
+ OFSPath parentOFSPath = new OFSPath(parent);
+ String parentKey = parentOFSPath.getKeyName() + "/";
+ OzoneKeyDetails parentKeyInfo = getKeyInBucket(parent, true);
+ assertEquals(parentKey, parentKeyInfo.getName());
+
+ // Recursive delete with DeleteIterator
+ assertTrue(fs.delete(grandparent, true));
+ }
+
+ @Test
+ public void testListStatus() throws Exception {
+ Path rootBucket = new Path("/" + volumeName + "/" + bucketName);
+ Path parent = new Path(rootBucket, "testListStatus");
+ Path file1 = new Path(parent, "key1");
+ Path file2 = new Path(parent, "key2");
+
+ FileStatus[] fileStatuses = ofs.listStatus(rootBucket);
+ assertEquals("Should be empty", 0, fileStatuses.length);
+
+ ContractTestUtils.touch(fs, file1);
+ ContractTestUtils.touch(fs, file2);
+
+ // ListStatus on a directory should return all subdirs along with
+ // files, even if there exists a file and sub-dir with the same name.
+ fileStatuses = ofs.listStatus(parent);
+ assertEquals("FileStatus did not return all children of the directory",
+ 2, fileStatuses.length);
+
+ // ListStatus should return only the immediate children of a directory.
+ Path file3 = new Path(parent, "dir1/key3");
+ Path file4 = new Path(parent, "dir1/key4");
+ ContractTestUtils.touch(fs, file3);
+ ContractTestUtils.touch(fs, file4);
+ fileStatuses = ofs.listStatus(parent);
+ assertEquals("FileStatus did not return all children of the directory",
+ 3, fileStatuses.length);
+ }
+
+ private String getRandomNonExistVolumeName() throws Exception {
+ final int numDigit = 5;
+ long retriesLeft = Math.round(Math.pow(10, 5));
+ String name = null;
+ while (name == null && retriesLeft-- > 0) {
+ name = "volume-" + RandomStringUtils.randomNumeric(numDigit);
+ // Check volume existence.
+ Iterator<? extends OzoneVolume> iter =
+ objectStore.listVolumesByUser(null, name, null);
+ if (iter.hasNext()) {
+ // If there is a match, try again.
+ // Note that volume name prefix match doesn't equal volume existence
+ // but the check is sufficient for this test.
+ name = null;
+ }
+ }
+ if (retriesLeft <= 0) {
+ throw new Exception(
+ "Failed to generate random volume name that doesn't exist already.");
+ }
+ return name;
+ }
+
+ /**
+ * Tests Mkdir operation on a volume that doesn't exist.
+ * Expect Mkdir to create the volume and bucket.
+ */
+ @Test
+ public void testMkdirOnNonExistentVolumeBucket() throws Exception {
+ String volumeNameLocal = getRandomNonExistVolumeName();
+ String bucketNameLocal = "bucket-" + RandomStringUtils.randomNumeric(5);
+ Path root = new Path("/" + volumeNameLocal + "/" + bucketNameLocal);
+ Path dir1 = new Path(root, "dir1");
+ Path dir12 = new Path(dir1, "dir12");
+ Path dir2 = new Path(root, "dir2");
+ fs.mkdirs(dir12);
+ fs.mkdirs(dir2);
+
+ // Check volume and bucket existence, they should both be created.
+ OzoneVolume ozoneVolume = objectStore.getVolume(volumeNameLocal);
+ OzoneBucket ozoneBucket = ozoneVolume.getBucket(bucketNameLocal);
+ OFSPath ofsPathDir1 = new OFSPath(dir12);
+ String key = ofsPathDir1.getKeyName() + "/";
+ OzoneKeyDetails ozoneKeyDetails = ozoneBucket.getKey(key);
+ assertEquals(key, ozoneKeyDetails.getName());
+
+ // Verify that directories are created.
+ FileStatus[] fileStatuses = ofs.listStatus(root);
+ assertEquals(fileStatuses[0].getPath().toUri().getPath(), dir1.toString());
+ assertEquals(fileStatuses[1].getPath().toUri().getPath(), dir2.toString());
+
+ fileStatuses = ofs.listStatus(dir1);
+ assertEquals(fileStatuses[0].getPath().toUri().getPath(), dir12.toString());
+ fileStatuses = ofs.listStatus(dir12);
+ assertEquals(fileStatuses.length, 0);
+ fileStatuses = ofs.listStatus(dir2);
+ assertEquals(fileStatuses.length, 0);
+ }
+
+ /**
+ * Tests listStatus operation on root directory.
+ */
+ @Test
+ public void testListStatusOnRoot() throws Exception {
+ Path root = new Path("/" + volumeName + "/" + bucketName);
+ Path dir1 = new Path(root, "dir1");
+ Path dir12 = new Path(dir1, "dir12");
+ Path dir2 = new Path(root, "dir2");
+ fs.mkdirs(dir12);
+ fs.mkdirs(dir2);
+
+ // ListStatus on root should return dir1 (even though /dir1 key does not
+ // exist) and dir2 only. dir12 is not an immediate child of root and
+ // hence should not be listed.
+ FileStatus[] fileStatuses = ofs.listStatus(root);
+ assertEquals("FileStatus should return only the immediate children", 2,
+ fileStatuses.length);
+
+ // Verify that dir12 is not included in the result of the listStatus on root
+ String fileStatus1 = fileStatuses[0].getPath().toUri().getPath();
+ String fileStatus2 = fileStatuses[1].getPath().toUri().getPath();
+ assertFalse(fileStatus1.equals(dir12.toString()));
+ assertFalse(fileStatus2.equals(dir12.toString()));
+ }
+
+ /**
+ * Tests listStatus operation on root directory.
+ */
+ @Test
+ public void testListStatusOnLargeDirectory() throws Exception {
+ Path root = new Path("/" + volumeName + "/" + bucketName);
+ Set<String> paths = new TreeSet<>();
+ int numDirs = 5111;
+ for(int i = 0; i < numDirs; i++) {
+ Path p = new Path(root, String.valueOf(i));
+ fs.mkdirs(p);
+ paths.add(p.getName());
+ }
+
+ FileStatus[] fileStatuses = ofs.listStatus(root);
+ assertEquals(
+ "Total directories listed do not match the existing directories",
+ numDirs, fileStatuses.length);
+
+ for (int i=0; i < numDirs; i++) {
+ assertTrue(paths.contains(fileStatuses[i].getPath().getName()));
+ }
+ }
+
+ /**
+ * Tests listStatus on a path with subdirs.
+ */
+ @Test
+ public void testListStatusOnSubDirs() throws Exception {
+ // Create the following key structure
+ // /dir1/dir11/dir111
+ // /dir1/dir12
+ // /dir1/dir12/file121
+ // /dir2
+ // ListStatus on /dir1 should return all its immediated subdirs only
+ // which are /dir1/dir11 and /dir1/dir12. Super child files/dirs
+ // (/dir1/dir12/file121 and /dir1/dir11/dir111) should not be returned by
+ // listStatus.
+ Path rootBucket = new Path("/" + volumeName + "/" + bucketName);
+ Path dir1 = new Path(rootBucket, "dir1");
+ Path dir11 = new Path(dir1, "dir11");
+ Path dir111 = new Path(dir11, "dir111");
+ Path dir12 = new Path(dir1, "dir12");
+ Path file121 = new Path(dir12, "file121");
+ Path dir2 = new Path(rootBucket, "dir2");
+ fs.mkdirs(dir111);
+ fs.mkdirs(dir12);
+ ContractTestUtils.touch(fs, file121);
+ fs.mkdirs(dir2);
+
+ FileStatus[] fileStatuses = ofs.listStatus(dir1);
+ assertEquals("FileStatus should return only the immediate children", 2,
+ fileStatuses.length);
+
+ // Verify that the two children of /dir1 returned by listStatus operation
+ // are /dir1/dir11 and /dir1/dir12.
+ String fileStatus1 = fileStatuses[0].getPath().toUri().getPath();
+ String fileStatus2 = fileStatuses[1].getPath().toUri().getPath();
+ assertTrue(fileStatus1.equals(dir11.toString()) ||
+ fileStatus1.equals(dir12.toString()));
+ assertTrue(fileStatus2.equals(dir11.toString()) ||
+ fileStatus2.equals(dir12.toString()));
+ }
+
+ @Test
+ public void testNonExplicitlyCreatedPathExistsAfterItsLeafsWereRemoved()
+ throws Exception {
+ Path rootBucket = new Path("/" + volumeName + "/" + bucketName);
+ Path source = new Path(rootBucket, "source");
+ Path interimPath = new Path(source, "interimPath");
+ Path leafInsideInterimPath = new Path(interimPath, "leaf");
+ Path target = new Path(rootBucket, "target");
+ Path leafInTarget = new Path(target, "leaf");
+
+ fs.mkdirs(source);
+ fs.mkdirs(target);
+ fs.mkdirs(leafInsideInterimPath);
+
+ // Attempt to rename the key to a different bucket
+ Path bucket2 = new Path("/" + volumeName + "/" + bucketName + "test");
+ Path leafInTargetInAnotherBucket = new Path(bucket2, "leaf");
+ try {
+ fs.rename(leafInsideInterimPath, leafInTargetInAnotherBucket);
+ fail("Should have thrown exception when renaming to a different bucket");
+ } catch (IOException ex) {
+ System.out.println("Exception thrown as expected");
Review comment:
I think we can avoid printing this here. Just a comment would suffice.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] smengcl commented on a change in pull request #415:
HDDS-2840. Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
smengcl commented on a change in pull request #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#discussion_r371196346
##########
File path: hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
##########
@@ -0,0 +1,805 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.ozone;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.CreateFlag;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.Progressable;
+import org.apache.http.client.utils.URIBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.EnumSet;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+import static org.apache.hadoop.fs.ozone.Constants.LISTING_PAGE_SIZE;
+import static org.apache.hadoop.fs.ozone.Constants.OZONE_DEFAULT_USER;
+import static org.apache.hadoop.fs.ozone.Constants.OZONE_USER_DIR;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_OFS_URI_SCHEME;
+
+/**
+ * The minimal Ozone Filesystem implementation.
+ * <p>
+ * This is a basic version which doesn't extend
+ * KeyProviderTokenIssuer and doesn't include statistics. It can be used
+ * from older hadoop version. For newer hadoop version use the full featured
+ * OFileSystem.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class BasicRootedOzoneFileSystem extends FileSystem {
+ static final Logger LOG =
+ LoggerFactory.getLogger(BasicRootedOzoneFileSystem.class);
+
+ /**
+ * The Ozone client for connecting to Ozone server.
+ */
+
+ private URI uri;
+ private String userName;
+ private Path workingDir;
+// private String gOmHost;
+// private int gOmPort;
+// private Configuration gConf;
+// private boolean gIsolatedClassloader;
+
+ private RootedOzoneClientAdapter adapter;
+// private String adapterPath;
+
+ private static final String URI_EXCEPTION_TEXT =
+ "URL should be one of the following formats: " +
+ "ofs://om-service-id/ OR " +
+ "ofs://om-host.example.com/ OR " +
+ "ofs://om-host.example.com:5678/";
Review comment:
Yep. Done: da12d487fe5c5bb05f013c33cb2d02aba96b63c3
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] smengcl commented on a change in pull request #415:
HDDS-2840. Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
smengcl commented on a change in pull request #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#discussion_r372547250
##########
File path: hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
##########
@@ -0,0 +1,694 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.ozone;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.key.KeyProvider;
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdds.client.ReplicationFactor;
+import org.apache.hadoop.hdds.client.ReplicationType;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ozone.OmUtils;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.client.OzoneKey;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
+import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
+import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenRenewer;
+
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .BUCKET_ALREADY_EXISTS;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .VOLUME_ALREADY_EXISTS;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .VOLUME_NOT_FOUND;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .BUCKET_NOT_FOUND;
+
+/**
+ * Basic Implementation of the OzoneFileSystem calls.
+ * <p>
+ * This is the minimal version which doesn't include any statistics.
+ * <p>
+ * For full featured version use OzoneClientAdapterImpl.
+ */
+public class BasicRootedOzoneClientAdapterImpl
+ implements RootedOzoneClientAdapter {
+
+ static final Logger LOG =
+ LoggerFactory.getLogger(BasicRootedOzoneClientAdapterImpl.class);
+
+ private OzoneClient ozoneClient;
+ private ClientProtocol proxy;
+ private ObjectStore objectStore;
+ private ReplicationType replicationType;
+ private ReplicationFactor replicationFactor;
+ private boolean securityEnabled;
+ private int configuredDnPort;
+
+ /**
+ * Create new OzoneClientAdapter implementation.
+ *
+ * @throws IOException In case of a problem.
+ */
+ public BasicRootedOzoneClientAdapterImpl() throws IOException {
+ this(createConf());
+ }
+
+ private static OzoneConfiguration createConf() {
+ ClassLoader contextClassLoader =
+ Thread.currentThread().getContextClassLoader();
+ Thread.currentThread().setContextClassLoader(null);
+ try {
+ return new OzoneConfiguration();
+ } finally {
+ Thread.currentThread().setContextClassLoader(contextClassLoader);
+ }
+ }
+
+ public BasicRootedOzoneClientAdapterImpl(OzoneConfiguration conf)
+ throws IOException {
+ this(null, -1, conf);
+ }
+
+ public BasicRootedOzoneClientAdapterImpl(String omHost, int omPort,
+ Configuration hadoopConf) throws IOException {
+
+ ClassLoader contextClassLoader =
+ Thread.currentThread().getContextClassLoader();
+ Thread.currentThread().setContextClassLoader(null);
+
+ try {
+ OzoneConfiguration conf = OzoneConfiguration.of(hadoopConf);
+
+ if (omHost == null && OmUtils.isServiceIdsDefined(conf)) {
+ // When the host name or service id isn't given
+ // but ozone.om.service.ids is defined, declare failure.
+
+ // This is a safety precaution that prevents the client from
+ // accidentally failing over to an unintended OM.
+ throw new IllegalArgumentException("Service ID or host name must not"
+ + " be omitted when ozone.om.service.ids is defined.");
+ }
+
+ if (omPort != -1) {
+ // When the port number is specified, perform the following check
+ if (OmUtils.isOmHAServiceId(conf, omHost)) {
+ // If omHost is a service id, it shouldn't use a port
+ throw new IllegalArgumentException("Port " + omPort +
+ " specified in URI but host '" + omHost + "' is a "
+ + "logical (HA) OzoneManager and does not use port information.");
+ }
+ } else {
+ // When port number is not specified, read it from config
+ omPort = OmUtils.getOmRpcPort(conf);
+ }
+
+ SecurityConfig secConfig = new SecurityConfig(conf);
+
+ if (secConfig.isSecurityEnabled()) {
+ this.securityEnabled = true;
+ }
+
+ String replicationTypeConf =
+ conf.get(OzoneConfigKeys.OZONE_REPLICATION_TYPE,
+ OzoneConfigKeys.OZONE_REPLICATION_TYPE_DEFAULT);
+
+ int replicationCountConf = conf.getInt(OzoneConfigKeys.OZONE_REPLICATION,
+ OzoneConfigKeys.OZONE_REPLICATION_DEFAULT);
+
+ if (OmUtils.isOmHAServiceId(conf, omHost)) {
+ // omHost is listed as one of the service ids in the config,
+ // thus we should treat omHost as omServiceId
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(omHost, conf);
+ } else if (StringUtils.isNotEmpty(omHost) && omPort != -1) {
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(omHost, omPort, conf);
+ } else {
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(conf);
+ }
+ objectStore = ozoneClient.getObjectStore();
+ proxy = objectStore.getClientProxy();
+ this.replicationType = ReplicationType.valueOf(replicationTypeConf);
+ this.replicationFactor = ReplicationFactor.valueOf(replicationCountConf);
+ this.configuredDnPort = conf.getInt(
+ OzoneConfigKeys.DFS_CONTAINER_IPC_PORT,
+ OzoneConfigKeys.DFS_CONTAINER_IPC_PORT_DEFAULT);
+ } finally {
+ Thread.currentThread().setContextClassLoader(contextClassLoader);
+ }
+ }
+
+ OzoneBucket getBucket(OFSPath ofsPath, boolean createIfNotExist)
+ throws IOException {
+
+ return getBucket(ofsPath.getVolumeName(), ofsPath.getBucketName(),
+ createIfNotExist);
+ }
+
+ /**
+ * Get OzoneBucket object to operate in.
+ * Optionally create volume and bucket if not found.
+ *
+ * @param createIfNotExist Set this to true if the caller is a write operation
+ * in order to create the volume and bucket.
+ * @throws IOException Exceptions other than OMException with result code
+ * VOLUME_NOT_FOUND or BUCKET_NOT_FOUND.
+ */
+ private OzoneBucket getBucket(String volumeStr, String bucketStr,
+ boolean createIfNotExist) throws IOException {
+
+ OzoneBucket bucket;
+ try {
+ bucket = proxy.getBucketDetails(volumeStr, bucketStr);
+ } catch (OMException ex) {
+ if (createIfNotExist) {
+ // Note: getBucketDetails always throws BUCKET_NOT_FOUND, even if
+ // the volume doesn't exist.
+ if (ex.getResult().equals(BUCKET_NOT_FOUND)) {
+ OzoneVolume volume;
+ try {
+ volume = proxy.getVolumeDetails(volumeStr);
+ } catch (OMException getVolEx) {
+ if (getVolEx.getResult().equals(VOLUME_NOT_FOUND)) {
+ // Volume doesn't exist. Create it
+ try {
+ objectStore.createVolume(volumeStr);
+ } catch (OMException newVolEx) {
+ // Ignore the case where another client created the volume
+ if (!newVolEx.getResult().equals(VOLUME_ALREADY_EXISTS)) {
Review comment:
As we discussed, currently in Ozone `createVolume` will set owner to current user when `volArg` owner is null: https://github.com/smengcl/hadoop-ozone/blob/046a06f02783da516179ee8d8d1bed862d22f78d/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java#L268
Can we resolve this one?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] smengcl commented on a change in pull request #415:
HDDS-2840. Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
smengcl commented on a change in pull request #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#discussion_r372552921
##########
File path: hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
##########
@@ -0,0 +1,694 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.ozone;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.key.KeyProvider;
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdds.client.ReplicationFactor;
+import org.apache.hadoop.hdds.client.ReplicationType;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ozone.OmUtils;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.client.OzoneKey;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
+import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
+import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenRenewer;
+
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .BUCKET_ALREADY_EXISTS;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .VOLUME_ALREADY_EXISTS;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .VOLUME_NOT_FOUND;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .BUCKET_NOT_FOUND;
+
+/**
+ * Basic Implementation of the OzoneFileSystem calls.
+ * <p>
+ * This is the minimal version which doesn't include any statistics.
+ * <p>
+ * For full featured version use OzoneClientAdapterImpl.
+ */
+public class BasicRootedOzoneClientAdapterImpl
+ implements RootedOzoneClientAdapter {
+
+ static final Logger LOG =
+ LoggerFactory.getLogger(BasicRootedOzoneClientAdapterImpl.class);
+
+ private OzoneClient ozoneClient;
+ private ClientProtocol proxy;
+ private ObjectStore objectStore;
+ private ReplicationType replicationType;
+ private ReplicationFactor replicationFactor;
+ private boolean securityEnabled;
+ private int configuredDnPort;
+
+ /**
+ * Create new OzoneClientAdapter implementation.
+ *
+ * @throws IOException In case of a problem.
+ */
+ public BasicRootedOzoneClientAdapterImpl() throws IOException {
+ this(createConf());
+ }
+
+ private static OzoneConfiguration createConf() {
+ ClassLoader contextClassLoader =
+ Thread.currentThread().getContextClassLoader();
+ Thread.currentThread().setContextClassLoader(null);
+ try {
+ return new OzoneConfiguration();
+ } finally {
+ Thread.currentThread().setContextClassLoader(contextClassLoader);
+ }
+ }
+
+ public BasicRootedOzoneClientAdapterImpl(OzoneConfiguration conf)
+ throws IOException {
+ this(null, -1, conf);
+ }
+
+ public BasicRootedOzoneClientAdapterImpl(String omHost, int omPort,
+ Configuration hadoopConf) throws IOException {
+
+ ClassLoader contextClassLoader =
+ Thread.currentThread().getContextClassLoader();
+ Thread.currentThread().setContextClassLoader(null);
+
+ try {
+ OzoneConfiguration conf = OzoneConfiguration.of(hadoopConf);
+
+ if (omHost == null && OmUtils.isServiceIdsDefined(conf)) {
+ // When the host name or service id isn't given
+ // but ozone.om.service.ids is defined, declare failure.
+
+ // This is a safety precaution that prevents the client from
+ // accidentally failing over to an unintended OM.
+ throw new IllegalArgumentException("Service ID or host name must not"
+ + " be omitted when ozone.om.service.ids is defined.");
+ }
+
+ if (omPort != -1) {
+ // When the port number is specified, perform the following check
+ if (OmUtils.isOmHAServiceId(conf, omHost)) {
+ // If omHost is a service id, it shouldn't use a port
+ throw new IllegalArgumentException("Port " + omPort +
+ " specified in URI but host '" + omHost + "' is a "
+ + "logical (HA) OzoneManager and does not use port information.");
+ }
+ } else {
+ // When port number is not specified, read it from config
+ omPort = OmUtils.getOmRpcPort(conf);
+ }
+
+ SecurityConfig secConfig = new SecurityConfig(conf);
+
+ if (secConfig.isSecurityEnabled()) {
+ this.securityEnabled = true;
+ }
+
+ String replicationTypeConf =
+ conf.get(OzoneConfigKeys.OZONE_REPLICATION_TYPE,
+ OzoneConfigKeys.OZONE_REPLICATION_TYPE_DEFAULT);
+
+ int replicationCountConf = conf.getInt(OzoneConfigKeys.OZONE_REPLICATION,
+ OzoneConfigKeys.OZONE_REPLICATION_DEFAULT);
+
+ if (OmUtils.isOmHAServiceId(conf, omHost)) {
+ // omHost is listed as one of the service ids in the config,
+ // thus we should treat omHost as omServiceId
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(omHost, conf);
+ } else if (StringUtils.isNotEmpty(omHost) && omPort != -1) {
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(omHost, omPort, conf);
+ } else {
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(conf);
+ }
+ objectStore = ozoneClient.getObjectStore();
+ proxy = objectStore.getClientProxy();
+ this.replicationType = ReplicationType.valueOf(replicationTypeConf);
+ this.replicationFactor = ReplicationFactor.valueOf(replicationCountConf);
+ this.configuredDnPort = conf.getInt(
+ OzoneConfigKeys.DFS_CONTAINER_IPC_PORT,
+ OzoneConfigKeys.DFS_CONTAINER_IPC_PORT_DEFAULT);
+ } finally {
+ Thread.currentThread().setContextClassLoader(contextClassLoader);
+ }
+ }
+
+ OzoneBucket getBucket(OFSPath ofsPath, boolean createIfNotExist)
+ throws IOException {
+
+ return getBucket(ofsPath.getVolumeName(), ofsPath.getBucketName(),
+ createIfNotExist);
+ }
+
+ /**
+ * Get OzoneBucket object to operate in.
+ * Optionally create volume and bucket if not found.
+ *
+ * @param createIfNotExist Set this to true if the caller is a write operation
+ * in order to create the volume and bucket.
+ * @throws IOException Exceptions other than OMException with result code
+ * VOLUME_NOT_FOUND or BUCKET_NOT_FOUND.
+ */
+ private OzoneBucket getBucket(String volumeStr, String bucketStr,
+ boolean createIfNotExist) throws IOException {
+
+ OzoneBucket bucket;
+ try {
+ bucket = proxy.getBucketDetails(volumeStr, bucketStr);
+ } catch (OMException ex) {
+ if (createIfNotExist) {
+ // Note: getBucketDetails always throws BUCKET_NOT_FOUND, even if
+ // the volume doesn't exist.
+ if (ex.getResult().equals(BUCKET_NOT_FOUND)) {
+ OzoneVolume volume;
+ try {
+ volume = proxy.getVolumeDetails(volumeStr);
+ } catch (OMException getVolEx) {
+ if (getVolEx.getResult().equals(VOLUME_NOT_FOUND)) {
+ // Volume doesn't exist. Create it
+ try {
+ objectStore.createVolume(volumeStr);
+ } catch (OMException newVolEx) {
+ // Ignore the case where another client created the volume
+ if (!newVolEx.getResult().equals(VOLUME_ALREADY_EXISTS)) {
+ throw newVolEx;
+ }
+ }
+ } else {
+ throw getVolEx;
+ }
+ // Try get volume again
+ volume = proxy.getVolumeDetails(volumeStr);
+ }
+ // Create the bucket
+ try {
+ volume.createBucket(bucketStr);
+ } catch (OMException newBucEx) {
+ // Ignore the case where another client created the bucket
+ if (!newBucEx.getResult().equals(BUCKET_ALREADY_EXISTS)) {
+ throw newBucEx;
+ }
+ }
+ }
+ // Try get bucket again
+ bucket = proxy.getBucketDetails(volumeStr, bucketStr);
+ } else {
+ throw ex;
+ }
+ }
+
+ return bucket;
+ }
+
+ @Override
+ public void close() throws IOException {
+ ozoneClient.close();
+ }
+
+ @Override
+ public InputStream readFile(String pathStr) throws IOException {
+ incrementCounter(Statistic.OBJECTS_READ);
+ OFSPath ofsPath = new OFSPath(pathStr);
+ String key = ofsPath.getKeyName();
+ try {
+ OzoneBucket bucket = getBucket(ofsPath, false);
+ return bucket.readFile(key).getInputStream();
+ } catch (OMException ex) {
+ if (ex.getResult() == OMException.ResultCodes.FILE_NOT_FOUND
+ || ex.getResult() == OMException.ResultCodes.NOT_A_FILE) {
+ throw new FileNotFoundException(
+ ex.getResult().name() + ": " + ex.getMessage());
+ } else {
+ throw ex;
+ }
+ }
+ }
+
+ protected void incrementCounter(Statistic objectsRead) {
+ //noop: Use OzoneClientAdapterImpl which supports statistics.
+ }
+
+ @Override
+ public OzoneFSOutputStream createFile(String pathStr, boolean overWrite,
+ boolean recursive) throws IOException {
+ incrementCounter(Statistic.OBJECTS_CREATED);
+ OFSPath ofsPath = new OFSPath(pathStr);
+ String key = ofsPath.getKeyName();
+ try {
+ OzoneBucket bucket = getBucket(ofsPath, false);
+ OzoneOutputStream ozoneOutputStream = bucket.createFile(
+ key, 0, replicationType, replicationFactor, overWrite, recursive);
+ return new OzoneFSOutputStream(ozoneOutputStream.getOutputStream());
+ } catch (OMException ex) {
+ if (ex.getResult() == OMException.ResultCodes.FILE_ALREADY_EXISTS
+ || ex.getResult() == OMException.ResultCodes.NOT_A_FILE) {
+ throw new FileAlreadyExistsException(
+ ex.getResult().name() + ": " + ex.getMessage());
+ } else {
+ throw ex;
+ }
+ }
+ }
+
+ @Override
+ public void renamePath(String path, String newPath) throws IOException {
+ incrementCounter(Statistic.OBJECTS_RENAMED);
+ OFSPath ofsPath = new OFSPath(path);
+ OFSPath ofsNewPath = new OFSPath(newPath);
+
+ // Check path and newPathName are in the same volume and same bucket.
+ // This should have been checked in BasicRootedOzoneFileSystem#rename
+ // already via regular call path unless bypassed.
+ if (!ofsPath.isInSameBucketAs(ofsNewPath)) {
+ throw new IOException("Cannot rename a key to a different bucket");
+ }
+
+ OzoneBucket bucket = getBucket(ofsPath, false);
+ String key = ofsPath.getKeyName();
+ String newKey = ofsNewPath.getKeyName();
+ bucket.renameKey(key, newKey);
+ }
+
+ /**
+ * Package-private helper function to reduce calls to getBucket().
+ * @param bucket Bucket to operate in.
+ * @param path Existing key path.
+ * @param newPath New key path.
+ * @throws IOException IOException from bucket.renameKey().
+ */
+ void renamePath(OzoneBucket bucket, String path, String newPath)
+ throws IOException {
+ incrementCounter(Statistic.OBJECTS_RENAMED);
+ OFSPath ofsPath = new OFSPath(path);
+ OFSPath ofsNewPath = new OFSPath(newPath);
+ // No same-bucket policy check here since this call path is controlled
+ String key = ofsPath.getKeyName();
+ String newKey = ofsNewPath.getKeyName();
+ bucket.renameKey(key, newKey);
+ }
+
+ /**
+ * Helper method to create an directory specified by key name in bucket.
+ *
+ * @param pathStr path to be created as directory
+ * @return true if the key is created, false otherwise
+ */
+ @Override
+ public boolean createDirectory(String pathStr) throws IOException {
+ LOG.trace("creating dir for path: {}", pathStr);
+ incrementCounter(Statistic.OBJECTS_CREATED);
+ OFSPath ofsPath = new OFSPath(pathStr);
+ // Volume name unspecified, return failure
+ if (ofsPath.getVolumeName().length() == 0) {
+ return false;
+ }
+ // Handle where only volume is specified in pathStr
+ if (ofsPath.getBucketName().length() == 0) {
+ objectStore.createVolume(ofsPath.getVolumeName());
+ return true;
+ }
+ String keyStr = ofsPath.getKeyName();
Review comment:
Comment updated in c0e2ed7f4cf08490b151613fdb3415b097ee5ac0.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] smengcl edited a comment on issue #415: HDDS-2840.
Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
smengcl edited a comment on issue #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#issuecomment-574954076
> Thanks for working on this @smengcl. I have started reviewing the PR.
>
> In BasicRootedOzoneClientAdapterImpl, setting volume and bucket globally could lead to discrepancies.
> Let's say my client creates one OzoneClient and multiple threads share this object.
> Thread1 - createFile /vol1/bucket1/key1
> Thread2 - createFile /vol1/bucket2/key2
>
> ```
> t1 - Thread1 sets bucket = bucket1
> t2 - Thread2 sets bucket = bucket2
> t3 - Thread1 executes bucket.createFile -> This would create key1 in bucket2.
> ```
Good point. The current implementation is not thread-safe.
I believe I can just make `volume` and `bucket` local in each function to eliminate such race condition.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] smengcl commented on a change in pull request #415:
HDDS-2840. Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
smengcl commented on a change in pull request #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#discussion_r373141001
##########
File path: hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
##########
@@ -0,0 +1,482 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.TestDataUtil;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClientException;
+import org.apache.hadoop.ozone.client.OzoneKeyDetails;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.Set;
+import java.util.TreeSet;
+
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/**
+ * Ozone file system tests that are not covered by contract tests.
+ *
+ * TODO: Refactor this and TestOzoneFileSystem to eliminate most
+ * code duplication.
+ */
+public class TestRootedOzoneFileSystem {
+
+ @Rule
+ public Timeout globalTimeout = new Timeout(300_000);
+
+ private static MiniOzoneCluster cluster = null;
+
+ private static FileSystem fs;
+ private static RootedOzoneFileSystem ofs;
+
+ private static ObjectStore objectStore;
+
+ private String volumeName;
+ private String bucketName;
+
+ private String rootPath;
+
+ // Store path commonly used by tests that test functionality within a bucket
+ private String testBucketStr;
+ private Path testBucketPath;
+
+ @Before
+ public void init() throws Exception {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ cluster = MiniOzoneCluster.newBuilder(conf)
+ .setNumDatanodes(3)
+ .build();
+ cluster.waitForClusterToBeReady();
+ objectStore = cluster.getClient().getObjectStore();
+
+ // create a volume and a bucket to be used by RootedOzoneFileSystem (OFS)
+ OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(cluster);
+ volumeName = bucket.getVolumeName();
+ bucketName = bucket.getName();
+ testBucketStr = "/" + volumeName + "/" + bucketName;
+ testBucketPath = new Path(testBucketStr);
+
+ rootPath = String.format("%s://%s/", OzoneConsts.OZONE_OFS_URI_SCHEME,
+ conf.get(OZONE_OM_ADDRESS_KEY));
+
+ // Set the fs.defaultFS and start the filesystem
+ conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, rootPath);
+ // Note: FileSystem#loadFileSystems won't load OFS class due to META-INF
+ // hence this workaround.
+ conf.set("fs.ofs.impl", "org.apache.hadoop.fs.ozone.RootedOzoneFileSystem");
+ fs = FileSystem.get(conf);
+ ofs = (RootedOzoneFileSystem) fs;
+ }
+
+ @After
+ public void teardown() {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ IOUtils.closeQuietly(fs);
+ }
+
+ @Test
+ public void testOzoneFsServiceLoader() throws IOException {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ // Note: FileSystem#loadFileSystems won't load OFS class due to META-INF
+ // hence this workaround.
+ conf.set("fs.ofs.impl", "org.apache.hadoop.fs.ozone.RootedOzoneFileSystem");
+ assertEquals(
+ FileSystem.getFileSystemClass(OzoneConsts.OZONE_OFS_URI_SCHEME, conf),
+ RootedOzoneFileSystem.class);
+ }
+
+ @Test
+ public void testCreateDoesNotAddParentDirKeys() throws Exception {
+ Path grandparent = new Path(testBucketPath,
+ "testCreateDoesNotAddParentDirKeys");
+ Path parent = new Path(grandparent, "parent");
+ Path child = new Path(parent, "child");
+ ContractTestUtils.touch(fs, child);
+
+ OzoneKeyDetails key = getKey(child, false);
+ OFSPath childOFSPath = new OFSPath(child);
+ assertEquals(key.getName(), childOFSPath.getKeyName());
+
+ // Creating a child should not add parent keys to the bucket
+ try {
+ getKey(parent, true);
+ } catch (IOException ex) {
+ assertKeyNotFoundException(ex);
+ }
+
+ // List status on the parent should show the child file
+ assertEquals("List status of parent should include the 1 child file", 1L,
+ (long)fs.listStatus(parent).length);
+ assertTrue("Parent directory does not appear to be a directory",
+ fs.getFileStatus(parent).isDirectory());
+ }
+
+ @Test
+ public void testDeleteCreatesFakeParentDir() throws Exception {
+ Path grandparent = new Path(testBucketPath,
+ "testDeleteCreatesFakeParentDir");
+ Path parent = new Path(grandparent, "parent");
+ Path child = new Path(parent, "child");
+ ContractTestUtils.touch(fs, child);
+
+ // Verify that parent dir key does not exist
+ // Creating a child should not add parent keys to the bucket
+ try {
+ getKey(parent, true);
+ } catch (IOException ex) {
+ assertKeyNotFoundException(ex);
+ }
+
+ // Delete the child key
+ assertTrue(fs.delete(child, false));
+
+ // Deleting the only child should create the parent dir key if it does
+ // not exist
+ OFSPath parentOFSPath = new OFSPath(parent);
+ String parentKey = parentOFSPath.getKeyName() + "/";
+ OzoneKeyDetails parentKeyInfo = getKey(parent, true);
+ assertEquals(parentKey, parentKeyInfo.getName());
+
+ // Recursive delete with DeleteIterator
+ assertTrue(fs.delete(grandparent, true));
+ }
+
+ @Test
+ public void testListStatus() throws Exception {
+ Path parent = new Path(testBucketPath, "testListStatus");
+ Path file1 = new Path(parent, "key1");
+ Path file2 = new Path(parent, "key2");
+
+ FileStatus[] fileStatuses = ofs.listStatus(testBucketPath);
+ assertEquals("Should be empty", 0, fileStatuses.length);
+
+ ContractTestUtils.touch(fs, file1);
+ ContractTestUtils.touch(fs, file2);
+
+ fileStatuses = ofs.listStatus(testBucketPath);
+ assertEquals("Should have created parent",
+ 1, fileStatuses.length);
+ assertEquals("Parent path doesn't match",
+ fileStatuses[0].getPath().toUri().getPath(), parent.toString());
+
+ // ListStatus on a directory should return all subdirs along with
+ // files, even if there exists a file and sub-dir with the same name.
+ fileStatuses = ofs.listStatus(parent);
+ assertEquals("FileStatus did not return all children of the directory",
+ 2, fileStatuses.length);
+
+ // ListStatus should return only the immediate children of a directory.
+ Path file3 = new Path(parent, "dir1/key3");
+ Path file4 = new Path(parent, "dir1/key4");
+ ContractTestUtils.touch(fs, file3);
+ ContractTestUtils.touch(fs, file4);
+ fileStatuses = ofs.listStatus(parent);
+ assertEquals("FileStatus did not return all children of the directory",
+ 3, fileStatuses.length);
+ }
+
+ /**
+ * OFS: Helper function for tests. Return a volume name that doesn't exist.
+ */
+ private String getRandomNonExistVolumeName() throws Exception {
+ final int numDigit = 5;
+ long retriesLeft = Math.round(Math.pow(10, 5));
+ String name = null;
+ while (name == null && retriesLeft-- > 0) {
+ name = "volume-" + RandomStringUtils.randomNumeric(numDigit);
+ // Check volume existence.
+ Iterator<? extends OzoneVolume> iter =
+ objectStore.listVolumesByUser(null, name, null);
+ if (iter.hasNext()) {
+ // If there is a match, try again.
+ // Note that volume name prefix match doesn't equal volume existence
+ // but the check is sufficient for this test.
+ name = null;
+ }
+ }
+ if (retriesLeft <= 0) {
+ throw new Exception(
+ "Failed to generate random volume name that doesn't exist already.");
+ }
+ return name;
+ }
+
+ /**
+ * OFS: Test mkdir on volume, bucket and dir that doesn't exist.
+ */
+ @Test
+ public void testMkdirOnNonExistentVolumeBucketDir() throws Exception {
+ String volumeNameLocal = getRandomNonExistVolumeName();
+ String bucketNameLocal = "bucket-" + RandomStringUtils.randomNumeric(5);
+ Path root = new Path("/" + volumeNameLocal + "/" + bucketNameLocal);
+ Path dir1 = new Path(root, "dir1");
+ Path dir12 = new Path(dir1, "dir12");
+ Path dir2 = new Path(root, "dir2");
+ fs.mkdirs(dir12);
+ fs.mkdirs(dir2);
+
+ // Check volume and bucket existence, they should both be created.
+ OzoneVolume ozoneVolume = objectStore.getVolume(volumeNameLocal);
+ OzoneBucket ozoneBucket = ozoneVolume.getBucket(bucketNameLocal);
+ OFSPath ofsPathDir1 = new OFSPath(dir12);
+ String key = ofsPathDir1.getKeyName() + "/";
+ OzoneKeyDetails ozoneKeyDetails = ozoneBucket.getKey(key);
+ assertEquals(key, ozoneKeyDetails.getName());
+
+ // Verify that directories are created.
+ FileStatus[] fileStatuses = ofs.listStatus(root);
+ assertEquals(fileStatuses[0].getPath().toUri().getPath(), dir1.toString());
+ assertEquals(fileStatuses[1].getPath().toUri().getPath(), dir2.toString());
+
+ fileStatuses = ofs.listStatus(dir1);
+ assertEquals(fileStatuses[0].getPath().toUri().getPath(), dir12.toString());
+ fileStatuses = ofs.listStatus(dir12);
+ assertEquals(fileStatuses.length, 0);
+ fileStatuses = ofs.listStatus(dir2);
+ assertEquals(fileStatuses.length, 0);
+ }
+
+ /**
+ * OFS: Tests mkdir on a volume and bucket that doesn't exist.
+ */
+ @Test
+ public void testMkdirNonExistentVolumeBucket() throws Exception {
+ String volumeNameLocal = getRandomNonExistVolumeName();
+ String bucketNameLocal = "bucket-" + RandomStringUtils.randomNumeric(5);
+ Path newVolBucket = new Path(
+ "/" + volumeNameLocal + "/" + bucketNameLocal);
+ fs.mkdirs(newVolBucket);
+
+ // Verify with listVolumes and listBuckets
+ Iterator<? extends OzoneVolume> iterVol =
+ objectStore.listVolumesByUser(null, volumeNameLocal, null);
+ OzoneVolume ozoneVolume = iterVol.next();
+ assertNotNull(ozoneVolume);
+ assertEquals(volumeNameLocal, ozoneVolume.getName());
+
+ Iterator<? extends OzoneBucket> iterBuc =
+ ozoneVolume.listBuckets("bucket-");
+ OzoneBucket ozoneBucket = iterBuc.next();
+ assertNotNull(ozoneBucket);
+ assertEquals(bucketNameLocal, ozoneBucket.getName());
+
+ // TODO: Use listStatus to check volume and bucket creation in HDDS-2928.
+ }
+
+ /**
+ * OFS: Tests mkdir on a volume that doesn't exist.
+ */
+ @Test
+ public void testMkdirNonExistentVolume() throws Exception {
+ String volumeNameLocal = getRandomNonExistVolumeName();
+ Path newVolume = new Path("/" + volumeNameLocal);
+ fs.mkdirs(newVolume);
+
+ // Verify with listVolumes and listBuckets
+ Iterator<? extends OzoneVolume> iterVol =
+ objectStore.listVolumesByUser(null, volumeNameLocal, null);
+ OzoneVolume ozoneVolume = iterVol.next();
+ assertNotNull(ozoneVolume);
+ assertEquals(volumeNameLocal, ozoneVolume.getName());
+
+ // TODO: Use listStatus to check volume and bucket creation in HDDS-2928.
+ }
+
+ /**
+ * Tests listStatus operation in a bucket.
+ */
+ @Test
+ public void testListStatusOnRoot() throws Exception {
+ Path root = new Path("/" + volumeName + "/" + bucketName);
+ Path dir1 = new Path(root, "dir1");
+ Path dir12 = new Path(dir1, "dir12");
+ Path dir2 = new Path(root, "dir2");
+ fs.mkdirs(dir12);
+ fs.mkdirs(dir2);
+
+ // ListStatus on root should return dir1 (even though /dir1 key does not
+ // exist) and dir2 only. dir12 is not an immediate child of root and
+ // hence should not be listed.
+ FileStatus[] fileStatuses = ofs.listStatus(root);
+ assertEquals("FileStatus should return only the immediate children", 2,
+ fileStatuses.length);
+
+ // Verify that dir12 is not included in the result of the listStatus on root
+ String fileStatus1 = fileStatuses[0].getPath().toUri().getPath();
+ String fileStatus2 = fileStatuses[1].getPath().toUri().getPath();
+ assertFalse(fileStatus1.equals(dir12.toString()));
+ assertFalse(fileStatus2.equals(dir12.toString()));
+ }
+
+ /**
+ * Tests listStatus operation on root directory.
+ */
+ @Test
+ public void testListStatusOnLargeDirectory() throws Exception {
+ Path root = new Path("/" + volumeName + "/" + bucketName);
+ Set<String> paths = new TreeSet<>();
+ int numDirs = 5111;
Review comment:
I'm not sure why o3fs test chose this random number. But basically `numDirs` only need to be larger than `LISTING_PAGE_SIZE` so it tests pagination. And BTW I believe integration tests are disabled in PR by default now. (They are not run in PR checks.)
Reduced in `0d7100604334f06fe304b143a3b906c9cbfc88dd`.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] hanishakoneru commented on a change in pull request
#415: HDDS-2840. Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
hanishakoneru commented on a change in pull request #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#discussion_r369308556
##########
File path: hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
##########
@@ -0,0 +1,408 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.TestDataUtil;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClientException;
+import org.apache.hadoop.ozone.client.OzoneKeyDetails;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.Set;
+import java.util.TreeSet;
+
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/**
+ * Ozone file system tests that are not covered by contract tests.
+ */
+public class TestRootedOzoneFileSystem {
+
+ @Rule
+ public Timeout globalTimeout = new Timeout(300_000);
+
+ private static MiniOzoneCluster cluster = null;
+
+ private static FileSystem fs;
+ private static RootedOzoneFileSystem ofs;
+
+ private static ObjectStore objectStore;
+
+ private String volumeName;
+ private String bucketName;
+
+ private String rootPath;
+
+ @Before
+ public void init() throws Exception {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ cluster = MiniOzoneCluster.newBuilder(conf)
+ .setNumDatanodes(3)
+ .build();
+ cluster.waitForClusterToBeReady();
+ objectStore = cluster.getClient().getObjectStore();
+
+ // create a volume and a bucket to be used by OFileSystem
+ OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(cluster);
+ volumeName = bucket.getVolumeName();
+ bucketName = bucket.getName();
+
+ // For now:
+ rootPath = String.format("%s://%s/", OzoneConsts.OZONE_OFS_URI_SCHEME,
+ conf.get(OZONE_OM_ADDRESS_KEY));
+
+ // Set the fs.defaultFS and start the filesystem
+ conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, rootPath);
+ // Note: FileSystem#loadFileSystems won't load OFS class due to META-INF
+ // hence this workaround.
+ conf.set("fs.ofs.impl", "org.apache.hadoop.fs.ozone.RootedOzoneFileSystem");
+ fs = FileSystem.get(conf);
+ ofs = (RootedOzoneFileSystem) fs;
+ }
+
+ @After
+ public void teardown() throws IOException {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ IOUtils.closeQuietly(fs);
+ }
+
+ @Test
+ public void testOzoneFsServiceLoader() throws IOException {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ // Note: FileSystem#loadFileSystems won't load OFS class due to META-INF
+ // hence this workaround.
+ conf.set("fs.ofs.impl", "org.apache.hadoop.fs.ozone.RootedOzoneFileSystem");
+ assertEquals(
+ FileSystem.getFileSystemClass(OzoneConsts.OZONE_OFS_URI_SCHEME, conf),
+ RootedOzoneFileSystem.class);
+ }
+
+ @Test
+ public void testCreateDoesNotAddParentDirKeys() throws Exception {
+ Path rootBucket = new Path("/" + volumeName + "/" + bucketName);
+ Path grandparent = new Path(rootBucket,
+ "testCreateDoesNotAddParentDirKeys");
+ Path parent = new Path(grandparent, "parent");
+ Path child = new Path(parent, "child");
+ ContractTestUtils.touch(fs, child);
+
+ OzoneKeyDetails key = getKeyInBucket(child, false);
+ OFSPath childOFSPath = new OFSPath(child);
+ assertEquals(key.getName(), childOFSPath.getKeyName());
+
+ // Creating a child should not add parent keys to the bucket
+ try {
+ getKeyInBucket(parent, true);
+ } catch (IOException ex) {
+ assertKeyNotFoundException(ex);
+ }
+
+ // List status on the parent should show the child file
+ assertEquals("List status of parent should include the 1 child file", 1L,
+ (long)fs.listStatus(parent).length);
+ assertTrue("Parent directory does not appear to be a directory",
+ fs.getFileStatus(parent).isDirectory());
+ }
+
+ @Test
+ public void testDeleteCreatesFakeParentDir() throws Exception {
+ Path rootBucket = new Path("/" + volumeName + "/" + bucketName);
+ Path grandparent = new Path(rootBucket, "testDeleteCreatesFakeParentDir");
+ Path parent = new Path(grandparent, "parent");
+ Path child = new Path(parent, "child");
+ ContractTestUtils.touch(fs, child);
+
+ // Verify that parent dir key does not exist
+ // Creating a child should not add parent keys to the bucket
+ try {
+ getKeyInBucket(parent, true);
+ } catch (IOException ex) {
+ assertKeyNotFoundException(ex);
+ }
+
+ // Delete the child key
+ assertTrue(fs.delete(child, false));
+
+ // Deleting the only child should create the parent dir key if it does
+ // not exist
+ OFSPath parentOFSPath = new OFSPath(parent);
+ String parentKey = parentOFSPath.getKeyName() + "/";
+ OzoneKeyDetails parentKeyInfo = getKeyInBucket(parent, true);
+ assertEquals(parentKey, parentKeyInfo.getName());
+
+ // Recursive delete with DeleteIterator
+ assertTrue(fs.delete(grandparent, true));
+ }
+
+ @Test
+ public void testListStatus() throws Exception {
+ Path rootBucket = new Path("/" + volumeName + "/" + bucketName);
+ Path parent = new Path(rootBucket, "testListStatus");
+ Path file1 = new Path(parent, "key1");
+ Path file2 = new Path(parent, "key2");
+
+ FileStatus[] fileStatuses = ofs.listStatus(rootBucket);
+ assertEquals("Should be empty", 0, fileStatuses.length);
+
+ ContractTestUtils.touch(fs, file1);
+ ContractTestUtils.touch(fs, file2);
+
+ // ListStatus on a directory should return all subdirs along with
+ // files, even if there exists a file and sub-dir with the same name.
+ fileStatuses = ofs.listStatus(parent);
+ assertEquals("FileStatus did not return all children of the directory",
+ 2, fileStatuses.length);
+
+ // ListStatus should return only the immediate children of a directory.
+ Path file3 = new Path(parent, "dir1/key3");
+ Path file4 = new Path(parent, "dir1/key4");
+ ContractTestUtils.touch(fs, file3);
+ ContractTestUtils.touch(fs, file4);
+ fileStatuses = ofs.listStatus(parent);
+ assertEquals("FileStatus did not return all children of the directory",
+ 3, fileStatuses.length);
+ }
+
+ private String getRandomNonExistVolumeName() throws Exception {
+ final int numDigit = 5;
+ long retriesLeft = Math.round(Math.pow(10, 5));
+ String name = null;
+ while (name == null && retriesLeft-- > 0) {
+ name = "volume-" + RandomStringUtils.randomNumeric(numDigit);
+ // Check volume existence.
+ Iterator<? extends OzoneVolume> iter =
+ objectStore.listVolumesByUser(null, name, null);
+ if (iter.hasNext()) {
+ // If there is a match, try again.
+ // Note that volume name prefix match doesn't equal volume existence
+ // but the check is sufficient for this test.
+ name = null;
+ }
+ }
+ if (retriesLeft <= 0) {
+ throw new Exception(
+ "Failed to generate random volume name that doesn't exist already.");
+ }
+ return name;
+ }
+
+ /**
+ * Tests Mkdir operation on a volume that doesn't exist.
+ * Expect Mkdir to create the volume and bucket.
+ */
+ @Test
+ public void testMkdirOnNonExistentVolumeBucket() throws Exception {
+ String volumeNameLocal = getRandomNonExistVolumeName();
+ String bucketNameLocal = "bucket-" + RandomStringUtils.randomNumeric(5);
+ Path root = new Path("/" + volumeNameLocal + "/" + bucketNameLocal);
+ Path dir1 = new Path(root, "dir1");
+ Path dir12 = new Path(dir1, "dir12");
+ Path dir2 = new Path(root, "dir2");
+ fs.mkdirs(dir12);
+ fs.mkdirs(dir2);
+
+ // Check volume and bucket existence, they should both be created.
+ OzoneVolume ozoneVolume = objectStore.getVolume(volumeNameLocal);
+ OzoneBucket ozoneBucket = ozoneVolume.getBucket(bucketNameLocal);
+ OFSPath ofsPathDir1 = new OFSPath(dir12);
+ String key = ofsPathDir1.getKeyName() + "/";
+ OzoneKeyDetails ozoneKeyDetails = ozoneBucket.getKey(key);
+ assertEquals(key, ozoneKeyDetails.getName());
+
+ // Verify that directories are created.
+ FileStatus[] fileStatuses = ofs.listStatus(root);
+ assertEquals(fileStatuses[0].getPath().toUri().getPath(), dir1.toString());
+ assertEquals(fileStatuses[1].getPath().toUri().getPath(), dir2.toString());
+
+ fileStatuses = ofs.listStatus(dir1);
+ assertEquals(fileStatuses[0].getPath().toUri().getPath(), dir12.toString());
+ fileStatuses = ofs.listStatus(dir12);
+ assertEquals(fileStatuses.length, 0);
+ fileStatuses = ofs.listStatus(dir2);
+ assertEquals(fileStatuses.length, 0);
+ }
+
+ /**
+ * Tests listStatus operation on root directory.
+ */
+ @Test
+ public void testListStatusOnRoot() throws Exception {
+ Path root = new Path("/" + volumeName + "/" + bucketName);
+ Path dir1 = new Path(root, "dir1");
+ Path dir12 = new Path(dir1, "dir12");
+ Path dir2 = new Path(root, "dir2");
+ fs.mkdirs(dir12);
+ fs.mkdirs(dir2);
+
+ // ListStatus on root should return dir1 (even though /dir1 key does not
+ // exist) and dir2 only. dir12 is not an immediate child of root and
+ // hence should not be listed.
+ FileStatus[] fileStatuses = ofs.listStatus(root);
+ assertEquals("FileStatus should return only the immediate children", 2,
+ fileStatuses.length);
+
+ // Verify that dir12 is not included in the result of the listStatus on root
+ String fileStatus1 = fileStatuses[0].getPath().toUri().getPath();
+ String fileStatus2 = fileStatuses[1].getPath().toUri().getPath();
+ assertFalse(fileStatus1.equals(dir12.toString()));
+ assertFalse(fileStatus2.equals(dir12.toString()));
+ }
+
+ /**
+ * Tests listStatus operation on root directory.
+ */
+ @Test
+ public void testListStatusOnLargeDirectory() throws Exception {
+ Path root = new Path("/" + volumeName + "/" + bucketName);
+ Set<String> paths = new TreeSet<>();
+ int numDirs = 5111;
+ for(int i = 0; i < numDirs; i++) {
+ Path p = new Path(root, String.valueOf(i));
+ fs.mkdirs(p);
+ paths.add(p.getName());
+ }
+
+ FileStatus[] fileStatuses = ofs.listStatus(root);
+ assertEquals(
+ "Total directories listed do not match the existing directories",
+ numDirs, fileStatuses.length);
+
+ for (int i=0; i < numDirs; i++) {
+ assertTrue(paths.contains(fileStatuses[i].getPath().getName()));
+ }
+ }
+
+ /**
+ * Tests listStatus on a path with subdirs.
+ */
+ @Test
+ public void testListStatusOnSubDirs() throws Exception {
+ // Create the following key structure
+ // /dir1/dir11/dir111
+ // /dir1/dir12
+ // /dir1/dir12/file121
+ // /dir2
+ // ListStatus on /dir1 should return all its immediated subdirs only
+ // which are /dir1/dir11 and /dir1/dir12. Super child files/dirs
+ // (/dir1/dir12/file121 and /dir1/dir11/dir111) should not be returned by
+ // listStatus.
+ Path rootBucket = new Path("/" + volumeName + "/" + bucketName);
+ Path dir1 = new Path(rootBucket, "dir1");
+ Path dir11 = new Path(dir1, "dir11");
+ Path dir111 = new Path(dir11, "dir111");
+ Path dir12 = new Path(dir1, "dir12");
+ Path file121 = new Path(dir12, "file121");
+ Path dir2 = new Path(rootBucket, "dir2");
+ fs.mkdirs(dir111);
+ fs.mkdirs(dir12);
+ ContractTestUtils.touch(fs, file121);
+ fs.mkdirs(dir2);
+
+ FileStatus[] fileStatuses = ofs.listStatus(dir1);
+ assertEquals("FileStatus should return only the immediate children", 2,
+ fileStatuses.length);
+
+ // Verify that the two children of /dir1 returned by listStatus operation
+ // are /dir1/dir11 and /dir1/dir12.
+ String fileStatus1 = fileStatuses[0].getPath().toUri().getPath();
+ String fileStatus2 = fileStatuses[1].getPath().toUri().getPath();
+ assertTrue(fileStatus1.equals(dir11.toString()) ||
+ fileStatus1.equals(dir12.toString()));
+ assertTrue(fileStatus2.equals(dir11.toString()) ||
+ fileStatus2.equals(dir12.toString()));
+ }
+
+ @Test
+ public void testNonExplicitlyCreatedPathExistsAfterItsLeafsWereRemoved()
+ throws Exception {
+ Path rootBucket = new Path("/" + volumeName + "/" + bucketName);
+ Path source = new Path(rootBucket, "source");
+ Path interimPath = new Path(source, "interimPath");
+ Path leafInsideInterimPath = new Path(interimPath, "leaf");
+ Path target = new Path(rootBucket, "target");
+ Path leafInTarget = new Path(target, "leaf");
+
+ fs.mkdirs(source);
+ fs.mkdirs(target);
+ fs.mkdirs(leafInsideInterimPath);
+
+ // Attempt to rename the key to a different bucket
+ Path bucket2 = new Path("/" + volumeName + "/" + bucketName + "test");
Review comment:
Let's test "rename to different bucket" in a separate method.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] hanishakoneru commented on a change in pull request
#415: HDDS-2840. Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
hanishakoneru commented on a change in pull request #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#discussion_r369857294
##########
File path: hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
##########
@@ -0,0 +1,805 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.ozone;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.CreateFlag;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.Progressable;
+import org.apache.http.client.utils.URIBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.EnumSet;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+import static org.apache.hadoop.fs.ozone.Constants.LISTING_PAGE_SIZE;
+import static org.apache.hadoop.fs.ozone.Constants.OZONE_DEFAULT_USER;
+import static org.apache.hadoop.fs.ozone.Constants.OZONE_USER_DIR;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_OFS_URI_SCHEME;
+
+/**
+ * The minimal Ozone Filesystem implementation.
+ * <p>
+ * This is a basic version which doesn't extend
+ * KeyProviderTokenIssuer and doesn't include statistics. It can be used
+ * from older hadoop version. For newer hadoop version use the full featured
+ * OFileSystem.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class BasicRootedOzoneFileSystem extends FileSystem {
+ static final Logger LOG =
+ LoggerFactory.getLogger(BasicRootedOzoneFileSystem.class);
+
+ /**
+ * The Ozone client for connecting to Ozone server.
+ */
+
+ private URI uri;
+ private String userName;
+ private Path workingDir;
+// private String gOmHost;
+// private int gOmPort;
+// private Configuration gConf;
+// private boolean gIsolatedClassloader;
+
+ private RootedOzoneClientAdapter adapter;
+// private String adapterPath;
+
+ private static final String URI_EXCEPTION_TEXT =
+ "URL should be one of the following formats: " +
+ "ofs://om-service-id/ OR " +
+ "ofs://om-host.example.com/ OR " +
+ "ofs://om-host.example.com:5678/";
+
+ @Override
+ public void initialize(URI name, Configuration conf) throws IOException {
+ super.initialize(name, conf);
+ setConf(conf);
+ Objects.requireNonNull(name.getScheme(), "No scheme provided in " + name);
+ Preconditions.checkArgument(getScheme().equals(name.getScheme()),
+ "Invalid scheme provided in " + name);
+
+ String authority = name.getAuthority();
+ if (authority == null) {
+ // authority is null when fs.defaultFS is not a qualified ofs URI and
+ // ofs:/// is passed to the client. matcher will NPE if authority is null
+ throw new IllegalArgumentException(URI_EXCEPTION_TEXT);
+ }
+
+ String omHost;
+ int omPort = -1;
+ // Parse hostname and port
+ String[] parts = authority.split(":");
+ if (parts.length > 2) {
+ throw new IllegalArgumentException(URI_EXCEPTION_TEXT);
+ }
+ omHost = parts[0];
+ if (parts.length == 2) {
+ try {
+ omPort = Integer.parseInt(parts[1]);
+ } catch (NumberFormatException e) {
+ throw new IllegalArgumentException(URI_EXCEPTION_TEXT);
+ }
+ }
+
+ try {
+ uri = new URIBuilder().setScheme(OZONE_OFS_URI_SCHEME)
+ .setHost(authority)
+ .build();
+ LOG.trace("Ozone URI for OFS initialization is " + uri);
+
+ //isolated is the default for ozonefs-lib-legacy which includes the
+ // /ozonefs.txt, otherwise the default is false. It could be overridden.
+ boolean defaultValue =
+ BasicRootedOzoneFileSystem.class.getClassLoader()
+ .getResource("ozonefs.txt") != null;
+
+ //Use string here instead of the constant as constant may not be available
+ //on the classpath of a hadoop 2.7
+ boolean isolatedClassloader =
+ conf.getBoolean("ozone.fs.isolated-classloader", defaultValue);
+
+ // adapter should be initialized in operations.
+ this.adapter = createAdapter(conf, omHost, omPort, isolatedClassloader);
+
+ try {
+ this.userName =
+ UserGroupInformation.getCurrentUser().getShortUserName();
+ } catch (IOException e) {
+ this.userName = OZONE_DEFAULT_USER;
+ }
+ this.workingDir = new Path(OZONE_USER_DIR, this.userName)
+ .makeQualified(this.uri, this.workingDir);
+ } catch (URISyntaxException ue) {
+ final String msg = "Invalid Ozone endpoint " + name;
+ LOG.error(msg, ue);
+ throw new IOException(msg, ue);
+ }
+ }
+
+ protected RootedOzoneClientAdapter createAdapter(Configuration conf,
+ String omHost, int omPort, boolean isolatedClassloader)
+ throws IOException {
+
+ if (isolatedClassloader) {
+ // TODO: Check how this code path need to be changed, for legacy Hadoop?
+ return RootedOzoneClientAdapterFactory.createAdapter();
+ } else {
+ return new BasicRootedOzoneClientAdapterImpl(omHost, omPort, conf);
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ try {
+ adapter.close();
+ } finally {
+ super.close();
+ }
+ }
+
+ @Override
+ public URI getUri() {
+ return uri;
+ }
+
+ @Override
+ public String getScheme() {
+ return OZONE_OFS_URI_SCHEME;
+ }
+
+ @Override
+ public FSDataInputStream open(Path path, int bufferSize) throws IOException {
+ incrementCounter(Statistic.INVOCATION_OPEN);
+ statistics.incrementReadOps(1);
+ LOG.trace("open() path: {}", path);
+ final String key = pathToKey(path);
+ return new FSDataInputStream(
+ new OzoneFSInputStream(adapter.readFile(key), statistics));
+ }
+
+ protected void incrementCounter(Statistic statistic) {
+ //don't do anything in this default implementation.
+ }
+
+ @Override
+ public FSDataOutputStream create(Path f, FsPermission permission,
+ boolean overwrite, int bufferSize,
+ short replication, long blockSize,
+ Progressable progress) throws IOException {
+ LOG.trace("create() path:{}", f);
+ incrementCounter(Statistic.INVOCATION_CREATE);
+ statistics.incrementWriteOps(1);
+ final String key = pathToKey(f);
+ return createOutputStream(key, overwrite, true);
+ }
+
+ @Override
+ public FSDataOutputStream createNonRecursive(Path path,
+ FsPermission permission,
+ EnumSet<CreateFlag> flags,
+ int bufferSize,
+ short replication,
+ long blockSize,
+ Progressable progress) throws IOException {
+ incrementCounter(Statistic.INVOCATION_CREATE_NON_RECURSIVE);
+ statistics.incrementWriteOps(1);
+ final String key = pathToKey(path);
+ return createOutputStream(key, flags.contains(CreateFlag.OVERWRITE), false);
+ }
+
+ private FSDataOutputStream createOutputStream(String key, boolean overwrite,
+ boolean recursive) throws IOException {
+ return new FSDataOutputStream(adapter.createFile(key, overwrite, recursive),
+ statistics);
+ }
+
+ @Override
+ public FSDataOutputStream append(Path f, int bufferSize,
+ Progressable progress) throws IOException {
+ throw new UnsupportedOperationException("append() Not implemented by the "
+ + getClass().getSimpleName() + " FileSystem implementation");
+ }
+
+ private class RenameIterator extends OzoneListingIterator {
+ private final String srcPath;
+ private final String dstPath;
+
+ RenameIterator(Path srcPath, Path dstPath)
+ throws IOException {
+ super(srcPath);
+ this.srcPath = pathToKey(srcPath);
+ this.dstPath = pathToKey(dstPath);
+ LOG.trace("rename from:{} to:{}", this.srcPath, this.dstPath);
+ }
+
+ @Override
+ boolean processKeyPath(String keyPath) throws IOException {
+ String newPath = dstPath.concat(keyPath.substring(srcPath.length()));
+ adapter.renamePath(keyPath, newPath);
+ return true;
+ }
+ }
+
+ /**
+ * Check whether the source and destination path are valid and then perform
+ * rename from source path to destination path.
+ * <p>
+ * The rename operation is performed by renaming the keys with src as prefix.
+ * For such keys the prefix is changed from src to dst.
+ *
+ * @param src source path for rename
+ * @param dst destination path for rename
+ * @return true if rename operation succeeded or
+ * if the src and dst have the same path and are of the same type
+ * @throws IOException on I/O errors or if the src/dst paths are invalid.
+ */
+ @Override
+ public boolean rename(Path src, Path dst) throws IOException {
+ incrementCounter(Statistic.INVOCATION_RENAME);
+ statistics.incrementWriteOps(1);
+ if (src.equals(dst)) {
+ return true;
+ }
+
+ LOG.trace("rename() from: {} to: {}", src, dst);
+ if (src.isRoot()) {
+ // Cannot rename root of file system
+ LOG.trace("Cannot rename the root of a filesystem");
+ return false;
+ }
+
+ // src and dst should be in the same bucket
+ OFSPath ofsSrc = new OFSPath(src);
+ OFSPath ofsDst = new OFSPath(dst);
+ if (!ofsSrc.isInSameBucketAs(ofsDst)) {
+ throw new IOException("Cannot rename a key to a different bucket");
+ }
+
+ // Cannot rename a directory to its own subdirectory
+ Path dstParent = dst.getParent();
+ while (dstParent != null && !src.equals(dstParent)) {
+ dstParent = dstParent.getParent();
+ }
+ Preconditions.checkArgument(dstParent == null,
+ "Cannot rename a directory to its own subdirectory");
+ // Check if the source exists
+ FileStatus srcStatus;
+ try {
+ srcStatus = getFileStatus(src);
+ } catch (FileNotFoundException fnfe) {
+ // source doesn't exist, return
+ return false;
+ }
+
+ // Check if the destination exists
+ FileStatus dstStatus;
+ try {
+ dstStatus = getFileStatus(dst);
+ } catch (FileNotFoundException fnde) {
+ dstStatus = null;
+ }
+
+ if (dstStatus == null) {
+ // If dst doesn't exist, check whether dst parent dir exists or not
+ // if the parent exists, the source can still be renamed to dst path
+ dstStatus = getFileStatus(dst.getParent());
+ if (!dstStatus.isDirectory()) {
+ throw new IOException(String.format(
+ "Failed to rename %s to %s, %s is a file", src, dst,
+ dst.getParent()));
+ }
+ } else {
+ // if dst exists and source and destination are same,
+ // check both the src and dst are of same type
+ if (srcStatus.getPath().equals(dstStatus.getPath())) {
+ return !srcStatus.isDirectory();
+ } else if (dstStatus.isDirectory()) {
+ // If dst is a directory, rename source as subpath of it.
+ // for example rename /source to /dst will lead to /dst/source
+ dst = new Path(dst, src.getName());
+ FileStatus[] statuses;
+ try {
+ statuses = listStatus(dst);
+ } catch (FileNotFoundException fnde) {
+ statuses = null;
+ }
+
+ if (statuses != null && statuses.length > 0) {
+ // If dst exists and not a directory not empty
+ throw new FileAlreadyExistsException(String.format(
+ "Failed to rename %s to %s, file already exists or not empty!",
+ src, dst));
+ }
+ } else {
+ // If dst is not a directory
+ throw new FileAlreadyExistsException(String.format(
+ "Failed to rename %s to %s, file already exists!", src, dst));
+ }
+ }
+
+ if (srcStatus.isDirectory()) {
+ if (dst.toString().startsWith(src.toString() + OZONE_URI_DELIMITER)) {
+ LOG.trace("Cannot rename a directory to a subdirectory of self");
+ return false;
+ }
+ }
+ RenameIterator iterator = new RenameIterator(src, dst);
+ boolean result = iterator.iterate();
+ if (result) {
+ createFakeParentDirectory(src);
+ }
+ return result;
+ }
+
+ private class DeleteIterator extends OzoneListingIterator {
+ private boolean recursive;
+
+ DeleteIterator(Path f, boolean recursive)
+ throws IOException {
+ super(f);
+ this.recursive = recursive;
+ if (getStatus().isDirectory()
+ && !this.recursive
+ && listStatus(f).length != 0) {
+ throw new PathIsNotEmptyDirectoryException(f.toString());
+ }
+ }
+
+ @Override
+ boolean processKeyPath(String keyPath) throws IOException {
+ if (keyPath.equals("")) {
+ LOG.trace("Skipping deleting root directory");
+ return true;
+ } else {
+ LOG.trace("deleting key path:" + keyPath);
+ boolean succeed = adapter.deleteObject(keyPath);
+ // if recursive delete is requested ignore the return value of
+ // deleteObject and issue deletes for other keys.
+ return recursive || succeed;
+ }
+ }
+ }
+
+ /**
+ * Deletes the children of the input dir path by iterating though the
+ * DeleteIterator.
+ *
+ * @param f directory path to be deleted
+ * @return true if successfully deletes all required keys, false otherwise
+ * @throws IOException
+ */
+ private boolean innerDelete(Path f, boolean recursive) throws IOException {
+ LOG.trace("delete() path:{} recursive:{}", f, recursive);
+ try {
+ DeleteIterator iterator = new DeleteIterator(f, recursive);
+ return iterator.iterate();
+ } catch (FileNotFoundException e) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Couldn't delete {} - does not exist", f);
+ }
+ return false;
+ }
+ }
+
+ @Override
+ public boolean delete(Path f, boolean recursive) throws IOException {
+ incrementCounter(Statistic.INVOCATION_DELETE);
+ statistics.incrementWriteOps(1);
+ LOG.debug("Delete path {} - recursive {}", f, recursive);
+ FileStatus status;
+ try {
+ status = getFileStatus(f);
+ } catch (FileNotFoundException ex) {
+ LOG.warn("delete: Path does not exist: {}", f);
+ return false;
+ }
+
+ String key = pathToKey(f);
+ boolean result;
+
+ if (status.isDirectory()) {
+ LOG.debug("delete: Path is a directory: {}", f);
+ key = addTrailingSlashIfNeeded(key);
+
+ if (key.equals("/")) {
+ LOG.warn("Cannot delete root directory.");
+ return false;
+ }
+
+ result = innerDelete(f, recursive);
+ } else {
+ LOG.debug("delete: Path is a file: {}", f);
+ result = adapter.deleteObject(key);
+ }
+
+ if (result) {
+ // If this delete operation removes all files/directories from the
+ // parent direcotry, then an empty parent directory must be created.
+ createFakeParentDirectory(f);
+ }
+
+ return result;
+ }
+
+ /**
+ * Create a fake parent directory key if it does not already exist and no
+ * other child of this parent directory exists.
+ *
+ * @param f path to the fake parent directory
+ * @throws IOException
+ */
+ private void createFakeParentDirectory(Path f) throws IOException {
+ Path parent = f.getParent();
+ if (parent != null && !parent.isRoot()) {
+ createFakeDirectoryIfNecessary(parent);
+ }
+ }
+
+ /**
+ * Create a fake directory key if it does not already exist.
+ *
+ * @param f path to the fake directory
+ * @throws IOException
+ */
+ private void createFakeDirectoryIfNecessary(Path f) throws IOException {
+ String key = pathToKey(f);
+ if (!key.isEmpty() && !o3Exists(f)) {
+ LOG.debug("Creating new fake directory at {}", f);
+ String dirKey = addTrailingSlashIfNeeded(key);
+ adapter.createDirectory(dirKey);
+ }
+ }
+
+ /**
+ * Check if a file or directory exists corresponding to given path.
+ *
+ * @param f path to file/directory.
+ * @return true if it exists, false otherwise.
+ * @throws IOException
+ */
+ private boolean o3Exists(final Path f) throws IOException {
+ Path path = makeQualified(f);
+ try {
+ getFileStatus(path);
+ return true;
+ } catch (FileNotFoundException ex) {
+ return false;
+ }
+ }
+
+ @Override
+ public FileStatus[] listStatus(Path f) throws IOException {
+ incrementCounter(Statistic.INVOCATION_LIST_STATUS);
+ statistics.incrementReadOps(1);
+ LOG.trace("listStatus() path:{}", f);
+ int numEntries = LISTING_PAGE_SIZE;
+ LinkedList<FileStatus> statuses = new LinkedList<>();
+ List<FileStatus> tmpStatusList;
+ String startPath = "";
+
+ do {
+ tmpStatusList =
+ adapter.listStatus(pathToKey(f), false, startPath,
+ numEntries, uri, workingDir, getUsername())
+ .stream()
+ .map(this::convertFileStatus)
+ .collect(Collectors.toList());
+
+ if (!tmpStatusList.isEmpty()) {
+ if (startPath.isEmpty()) {
+ statuses.addAll(tmpStatusList);
+ } else {
+ statuses.addAll(tmpStatusList.subList(1, tmpStatusList.size()));
+ }
+ startPath = pathToKey(statuses.getLast().getPath());
+ }
+ // listStatus returns entries numEntries in size if available.
+ // Any lesser number of entries indicate that the required entries have
+ // exhausted.
+ } while (tmpStatusList.size() == numEntries);
+
+ return statuses.toArray(new FileStatus[0]);
+ }
+
+ @Override
+ public void setWorkingDirectory(Path newDir) {
+ workingDir = newDir;
+ }
+
+ @Override
+ public Path getWorkingDirectory() {
+ return workingDir;
+ }
+
+ @Override
+ public Token<?> getDelegationToken(String renewer) throws IOException {
+ return adapter.getDelegationToken(renewer);
+ }
+
+ /**
+ * Get a canonical service name for this file system. If the URI is logical,
+ * the hostname part of the URI will be returned.
+ *
+ * @return a service string that uniquely identifies this file system.
+ */
+ @Override
+ public String getCanonicalServiceName() {
+ return adapter.getCanonicalServiceName();
+ }
+
+ /**
+ * Get the username of the FS.
+ *
+ * @return the short name of the user who instantiated the FS
+ */
+ public String getUsername() {
+ return userName;
+ }
+
+ /**
+ * Creates a directory. Directory is represented using a key with no value.
+ *
+ * @param path directory path to be created
+ * @return true if directory exists or created successfully.
+ * @throws IOException
+ */
+ private boolean mkdir(Path path) throws IOException {
+ return adapter.createDirectory(pathToKey(path));
+ }
+
+ @Override
+ public boolean mkdirs(Path f, FsPermission permission) throws IOException {
+ LOG.trace("mkdir() path:{} ", f);
+ String key = pathToKey(f);
+ if (isEmpty(key)) {
+ return false;
+ }
+ return mkdir(f);
+ }
+
+ @Override
+ public FileStatus getFileStatus(Path f) throws IOException {
+ incrementCounter(Statistic.INVOCATION_GET_FILE_STATUS);
+ statistics.incrementReadOps(1);
+ LOG.trace("getFileStatus() path:{}", f);
+ Path qualifiedPath = f.makeQualified(uri, workingDir);
+ String key = pathToKey(qualifiedPath);
+ FileStatus fileStatus = null;
+ try {
+ fileStatus = convertFileStatus(
+ adapter.getFileStatus(key, uri, qualifiedPath, getUsername()));
+ } catch (OMException ex) {
+ if (ex.getResult().equals(OMException.ResultCodes.KEY_NOT_FOUND)) {
+ throw new FileNotFoundException("File not found. path:" + f);
+ }
+ }
+ return fileStatus;
+ }
+
+ @Override
+ public BlockLocation[] getFileBlockLocations(FileStatus fileStatus,
+ long start, long len)
+ throws IOException {
+ if (fileStatus instanceof LocatedFileStatus) {
+ return ((LocatedFileStatus) fileStatus).getBlockLocations();
+ } else {
+ return super.getFileBlockLocations(fileStatus, start, len);
+ }
+ }
+
+ /**
+ * Turn a path (relative or otherwise) into an Ozone key.
+ *
+ * @param path the path of the file.
+ * @return the key of the object that represents the file.
+ */
+ public String pathToKey(Path path) {
+ Objects.requireNonNull(path, "Path can't be null!");
Review comment:
This is a design decision which we can leave out in this Jira and probably add a TODO to keep track, if you agree.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] smengcl commented on a change in pull request #415:
HDDS-2840. Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
smengcl commented on a change in pull request #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#discussion_r372094342
##########
File path: hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OFSPath.java
##########
@@ -0,0 +1,143 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.ozone;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.yetus.audience.InterfaceAudience;
+import org.apache.yetus.audience.InterfaceStability;
+import java.util.StringTokenizer;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
+
+/**
+ * Utility class for Rooted Ozone Filesystem (OFS) path processing.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+class OFSPath {
+ /**
+ * Here is a table illustrating what each name variable is given an input path
+ * Assuming /tmp is mounted to /tempVol/tempBucket
Review comment:
This is undecided. We could give admin a CLI command to do the initialization. But we might want to leave this for jira HDDS-2929.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] smengcl commented on a change in pull request #415:
HDDS-2840. Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
smengcl commented on a change in pull request #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#discussion_r372062918
##########
File path: hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
##########
@@ -0,0 +1,694 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.ozone;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.key.KeyProvider;
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdds.client.ReplicationFactor;
+import org.apache.hadoop.hdds.client.ReplicationType;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ozone.OmUtils;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.client.OzoneKey;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
+import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
+import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenRenewer;
+
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .BUCKET_ALREADY_EXISTS;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .VOLUME_ALREADY_EXISTS;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .VOLUME_NOT_FOUND;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .BUCKET_NOT_FOUND;
+
+/**
+ * Basic Implementation of the OzoneFileSystem calls.
+ * <p>
+ * This is the minimal version which doesn't include any statistics.
+ * <p>
+ * For full featured version use OzoneClientAdapterImpl.
+ */
+public class BasicRootedOzoneClientAdapterImpl
+ implements RootedOzoneClientAdapter {
+
+ static final Logger LOG =
+ LoggerFactory.getLogger(BasicRootedOzoneClientAdapterImpl.class);
+
+ private OzoneClient ozoneClient;
+ private ClientProtocol proxy;
+ private ObjectStore objectStore;
+ private ReplicationType replicationType;
+ private ReplicationFactor replicationFactor;
+ private boolean securityEnabled;
+ private int configuredDnPort;
+
+ /**
+ * Create new OzoneClientAdapter implementation.
+ *
+ * @throws IOException In case of a problem.
+ */
+ public BasicRootedOzoneClientAdapterImpl() throws IOException {
+ this(createConf());
+ }
+
+ private static OzoneConfiguration createConf() {
+ ClassLoader contextClassLoader =
+ Thread.currentThread().getContextClassLoader();
+ Thread.currentThread().setContextClassLoader(null);
+ try {
+ return new OzoneConfiguration();
+ } finally {
+ Thread.currentThread().setContextClassLoader(contextClassLoader);
+ }
+ }
+
+ public BasicRootedOzoneClientAdapterImpl(OzoneConfiguration conf)
+ throws IOException {
+ this(null, -1, conf);
+ }
+
+ public BasicRootedOzoneClientAdapterImpl(String omHost, int omPort,
+ Configuration hadoopConf) throws IOException {
+
+ ClassLoader contextClassLoader =
+ Thread.currentThread().getContextClassLoader();
+ Thread.currentThread().setContextClassLoader(null);
+
+ try {
+ OzoneConfiguration conf = OzoneConfiguration.of(hadoopConf);
+
+ if (omHost == null && OmUtils.isServiceIdsDefined(conf)) {
+ // When the host name or service id isn't given
+ // but ozone.om.service.ids is defined, declare failure.
+
+ // This is a safety precaution that prevents the client from
+ // accidentally failing over to an unintended OM.
+ throw new IllegalArgumentException("Service ID or host name must not"
+ + " be omitted when ozone.om.service.ids is defined.");
+ }
+
+ if (omPort != -1) {
+ // When the port number is specified, perform the following check
+ if (OmUtils.isOmHAServiceId(conf, omHost)) {
+ // If omHost is a service id, it shouldn't use a port
+ throw new IllegalArgumentException("Port " + omPort +
+ " specified in URI but host '" + omHost + "' is a "
+ + "logical (HA) OzoneManager and does not use port information.");
+ }
+ } else {
+ // When port number is not specified, read it from config
+ omPort = OmUtils.getOmRpcPort(conf);
+ }
+
+ SecurityConfig secConfig = new SecurityConfig(conf);
+
+ if (secConfig.isSecurityEnabled()) {
+ this.securityEnabled = true;
+ }
+
+ String replicationTypeConf =
+ conf.get(OzoneConfigKeys.OZONE_REPLICATION_TYPE,
+ OzoneConfigKeys.OZONE_REPLICATION_TYPE_DEFAULT);
+
+ int replicationCountConf = conf.getInt(OzoneConfigKeys.OZONE_REPLICATION,
+ OzoneConfigKeys.OZONE_REPLICATION_DEFAULT);
+
+ if (OmUtils.isOmHAServiceId(conf, omHost)) {
+ // omHost is listed as one of the service ids in the config,
+ // thus we should treat omHost as omServiceId
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(omHost, conf);
+ } else if (StringUtils.isNotEmpty(omHost) && omPort != -1) {
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(omHost, omPort, conf);
+ } else {
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(conf);
+ }
+ objectStore = ozoneClient.getObjectStore();
+ proxy = objectStore.getClientProxy();
+ this.replicationType = ReplicationType.valueOf(replicationTypeConf);
+ this.replicationFactor = ReplicationFactor.valueOf(replicationCountConf);
+ this.configuredDnPort = conf.getInt(
+ OzoneConfigKeys.DFS_CONTAINER_IPC_PORT,
+ OzoneConfigKeys.DFS_CONTAINER_IPC_PORT_DEFAULT);
+ } finally {
+ Thread.currentThread().setContextClassLoader(contextClassLoader);
+ }
+ }
+
+ OzoneBucket getBucket(OFSPath ofsPath, boolean createIfNotExist)
+ throws IOException {
+
+ return getBucket(ofsPath.getVolumeName(), ofsPath.getBucketName(),
+ createIfNotExist);
+ }
+
+ /**
+ * Get OzoneBucket object to operate in.
+ * Optionally create volume and bucket if not found.
+ *
+ * @param createIfNotExist Set this to true if the caller is a write operation
+ * in order to create the volume and bucket.
+ * @throws IOException Exceptions other than OMException with result code
+ * VOLUME_NOT_FOUND or BUCKET_NOT_FOUND.
+ */
+ private OzoneBucket getBucket(String volumeStr, String bucketStr,
+ boolean createIfNotExist) throws IOException {
+
+ OzoneBucket bucket;
+ try {
+ bucket = proxy.getBucketDetails(volumeStr, bucketStr);
+ } catch (OMException ex) {
+ if (createIfNotExist) {
+ // Note: getBucketDetails always throws BUCKET_NOT_FOUND, even if
+ // the volume doesn't exist.
+ if (ex.getResult().equals(BUCKET_NOT_FOUND)) {
+ OzoneVolume volume;
+ try {
+ volume = proxy.getVolumeDetails(volumeStr);
Review comment:
@xiaoyuyao Actually this part was coded previously that it checks if the volume exists first. Then @bharatviswa504 argued that this way we would always have at least two round trips for each `getBucket()` (first `getVolumeDetails`, then `getBucketDetails`) regardless of whether the bucket already exists. This change is made in d2a01d19f0cfa2cdce5e252be2dd33fcb0d4fb6a.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] smengcl commented on a change in pull request #415:
HDDS-2840. Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
smengcl commented on a change in pull request #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#discussion_r371274576
##########
File path: hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystemWithMocks.java
##########
@@ -0,0 +1,152 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.ozone;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.OmUtils;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PowerMockIgnore;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+
+import java.net.URI;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * Ozone File system tests that are light weight and use mocks.
+ */
+@RunWith(PowerMockRunner.class)
+@PrepareForTest({ OzoneClientFactory.class, UserGroupInformation.class })
+@PowerMockIgnore("javax.management.*")
+public class TestRootedOzoneFileSystemWithMocks {
+
+ @Test
+ public void testFSUriWithHostPortOverrides() throws Exception {
+ Configuration conf = new OzoneConfiguration();
+ OzoneClient ozoneClient = mock(OzoneClient.class);
+ ObjectStore objectStore = mock(ObjectStore.class);
+ OzoneVolume volume = mock(OzoneVolume.class);
+ OzoneBucket bucket = mock(OzoneBucket.class);
+
+ when(ozoneClient.getObjectStore()).thenReturn(objectStore);
+ when(objectStore.getVolume(eq("volume1"))).thenReturn(volume);
+ when(volume.getBucket("bucket1")).thenReturn(bucket);
+
+ PowerMockito.mockStatic(OzoneClientFactory.class);
+ PowerMockito.when(OzoneClientFactory.getRpcClient(eq("local.host"),
+ eq(5899), eq(conf))).thenReturn(ozoneClient);
+
+ UserGroupInformation ugi = mock(UserGroupInformation.class);
+ PowerMockito.mockStatic(UserGroupInformation.class);
+ PowerMockito.when(UserGroupInformation.getCurrentUser()).thenReturn(ugi);
+ when(ugi.getShortUserName()).thenReturn("user1");
+
+ // Note: FileSystem#loadFileSystems doesn't load OFS class because
+ // META-INF still points to org.apache.hadoop.fs.ozone.OzoneFileSystem
+ conf.set("fs.ofs.impl", "org.apache.hadoop.fs.ozone.RootedOzoneFileSystem");
+ URI uri = new URI("ofs://local.host:5899/volume1/bucket1");
+
+ FileSystem fileSystem = FileSystem.get(uri, conf);
+ RootedOzoneFileSystem ofs = (RootedOzoneFileSystem) fileSystem;
+
+ assertEquals(ofs.getUri().getAuthority(), "local.host:5899");
+ PowerMockito.verifyStatic();
+ OzoneClientFactory.getRpcClient("local.host", 5899, conf);
+ }
+
+ @Test
+ public void testFSUriWithHostPortUnspecified() throws Exception {
+ Configuration conf = new OzoneConfiguration();
+ final int omPort = OmUtils.getOmRpcPort(conf);
+
+ OzoneClient ozoneClient = mock(OzoneClient.class);
+ ObjectStore objectStore = mock(ObjectStore.class);
+ OzoneVolume volume = mock(OzoneVolume.class);
+ OzoneBucket bucket = mock(OzoneBucket.class);
+
+ when(ozoneClient.getObjectStore()).thenReturn(objectStore);
+ when(objectStore.getVolume(eq("volume1"))).thenReturn(volume);
+ when(volume.getBucket("bucket1")).thenReturn(bucket);
+
+ PowerMockito.mockStatic(OzoneClientFactory.class);
+ PowerMockito.when(OzoneClientFactory.getRpcClient(eq("local.host"),
+ eq(omPort), eq(conf))).thenReturn(ozoneClient);
+
+ UserGroupInformation ugi = mock(UserGroupInformation.class);
+ PowerMockito.mockStatic(UserGroupInformation.class);
+ PowerMockito.when(UserGroupInformation.getCurrentUser()).thenReturn(ugi);
+ when(ugi.getShortUserName()).thenReturn("user1");
+
+ URI uri = new URI("o3fs://bucket1.volume1.local.host");
+
Review comment:
Done in 5a20f9da30a25fd40fa1a73e6b0e742f40647a94.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] smengcl commented on issue #415: HDDS-2840.
Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
smengcl commented on issue #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#issuecomment-579899200
Should be almost good to go. We can do one more round of checking using diff https://github.com/smengcl/hadoop-ozone/compare/0bee28acadfb2c358c6f008173e7eca6ed7fa23f...smengcl:HDDS-2840
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] hanishakoneru commented on issue #415: HDDS-2840.
Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
hanishakoneru commented on issue #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#issuecomment-579026020
Thanks @smengcl. We can address the refactoring in subsequent Jiras.
I am +1 for merging this PR into branch HDDS-2665-ofs.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] smengcl commented on a change in pull request #415:
HDDS-2840. Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
smengcl commented on a change in pull request #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#discussion_r371245197
##########
File path: hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
##########
@@ -0,0 +1,805 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.ozone;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.CreateFlag;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.Progressable;
+import org.apache.http.client.utils.URIBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.EnumSet;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+import static org.apache.hadoop.fs.ozone.Constants.LISTING_PAGE_SIZE;
+import static org.apache.hadoop.fs.ozone.Constants.OZONE_DEFAULT_USER;
+import static org.apache.hadoop.fs.ozone.Constants.OZONE_USER_DIR;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_OFS_URI_SCHEME;
+
+/**
+ * The minimal Ozone Filesystem implementation.
+ * <p>
+ * This is a basic version which doesn't extend
+ * KeyProviderTokenIssuer and doesn't include statistics. It can be used
+ * from older hadoop version. For newer hadoop version use the full featured
+ * OFileSystem.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class BasicRootedOzoneFileSystem extends FileSystem {
+ static final Logger LOG =
+ LoggerFactory.getLogger(BasicRootedOzoneFileSystem.class);
+
+ /**
+ * The Ozone client for connecting to Ozone server.
+ */
+
+ private URI uri;
+ private String userName;
+ private Path workingDir;
+// private String gOmHost;
+// private int gOmPort;
+// private Configuration gConf;
+// private boolean gIsolatedClassloader;
+
+ private RootedOzoneClientAdapter adapter;
+// private String adapterPath;
+
+ private static final String URI_EXCEPTION_TEXT =
+ "URL should be one of the following formats: " +
+ "ofs://om-service-id/ OR " +
+ "ofs://om-host.example.com/ OR " +
+ "ofs://om-host.example.com:5678/";
+
+ @Override
+ public void initialize(URI name, Configuration conf) throws IOException {
+ super.initialize(name, conf);
+ setConf(conf);
+ Objects.requireNonNull(name.getScheme(), "No scheme provided in " + name);
+ Preconditions.checkArgument(getScheme().equals(name.getScheme()),
+ "Invalid scheme provided in " + name);
+
+ String authority = name.getAuthority();
+ if (authority == null) {
+ // authority is null when fs.defaultFS is not a qualified ofs URI and
+ // ofs:/// is passed to the client. matcher will NPE if authority is null
+ throw new IllegalArgumentException(URI_EXCEPTION_TEXT);
+ }
+
+ String omHost;
+ int omPort = -1;
+ // Parse hostname and port
+ String[] parts = authority.split(":");
+ if (parts.length > 2) {
+ throw new IllegalArgumentException(URI_EXCEPTION_TEXT);
+ }
+ omHost = parts[0];
+ if (parts.length == 2) {
+ try {
+ omPort = Integer.parseInt(parts[1]);
+ } catch (NumberFormatException e) {
+ throw new IllegalArgumentException(URI_EXCEPTION_TEXT);
+ }
+ }
+
+ try {
+ uri = new URIBuilder().setScheme(OZONE_OFS_URI_SCHEME)
+ .setHost(authority)
+ .build();
+ LOG.trace("Ozone URI for OFS initialization is " + uri);
+
+ //isolated is the default for ozonefs-lib-legacy which includes the
+ // /ozonefs.txt, otherwise the default is false. It could be overridden.
+ boolean defaultValue =
+ BasicRootedOzoneFileSystem.class.getClassLoader()
+ .getResource("ozonefs.txt") != null;
+
+ //Use string here instead of the constant as constant may not be available
+ //on the classpath of a hadoop 2.7
+ boolean isolatedClassloader =
+ conf.getBoolean("ozone.fs.isolated-classloader", defaultValue);
+
+ // adapter should be initialized in operations.
+ this.adapter = createAdapter(conf, omHost, omPort, isolatedClassloader);
+
+ try {
+ this.userName =
+ UserGroupInformation.getCurrentUser().getShortUserName();
+ } catch (IOException e) {
+ this.userName = OZONE_DEFAULT_USER;
+ }
+ this.workingDir = new Path(OZONE_USER_DIR, this.userName)
+ .makeQualified(this.uri, this.workingDir);
+ } catch (URISyntaxException ue) {
+ final String msg = "Invalid Ozone endpoint " + name;
+ LOG.error(msg, ue);
+ throw new IOException(msg, ue);
+ }
+ }
+
+ protected RootedOzoneClientAdapter createAdapter(Configuration conf,
+ String omHost, int omPort, boolean isolatedClassloader)
+ throws IOException {
+
+ if (isolatedClassloader) {
+ // TODO: Check how this code path need to be changed, for legacy Hadoop?
+ return RootedOzoneClientAdapterFactory.createAdapter();
+ } else {
+ return new BasicRootedOzoneClientAdapterImpl(omHost, omPort, conf);
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ try {
+ adapter.close();
+ } finally {
+ super.close();
+ }
+ }
+
+ @Override
+ public URI getUri() {
+ return uri;
+ }
+
+ @Override
+ public String getScheme() {
+ return OZONE_OFS_URI_SCHEME;
+ }
+
+ @Override
+ public FSDataInputStream open(Path path, int bufferSize) throws IOException {
+ incrementCounter(Statistic.INVOCATION_OPEN);
+ statistics.incrementReadOps(1);
+ LOG.trace("open() path: {}", path);
+ final String key = pathToKey(path);
+ return new FSDataInputStream(
+ new OzoneFSInputStream(adapter.readFile(key), statistics));
+ }
+
+ protected void incrementCounter(Statistic statistic) {
+ //don't do anything in this default implementation.
+ }
+
+ @Override
+ public FSDataOutputStream create(Path f, FsPermission permission,
+ boolean overwrite, int bufferSize,
+ short replication, long blockSize,
+ Progressable progress) throws IOException {
+ LOG.trace("create() path:{}", f);
+ incrementCounter(Statistic.INVOCATION_CREATE);
+ statistics.incrementWriteOps(1);
+ final String key = pathToKey(f);
+ return createOutputStream(key, overwrite, true);
+ }
+
+ @Override
+ public FSDataOutputStream createNonRecursive(Path path,
+ FsPermission permission,
+ EnumSet<CreateFlag> flags,
+ int bufferSize,
+ short replication,
+ long blockSize,
+ Progressable progress) throws IOException {
+ incrementCounter(Statistic.INVOCATION_CREATE_NON_RECURSIVE);
+ statistics.incrementWriteOps(1);
+ final String key = pathToKey(path);
+ return createOutputStream(key, flags.contains(CreateFlag.OVERWRITE), false);
+ }
+
+ private FSDataOutputStream createOutputStream(String key, boolean overwrite,
+ boolean recursive) throws IOException {
+ return new FSDataOutputStream(adapter.createFile(key, overwrite, recursive),
+ statistics);
+ }
+
+ @Override
+ public FSDataOutputStream append(Path f, int bufferSize,
+ Progressable progress) throws IOException {
+ throw new UnsupportedOperationException("append() Not implemented by the "
+ + getClass().getSimpleName() + " FileSystem implementation");
+ }
+
+ private class RenameIterator extends OzoneListingIterator {
+ private final String srcPath;
+ private final String dstPath;
+
+ RenameIterator(Path srcPath, Path dstPath)
+ throws IOException {
+ super(srcPath);
+ this.srcPath = pathToKey(srcPath);
+ this.dstPath = pathToKey(dstPath);
+ LOG.trace("rename from:{} to:{}", this.srcPath, this.dstPath);
+ }
+
+ @Override
+ boolean processKeyPath(String keyPath) throws IOException {
+ String newPath = dstPath.concat(keyPath.substring(srcPath.length()));
+ adapter.renamePath(keyPath, newPath);
+ return true;
+ }
+ }
+
+ /**
+ * Check whether the source and destination path are valid and then perform
+ * rename from source path to destination path.
+ * <p>
+ * The rename operation is performed by renaming the keys with src as prefix.
+ * For such keys the prefix is changed from src to dst.
+ *
+ * @param src source path for rename
+ * @param dst destination path for rename
+ * @return true if rename operation succeeded or
+ * if the src and dst have the same path and are of the same type
+ * @throws IOException on I/O errors or if the src/dst paths are invalid.
+ */
+ @Override
+ public boolean rename(Path src, Path dst) throws IOException {
+ incrementCounter(Statistic.INVOCATION_RENAME);
+ statistics.incrementWriteOps(1);
+ if (src.equals(dst)) {
+ return true;
+ }
+
+ LOG.trace("rename() from: {} to: {}", src, dst);
+ if (src.isRoot()) {
+ // Cannot rename root of file system
+ LOG.trace("Cannot rename the root of a filesystem");
+ return false;
+ }
+
+ // src and dst should be in the same bucket
+ OFSPath ofsSrc = new OFSPath(src);
+ OFSPath ofsDst = new OFSPath(dst);
+ if (!ofsSrc.isInSameBucketAs(ofsDst)) {
+ throw new IOException("Cannot rename a key to a different bucket");
+ }
+
+ // Cannot rename a directory to its own subdirectory
+ Path dstParent = dst.getParent();
+ while (dstParent != null && !src.equals(dstParent)) {
+ dstParent = dstParent.getParent();
+ }
+ Preconditions.checkArgument(dstParent == null,
+ "Cannot rename a directory to its own subdirectory");
+ // Check if the source exists
+ FileStatus srcStatus;
+ try {
+ srcStatus = getFileStatus(src);
+ } catch (FileNotFoundException fnfe) {
+ // source doesn't exist, return
+ return false;
+ }
+
+ // Check if the destination exists
+ FileStatus dstStatus;
+ try {
+ dstStatus = getFileStatus(dst);
+ } catch (FileNotFoundException fnde) {
+ dstStatus = null;
+ }
+
+ if (dstStatus == null) {
+ // If dst doesn't exist, check whether dst parent dir exists or not
+ // if the parent exists, the source can still be renamed to dst path
+ dstStatus = getFileStatus(dst.getParent());
+ if (!dstStatus.isDirectory()) {
+ throw new IOException(String.format(
+ "Failed to rename %s to %s, %s is a file", src, dst,
+ dst.getParent()));
+ }
+ } else {
+ // if dst exists and source and destination are same,
+ // check both the src and dst are of same type
+ if (srcStatus.getPath().equals(dstStatus.getPath())) {
+ return !srcStatus.isDirectory();
+ } else if (dstStatus.isDirectory()) {
+ // If dst is a directory, rename source as subpath of it.
+ // for example rename /source to /dst will lead to /dst/source
+ dst = new Path(dst, src.getName());
+ FileStatus[] statuses;
+ try {
+ statuses = listStatus(dst);
+ } catch (FileNotFoundException fnde) {
+ statuses = null;
+ }
+
+ if (statuses != null && statuses.length > 0) {
+ // If dst exists and not a directory not empty
+ throw new FileAlreadyExistsException(String.format(
+ "Failed to rename %s to %s, file already exists or not empty!",
+ src, dst));
+ }
+ } else {
+ // If dst is not a directory
+ throw new FileAlreadyExistsException(String.format(
+ "Failed to rename %s to %s, file already exists!", src, dst));
+ }
+ }
+
+ if (srcStatus.isDirectory()) {
+ if (dst.toString().startsWith(src.toString() + OZONE_URI_DELIMITER)) {
+ LOG.trace("Cannot rename a directory to a subdirectory of self");
+ return false;
+ }
+ }
+ RenameIterator iterator = new RenameIterator(src, dst);
+ boolean result = iterator.iterate();
+ if (result) {
+ createFakeParentDirectory(src);
+ }
+ return result;
+ }
+
+ private class DeleteIterator extends OzoneListingIterator {
+ private boolean recursive;
+
+ DeleteIterator(Path f, boolean recursive)
+ throws IOException {
+ super(f);
+ this.recursive = recursive;
+ if (getStatus().isDirectory()
+ && !this.recursive
+ && listStatus(f).length != 0) {
+ throw new PathIsNotEmptyDirectoryException(f.toString());
+ }
+ }
+
+ @Override
+ boolean processKeyPath(String keyPath) throws IOException {
+ if (keyPath.equals("")) {
+ LOG.trace("Skipping deleting root directory");
+ return true;
+ } else {
+ LOG.trace("deleting key path:" + keyPath);
+ boolean succeed = adapter.deleteObject(keyPath);
Review comment:
Attempted to address it in commit dfc50cc8b0f9d35aa5d9dfcea3e4beedb4930341. See my comment above.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] hanishakoneru commented on a change in pull request
#415: HDDS-2840. Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
hanishakoneru commented on a change in pull request #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#discussion_r369328194
##########
File path: hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
##########
@@ -0,0 +1,805 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.ozone;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.CreateFlag;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.Progressable;
+import org.apache.http.client.utils.URIBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.EnumSet;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+import static org.apache.hadoop.fs.ozone.Constants.LISTING_PAGE_SIZE;
+import static org.apache.hadoop.fs.ozone.Constants.OZONE_DEFAULT_USER;
+import static org.apache.hadoop.fs.ozone.Constants.OZONE_USER_DIR;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_OFS_URI_SCHEME;
+
+/**
+ * The minimal Ozone Filesystem implementation.
+ * <p>
+ * This is a basic version which doesn't extend
+ * KeyProviderTokenIssuer and doesn't include statistics. It can be used
+ * from older hadoop version. For newer hadoop version use the full featured
+ * OFileSystem.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class BasicRootedOzoneFileSystem extends FileSystem {
+ static final Logger LOG =
+ LoggerFactory.getLogger(BasicRootedOzoneFileSystem.class);
+
+ /**
+ * The Ozone client for connecting to Ozone server.
+ */
+
+ private URI uri;
+ private String userName;
+ private Path workingDir;
+// private String gOmHost;
+// private int gOmPort;
+// private Configuration gConf;
+// private boolean gIsolatedClassloader;
Review comment:
Commented variables
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org
[GitHub] [hadoop-ozone] xiaoyuyao commented on a change in pull request
#415: HDDS-2840. Implement ofs://: mkdir
Posted by GitBox <gi...@apache.org>.
xiaoyuyao commented on a change in pull request #415: HDDS-2840. Implement ofs://: mkdir
URL: https://github.com/apache/hadoop-ozone/pull/415#discussion_r372026827
##########
File path: hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
##########
@@ -0,0 +1,694 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.ozone;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.key.KeyProvider;
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdds.client.ReplicationFactor;
+import org.apache.hadoop.hdds.client.ReplicationType;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ozone.OmUtils;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.client.OzoneKey;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
+import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
+import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenRenewer;
+
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .BUCKET_ALREADY_EXISTS;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .VOLUME_ALREADY_EXISTS;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .VOLUME_NOT_FOUND;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
+ .BUCKET_NOT_FOUND;
+
+/**
+ * Basic Implementation of the OzoneFileSystem calls.
+ * <p>
+ * This is the minimal version which doesn't include any statistics.
+ * <p>
+ * For full featured version use OzoneClientAdapterImpl.
+ */
+public class BasicRootedOzoneClientAdapterImpl
+ implements RootedOzoneClientAdapter {
+
+ static final Logger LOG =
+ LoggerFactory.getLogger(BasicRootedOzoneClientAdapterImpl.class);
+
+ private OzoneClient ozoneClient;
+ private ClientProtocol proxy;
+ private ObjectStore objectStore;
+ private ReplicationType replicationType;
+ private ReplicationFactor replicationFactor;
+ private boolean securityEnabled;
+ private int configuredDnPort;
+
+ /**
+ * Create new OzoneClientAdapter implementation.
+ *
+ * @throws IOException In case of a problem.
+ */
+ public BasicRootedOzoneClientAdapterImpl() throws IOException {
+ this(createConf());
+ }
+
+ private static OzoneConfiguration createConf() {
+ ClassLoader contextClassLoader =
+ Thread.currentThread().getContextClassLoader();
+ Thread.currentThread().setContextClassLoader(null);
+ try {
+ return new OzoneConfiguration();
+ } finally {
+ Thread.currentThread().setContextClassLoader(contextClassLoader);
+ }
+ }
+
+ public BasicRootedOzoneClientAdapterImpl(OzoneConfiguration conf)
+ throws IOException {
+ this(null, -1, conf);
+ }
+
+ public BasicRootedOzoneClientAdapterImpl(String omHost, int omPort,
+ Configuration hadoopConf) throws IOException {
+
+ ClassLoader contextClassLoader =
+ Thread.currentThread().getContextClassLoader();
+ Thread.currentThread().setContextClassLoader(null);
+
+ try {
+ OzoneConfiguration conf = OzoneConfiguration.of(hadoopConf);
+
+ if (omHost == null && OmUtils.isServiceIdsDefined(conf)) {
+ // When the host name or service id isn't given
+ // but ozone.om.service.ids is defined, declare failure.
+
+ // This is a safety precaution that prevents the client from
+ // accidentally failing over to an unintended OM.
+ throw new IllegalArgumentException("Service ID or host name must not"
+ + " be omitted when ozone.om.service.ids is defined.");
+ }
+
+ if (omPort != -1) {
+ // When the port number is specified, perform the following check
+ if (OmUtils.isOmHAServiceId(conf, omHost)) {
+ // If omHost is a service id, it shouldn't use a port
+ throw new IllegalArgumentException("Port " + omPort +
+ " specified in URI but host '" + omHost + "' is a "
+ + "logical (HA) OzoneManager and does not use port information.");
+ }
+ } else {
+ // When port number is not specified, read it from config
+ omPort = OmUtils.getOmRpcPort(conf);
+ }
+
+ SecurityConfig secConfig = new SecurityConfig(conf);
+
+ if (secConfig.isSecurityEnabled()) {
+ this.securityEnabled = true;
+ }
+
+ String replicationTypeConf =
+ conf.get(OzoneConfigKeys.OZONE_REPLICATION_TYPE,
+ OzoneConfigKeys.OZONE_REPLICATION_TYPE_DEFAULT);
+
+ int replicationCountConf = conf.getInt(OzoneConfigKeys.OZONE_REPLICATION,
+ OzoneConfigKeys.OZONE_REPLICATION_DEFAULT);
+
+ if (OmUtils.isOmHAServiceId(conf, omHost)) {
+ // omHost is listed as one of the service ids in the config,
+ // thus we should treat omHost as omServiceId
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(omHost, conf);
+ } else if (StringUtils.isNotEmpty(omHost) && omPort != -1) {
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(omHost, omPort, conf);
+ } else {
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(conf);
+ }
+ objectStore = ozoneClient.getObjectStore();
+ proxy = objectStore.getClientProxy();
+ this.replicationType = ReplicationType.valueOf(replicationTypeConf);
+ this.replicationFactor = ReplicationFactor.valueOf(replicationCountConf);
+ this.configuredDnPort = conf.getInt(
+ OzoneConfigKeys.DFS_CONTAINER_IPC_PORT,
+ OzoneConfigKeys.DFS_CONTAINER_IPC_PORT_DEFAULT);
+ } finally {
+ Thread.currentThread().setContextClassLoader(contextClassLoader);
+ }
+ }
+
+ OzoneBucket getBucket(OFSPath ofsPath, boolean createIfNotExist)
+ throws IOException {
+
+ return getBucket(ofsPath.getVolumeName(), ofsPath.getBucketName(),
+ createIfNotExist);
+ }
+
+ /**
+ * Get OzoneBucket object to operate in.
+ * Optionally create volume and bucket if not found.
+ *
+ * @param createIfNotExist Set this to true if the caller is a write operation
+ * in order to create the volume and bucket.
+ * @throws IOException Exceptions other than OMException with result code
+ * VOLUME_NOT_FOUND or BUCKET_NOT_FOUND.
+ */
+ private OzoneBucket getBucket(String volumeStr, String bucketStr,
+ boolean createIfNotExist) throws IOException {
+
+ OzoneBucket bucket;
+ try {
+ bucket = proxy.getBucketDetails(volumeStr, bucketStr);
+ } catch (OMException ex) {
+ if (createIfNotExist) {
+ // Note: getBucketDetails always throws BUCKET_NOT_FOUND, even if
+ // the volume doesn't exist.
+ if (ex.getResult().equals(BUCKET_NOT_FOUND)) {
+ OzoneVolume volume;
+ try {
+ volume = proxy.getVolumeDetails(volumeStr);
+ } catch (OMException getVolEx) {
+ if (getVolEx.getResult().equals(VOLUME_NOT_FOUND)) {
+ // Volume doesn't exist. Create it
+ try {
+ objectStore.createVolume(volumeStr);
+ } catch (OMException newVolEx) {
+ // Ignore the case where another client created the volume
+ if (!newVolEx.getResult().equals(VOLUME_ALREADY_EXISTS)) {
+ throw newVolEx;
+ }
+ }
+ } else {
+ throw getVolEx;
+ }
+ // Try get volume again
+ volume = proxy.getVolumeDetails(volumeStr);
+ }
+ // Create the bucket
+ try {
+ volume.createBucket(bucketStr);
+ } catch (OMException newBucEx) {
+ // Ignore the case where another client created the bucket
+ if (!newBucEx.getResult().equals(BUCKET_ALREADY_EXISTS)) {
+ throw newBucEx;
+ }
+ }
+ }
+ // Try get bucket again
+ bucket = proxy.getBucketDetails(volumeStr, bucketStr);
+ } else {
+ throw ex;
+ }
+ }
+
+ return bucket;
+ }
+
+ @Override
+ public void close() throws IOException {
+ ozoneClient.close();
+ }
+
+ @Override
+ public InputStream readFile(String pathStr) throws IOException {
+ incrementCounter(Statistic.OBJECTS_READ);
+ OFSPath ofsPath = new OFSPath(pathStr);
+ String key = ofsPath.getKeyName();
+ try {
+ OzoneBucket bucket = getBucket(ofsPath, false);
+ return bucket.readFile(key).getInputStream();
+ } catch (OMException ex) {
+ if (ex.getResult() == OMException.ResultCodes.FILE_NOT_FOUND
+ || ex.getResult() == OMException.ResultCodes.NOT_A_FILE) {
+ throw new FileNotFoundException(
+ ex.getResult().name() + ": " + ex.getMessage());
+ } else {
+ throw ex;
+ }
+ }
+ }
+
+ protected void incrementCounter(Statistic objectsRead) {
+ //noop: Use OzoneClientAdapterImpl which supports statistics.
+ }
+
+ @Override
+ public OzoneFSOutputStream createFile(String pathStr, boolean overWrite,
+ boolean recursive) throws IOException {
+ incrementCounter(Statistic.OBJECTS_CREATED);
+ OFSPath ofsPath = new OFSPath(pathStr);
+ String key = ofsPath.getKeyName();
+ try {
+ OzoneBucket bucket = getBucket(ofsPath, false);
+ OzoneOutputStream ozoneOutputStream = bucket.createFile(
+ key, 0, replicationType, replicationFactor, overWrite, recursive);
+ return new OzoneFSOutputStream(ozoneOutputStream.getOutputStream());
+ } catch (OMException ex) {
+ if (ex.getResult() == OMException.ResultCodes.FILE_ALREADY_EXISTS
+ || ex.getResult() == OMException.ResultCodes.NOT_A_FILE) {
+ throw new FileAlreadyExistsException(
+ ex.getResult().name() + ": " + ex.getMessage());
+ } else {
+ throw ex;
+ }
+ }
+ }
+
+ @Override
+ public void renamePath(String path, String newPath) throws IOException {
+ incrementCounter(Statistic.OBJECTS_RENAMED);
+ OFSPath ofsPath = new OFSPath(path);
+ OFSPath ofsNewPath = new OFSPath(newPath);
+
+ // Check path and newPathName are in the same volume and same bucket.
+ // This should have been checked in BasicRootedOzoneFileSystem#rename
+ // already via regular call path unless bypassed.
+ if (!ofsPath.isInSameBucketAs(ofsNewPath)) {
+ throw new IOException("Cannot rename a key to a different bucket");
+ }
+
+ OzoneBucket bucket = getBucket(ofsPath, false);
+ String key = ofsPath.getKeyName();
+ String newKey = ofsNewPath.getKeyName();
+ bucket.renameKey(key, newKey);
+ }
+
+ /**
+ * Package-private helper function to reduce calls to getBucket().
+ * @param bucket Bucket to operate in.
+ * @param path Existing key path.
+ * @param newPath New key path.
+ * @throws IOException IOException from bucket.renameKey().
+ */
+ void renamePath(OzoneBucket bucket, String path, String newPath)
+ throws IOException {
+ incrementCounter(Statistic.OBJECTS_RENAMED);
+ OFSPath ofsPath = new OFSPath(path);
+ OFSPath ofsNewPath = new OFSPath(newPath);
+ // No same-bucket policy check here since this call path is controlled
+ String key = ofsPath.getKeyName();
+ String newKey = ofsNewPath.getKeyName();
+ bucket.renameKey(key, newKey);
+ }
+
+ /**
+ * Helper method to create an directory specified by key name in bucket.
+ *
+ * @param pathStr path to be created as directory
+ * @return true if the key is created, false otherwise
+ */
+ @Override
+ public boolean createDirectory(String pathStr) throws IOException {
+ LOG.trace("creating dir for path: {}", pathStr);
+ incrementCounter(Statistic.OBJECTS_CREATED);
+ OFSPath ofsPath = new OFSPath(pathStr);
+ // Volume name unspecified, return failure
+ if (ofsPath.getVolumeName().length() == 0) {
+ return false;
+ }
+ // Handle where only volume is specified in pathStr
+ if (ofsPath.getBucketName().length() == 0) {
Review comment:
In the /tmp case, the bucketName length is also 0? do we want to create a /tmp volume here?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org