You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by su...@apache.org on 2018/10/05 12:56:01 UTC
[06/51] [partial] hadoop git commit: HADOOP-15791. Remove Ozone
related sources from the 3.2 branch. Contributed by Elek, Marton.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
deleted file mode 100644
index fdd049a..0000000
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
+++ /dev/null
@@ -1,912 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.client.rest;
-
-import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.base.Preconditions;
-import com.google.common.base.Strings;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.StorageType;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.ozone.OzoneAcl;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.client.*;
-import org.apache.hadoop.hdds.client.OzoneQuota;
-import org.apache.hadoop.hdds.client.ReplicationFactor;
-import org.apache.hadoop.hdds.client.ReplicationType;
-import org.apache.hadoop.ozone.client.io.OzoneInputStream;
-import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
-import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
-import org.apache.hadoop.ozone.client.rest.headers.Header;
-import org.apache.hadoop.ozone.client.rest.response.BucketInfo;
-import org.apache.hadoop.ozone.client.rest.response.KeyInfoDetails;
-import org.apache.hadoop.ozone.client.rest.response.VolumeInfo;
-import org.apache.hadoop.ozone.om.OMConfigKeys;
-import org.apache.hadoop.ozone.om.helpers.ServiceInfo;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ServicePort;
-import org.apache.hadoop.ozone.web.response.ListBuckets;
-import org.apache.hadoop.ozone.web.response.ListKeys;
-import org.apache.hadoop.ozone.web.response.ListVolumes;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.util.Time;
-import org.apache.http.HttpEntity;
-import org.apache.http.HttpHeaders;
-import org.apache.http.HttpResponse;
-import org.apache.http.client.config.RequestConfig;
-import org.apache.http.client.methods.HttpDelete;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.client.methods.HttpPost;
-import org.apache.http.client.methods.HttpPut;
-import org.apache.http.client.methods.HttpUriRequest;
-import org.apache.http.client.utils.URIBuilder;
-import org.apache.http.entity.InputStreamEntity;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClients;
-import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
-import org.apache.http.util.EntityUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.io.PipedInputStream;
-import java.io.PipedOutputStream;
-import java.net.InetSocketAddress;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.text.ParseException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.FutureTask;
-import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
-
-import static java.net.HttpURLConnection.HTTP_CREATED;
-import static java.net.HttpURLConnection.HTTP_OK;
-
-/**
- * Ozone Client REST protocol implementation. It uses REST protocol to
- * connect to Ozone Handler that executes client calls.
- */
-public class RestClient implements ClientProtocol {
-
- private static final String PATH_SEPARATOR = "/";
- private static final Logger LOG = LoggerFactory.getLogger(RestClient.class);
-
- private final Configuration conf;
- private final URI ozoneRestUri;
- private final CloseableHttpClient httpClient;
- private final UserGroupInformation ugi;
- private final OzoneAcl.OzoneACLRights userRights;
-
- /**
- * Creates RestClient instance with the given configuration.
- * @param conf Configuration
- * @throws IOException
- */
- public RestClient(Configuration conf)
- throws IOException {
- try {
- Preconditions.checkNotNull(conf);
- this.conf = conf;
-
- long socketTimeout = conf.getTimeDuration(
- OzoneConfigKeys.OZONE_CLIENT_SOCKET_TIMEOUT,
- OzoneConfigKeys.OZONE_CLIENT_SOCKET_TIMEOUT_DEFAULT,
- TimeUnit.MILLISECONDS);
- long connectionTimeout = conf.getTimeDuration(
- OzoneConfigKeys.OZONE_CLIENT_CONNECTION_TIMEOUT,
- OzoneConfigKeys.OZONE_CLIENT_CONNECTION_TIMEOUT_DEFAULT,
- TimeUnit.MILLISECONDS);
- int maxConnection = conf.getInt(
- OzoneConfigKeys.OZONE_REST_CLIENT_HTTP_CONNECTION_MAX,
- OzoneConfigKeys.OZONE_REST_CLIENT_HTTP_CONNECTION_DEFAULT);
-
- int maxConnectionPerRoute = conf.getInt(
- OzoneConfigKeys.OZONE_REST_CLIENT_HTTP_CONNECTION_PER_ROUTE_MAX,
- OzoneConfigKeys
- .OZONE_REST_CLIENT_HTTP_CONNECTION_PER_ROUTE_MAX_DEFAULT
- );
-
- /*
- To make RestClient Thread safe, creating the HttpClient with
- ThreadSafeClientConnManager.
- */
- PoolingHttpClientConnectionManager connManager =
- new PoolingHttpClientConnectionManager();
- connManager.setMaxTotal(maxConnection);
- connManager.setDefaultMaxPerRoute(maxConnectionPerRoute);
-
- this.httpClient = HttpClients.custom()
- .setConnectionManager(connManager)
- .setDefaultRequestConfig(
- RequestConfig.custom()
- .setSocketTimeout(Math.toIntExact(socketTimeout))
- .setConnectTimeout(Math.toIntExact(connectionTimeout))
- .build())
- .build();
- this.ugi = UserGroupInformation.getCurrentUser();
- this.userRights = conf.getEnum(OMConfigKeys.OZONE_OM_USER_RIGHTS,
- OMConfigKeys.OZONE_OM_USER_RIGHTS_DEFAULT);
-
- // TODO: Add new configuration parameter to configure RestServerSelector.
- RestServerSelector defaultSelector = new DefaultRestServerSelector();
- InetSocketAddress restServer = getOzoneRestServerAddress(defaultSelector);
- URIBuilder uriBuilder = new URIBuilder()
- .setScheme("http")
- .setHost(restServer.getHostName())
- .setPort(restServer.getPort());
- this.ozoneRestUri = uriBuilder.build();
-
- } catch (URISyntaxException e) {
- throw new IOException(e);
- }
- }
-
- private InetSocketAddress getOzoneRestServerAddress(
- RestServerSelector selector) throws IOException {
- String httpAddress = conf.get(OMConfigKeys.OZONE_OM_HTTP_ADDRESS_KEY);
-
- if (httpAddress == null) {
- throw new IllegalArgumentException(
- OMConfigKeys.OZONE_OM_HTTP_ADDRESS_KEY + " must be defined. See" +
- " https://wiki.apache.org/hadoop/Ozone#Configuration for" +
- " details on configuring Ozone.");
- }
-
- HttpGet httpGet = new HttpGet("http://" + httpAddress + "/serviceList");
- HttpEntity entity = executeHttpRequest(httpGet);
- try {
- String serviceListJson = EntityUtils.toString(entity);
-
- ObjectMapper objectMapper = new ObjectMapper();
- TypeReference<List<ServiceInfo>> serviceInfoReference =
- new TypeReference<List<ServiceInfo>>() {
- };
- List<ServiceInfo> services = objectMapper.readValue(
- serviceListJson, serviceInfoReference);
-
- List<ServiceInfo> dataNodeInfos = services.stream().filter(
- a -> a.getNodeType().equals(HddsProtos.NodeType.DATANODE))
- .collect(Collectors.toList());
-
- ServiceInfo restServer = selector.getRestServer(dataNodeInfos);
-
- return NetUtils.createSocketAddr(
- NetUtils.normalizeHostName(restServer.getHostname()) + ":"
- + restServer.getPort(ServicePort.Type.HTTP));
- } finally {
- EntityUtils.consume(entity);
- }
- }
-
- @Override
- public void createVolume(String volumeName) throws IOException {
- createVolume(volumeName, VolumeArgs.newBuilder().build());
- }
-
- @Override
- public void createVolume(String volumeName, VolumeArgs volArgs)
- throws IOException {
- try {
- HddsClientUtils.verifyResourceName(volumeName);
- Preconditions.checkNotNull(volArgs);
- URIBuilder builder = new URIBuilder(ozoneRestUri);
- String owner = volArgs.getOwner() == null ?
- ugi.getUserName() : volArgs.getOwner();
- //TODO: support for ACLs has to be done in OzoneHandler (rest server)
- /**
- List<OzoneAcl> listOfAcls = new ArrayList<>();
- //User ACL
- listOfAcls.add(new OzoneAcl(OzoneAcl.OzoneACLType.USER,
- owner, userRights));
- //ACLs from VolumeArgs
- if(volArgs.getAcls() != null) {
- listOfAcls.addAll(volArgs.getAcls());
- }
- */
- builder.setPath(PATH_SEPARATOR + volumeName);
-
- String quota = volArgs.getQuota();
- if(quota != null) {
- builder.setParameter(Header.OZONE_QUOTA_QUERY_TAG, quota);
- }
-
- HttpPost httpPost = new HttpPost(builder.build());
- addOzoneHeaders(httpPost);
- //use admin from VolumeArgs, if it's present
- if(volArgs.getAdmin() != null) {
- httpPost.removeHeaders(HttpHeaders.AUTHORIZATION);
- httpPost.addHeader(HttpHeaders.AUTHORIZATION,
- Header.OZONE_SIMPLE_AUTHENTICATION_SCHEME + " " +
- volArgs.getAdmin());
- }
- httpPost.addHeader(Header.OZONE_USER, owner);
- LOG.info("Creating Volume: {}, with {} as owner and quota set to {}.",
- volumeName, owner, quota == null ? "default" : quota);
- EntityUtils.consume(executeHttpRequest(httpPost));
- } catch (URISyntaxException e) {
- throw new IOException(e);
- }
- }
-
-
- @Override
- public void setVolumeOwner(String volumeName, String owner)
- throws IOException {
- try {
- HddsClientUtils.verifyResourceName(volumeName);
- Preconditions.checkNotNull(owner);
- URIBuilder builder = new URIBuilder(ozoneRestUri);
- builder.setPath(PATH_SEPARATOR + volumeName);
- HttpPut httpPut = new HttpPut(builder.build());
- addOzoneHeaders(httpPut);
- httpPut.addHeader(Header.OZONE_USER, owner);
- EntityUtils.consume(executeHttpRequest(httpPut));
- } catch (URISyntaxException e) {
- throw new IOException(e);
- }
- }
-
- @Override
- public void setVolumeQuota(String volumeName, OzoneQuota quota)
- throws IOException {
- try {
- HddsClientUtils.verifyResourceName(volumeName);
- Preconditions.checkNotNull(quota);
- String quotaString = quota.toString();
- URIBuilder builder = new URIBuilder(ozoneRestUri);
- builder.setPath(PATH_SEPARATOR + volumeName);
- builder.setParameter(Header.OZONE_QUOTA_QUERY_TAG, quotaString);
- HttpPut httpPut = new HttpPut(builder.build());
- addOzoneHeaders(httpPut);
- EntityUtils.consume(executeHttpRequest(httpPut));
- } catch (URISyntaxException e) {
- throw new IOException(e);
- }
- }
-
- @Override
- public OzoneVolume getVolumeDetails(String volumeName)
- throws IOException {
- try {
- HddsClientUtils.verifyResourceName(volumeName);
- URIBuilder builder = new URIBuilder(ozoneRestUri);
- builder.setPath(PATH_SEPARATOR + volumeName);
- builder.setParameter(Header.OZONE_INFO_QUERY_TAG,
- Header.OZONE_INFO_QUERY_VOLUME);
- HttpGet httpGet = new HttpGet(builder.build());
- addOzoneHeaders(httpGet);
- HttpEntity response = executeHttpRequest(httpGet);
- VolumeInfo volInfo =
- VolumeInfo.parse(EntityUtils.toString(response));
- //TODO: OzoneHandler in datanode has to be modified to send ACLs
- OzoneVolume volume = new OzoneVolume(conf,
- this,
- volInfo.getVolumeName(),
- volInfo.getCreatedBy(),
- volInfo.getOwner().getName(),
- volInfo.getQuota().sizeInBytes(),
- HddsClientUtils.formatDateTime(volInfo.getCreatedOn()),
- null);
- EntityUtils.consume(response);
- return volume;
- } catch (URISyntaxException | ParseException e) {
- throw new IOException(e);
- }
- }
-
- @Override
- public boolean checkVolumeAccess(String volumeName, OzoneAcl acl)
- throws IOException {
- throw new UnsupportedOperationException("Not yet implemented.");
- }
-
- @Override
- public void deleteVolume(String volumeName) throws IOException {
- try {
- HddsClientUtils.verifyResourceName(volumeName);
- URIBuilder builder = new URIBuilder(ozoneRestUri);
- builder.setPath(PATH_SEPARATOR + volumeName);
- HttpDelete httpDelete = new HttpDelete(builder.build());
- addOzoneHeaders(httpDelete);
- EntityUtils.consume(executeHttpRequest(httpDelete));
- } catch (URISyntaxException e) {
- throw new IOException(e);
- }
- }
-
- @Override
- public List<OzoneVolume> listVolumes(String volumePrefix, String prevKey,
- int maxListResult)
- throws IOException {
- return listVolumes(null, volumePrefix, prevKey, maxListResult);
- }
-
- @Override
- public List<OzoneVolume> listVolumes(String user, String volumePrefix,
- String prevKey, int maxListResult)
- throws IOException {
- try {
- URIBuilder builder = new URIBuilder(ozoneRestUri);
- builder.setPath(PATH_SEPARATOR);
- builder.addParameter(Header.OZONE_INFO_QUERY_TAG,
- Header.OZONE_LIST_QUERY_SERVICE);
- builder.addParameter(Header.OZONE_LIST_QUERY_MAXKEYS,
- String.valueOf(maxListResult));
- addQueryParamter(Header.OZONE_LIST_QUERY_PREFIX, volumePrefix, builder);
- addQueryParamter(Header.OZONE_LIST_QUERY_PREVKEY, prevKey, builder);
- HttpGet httpGet = new HttpGet(builder.build());
- if (!Strings.isNullOrEmpty(user)) {
- httpGet.addHeader(Header.OZONE_USER, user);
- }
- addOzoneHeaders(httpGet);
- HttpEntity response = executeHttpRequest(httpGet);
- ListVolumes volumeList =
- ListVolumes.parse(EntityUtils.toString(response));
- EntityUtils.consume(response);
- return volumeList.getVolumes().stream().map(volInfo -> {
- long creationTime = 0;
- try {
- creationTime = HddsClientUtils.formatDateTime(volInfo.getCreatedOn());
- } catch (ParseException e) {
- LOG.warn("Parse exception in getting creation time for volume", e);
- }
- return new OzoneVolume(conf, this, volInfo.getVolumeName(),
- volInfo.getCreatedBy(), volInfo.getOwner().getName(),
- volInfo.getQuota().sizeInBytes(), creationTime, null);
- }).collect(Collectors.toList());
- } catch (URISyntaxException e) {
- throw new IOException(e);
- }
- }
-
- @Override
- public void createBucket(String volumeName, String bucketName)
- throws IOException {
- createBucket(volumeName, bucketName, BucketArgs.newBuilder().build());
- }
-
- @Override
- public void createBucket(
- String volumeName, String bucketName, BucketArgs bucketArgs)
- throws IOException {
- try {
- HddsClientUtils.verifyResourceName(volumeName, bucketName);
- Preconditions.checkNotNull(bucketArgs);
- URIBuilder builder = new URIBuilder(ozoneRestUri);
- OzoneConsts.Versioning versioning = OzoneConsts.Versioning.DISABLED;
- if(bucketArgs.getVersioning() != null &&
- bucketArgs.getVersioning()) {
- versioning = OzoneConsts.Versioning.ENABLED;
- }
- StorageType storageType = bucketArgs.getStorageType() == null ?
- StorageType.DEFAULT : bucketArgs.getStorageType();
-
- builder.setPath(PATH_SEPARATOR + volumeName +
- PATH_SEPARATOR + bucketName);
- HttpPost httpPost = new HttpPost(builder.build());
- addOzoneHeaders(httpPost);
-
- //ACLs from BucketArgs
- if(bucketArgs.getAcls() != null) {
- for (OzoneAcl acl : bucketArgs.getAcls()) {
- httpPost.addHeader(
- Header.OZONE_ACLS, Header.OZONE_ACL_ADD + " " + acl.toString());
- }
- }
- httpPost.addHeader(Header.OZONE_STORAGE_TYPE, storageType.toString());
- httpPost.addHeader(Header.OZONE_BUCKET_VERSIONING,
- versioning.toString());
- LOG.info("Creating Bucket: {}/{}, with Versioning {} and Storage Type" +
- " set to {}", volumeName, bucketName, versioning,
- storageType);
-
- EntityUtils.consume(executeHttpRequest(httpPost));
- } catch (URISyntaxException e) {
- throw new IOException(e);
- }
- }
-
- @Override
- public void addBucketAcls(
- String volumeName, String bucketName, List<OzoneAcl> addAcls)
- throws IOException {
- try {
- HddsClientUtils.verifyResourceName(volumeName, bucketName);
- Preconditions.checkNotNull(addAcls);
- URIBuilder builder = new URIBuilder(ozoneRestUri);
-
- builder.setPath(PATH_SEPARATOR + volumeName +
- PATH_SEPARATOR + bucketName);
- HttpPut httpPut = new HttpPut(builder.build());
- addOzoneHeaders(httpPut);
-
- for (OzoneAcl acl : addAcls) {
- httpPut.addHeader(
- Header.OZONE_ACLS, Header.OZONE_ACL_ADD + " " + acl.toString());
- }
- EntityUtils.consume(executeHttpRequest(httpPut));
- } catch (URISyntaxException e) {
- throw new IOException(e);
- }
- }
-
- @Override
- public void removeBucketAcls(
- String volumeName, String bucketName, List<OzoneAcl> removeAcls)
- throws IOException {
- try {
- HddsClientUtils.verifyResourceName(volumeName, bucketName);
- Preconditions.checkNotNull(removeAcls);
- URIBuilder builder = new URIBuilder(ozoneRestUri);
-
- builder.setPath(PATH_SEPARATOR + volumeName +
- PATH_SEPARATOR + bucketName);
- HttpPut httpPut = new HttpPut(builder.build());
- addOzoneHeaders(httpPut);
-
- for (OzoneAcl acl : removeAcls) {
- httpPut.addHeader(
- Header.OZONE_ACLS, Header.OZONE_ACL_REMOVE + " " + acl.toString());
- }
- EntityUtils.consume(executeHttpRequest(httpPut));
- } catch (URISyntaxException e) {
- throw new IOException(e);
- }
- }
-
- @Override
- public void setBucketVersioning(
- String volumeName, String bucketName, Boolean versioning)
- throws IOException {
- try {
- HddsClientUtils.verifyResourceName(volumeName, bucketName);
- Preconditions.checkNotNull(versioning);
- URIBuilder builder = new URIBuilder(ozoneRestUri);
-
- builder.setPath(PATH_SEPARATOR + volumeName +
- PATH_SEPARATOR + bucketName);
- HttpPut httpPut = new HttpPut(builder.build());
- addOzoneHeaders(httpPut);
-
- httpPut.addHeader(Header.OZONE_BUCKET_VERSIONING,
- getBucketVersioning(versioning).toString());
- EntityUtils.consume(executeHttpRequest(httpPut));
- } catch (URISyntaxException e) {
- throw new IOException(e);
- }
- }
-
- @Override
- public void setBucketStorageType(
- String volumeName, String bucketName, StorageType storageType)
- throws IOException {
- try {
- HddsClientUtils.verifyResourceName(volumeName, bucketName);
- Preconditions.checkNotNull(storageType);
- URIBuilder builder = new URIBuilder(ozoneRestUri);
-
- builder.setPath(PATH_SEPARATOR + volumeName +
- PATH_SEPARATOR + bucketName);
- HttpPut httpPut = new HttpPut(builder.build());
- addOzoneHeaders(httpPut);
-
- httpPut.addHeader(Header.OZONE_STORAGE_TYPE, storageType.toString());
- EntityUtils.consume(executeHttpRequest(httpPut));
- } catch (URISyntaxException e) {
- throw new IOException(e);
- }
- }
-
- @Override
- public void deleteBucket(String volumeName, String bucketName)
- throws IOException {
- try {
- HddsClientUtils.verifyResourceName(volumeName, bucketName);
- URIBuilder builder = new URIBuilder(ozoneRestUri);
- builder.setPath(PATH_SEPARATOR + volumeName +
- PATH_SEPARATOR + bucketName);
- HttpDelete httpDelete = new HttpDelete(builder.build());
- addOzoneHeaders(httpDelete);
- EntityUtils.consume(executeHttpRequest(httpDelete));
- } catch (URISyntaxException e) {
- throw new IOException(e);
- }
- }
-
- @Override
- public void checkBucketAccess(String volumeName, String bucketName)
- throws IOException {
- throw new UnsupportedOperationException("Not yet implemented.");
- }
-
- @Override
- public OzoneBucket getBucketDetails(String volumeName, String bucketName)
- throws IOException {
- try {
- HddsClientUtils.verifyResourceName(volumeName, bucketName);
- URIBuilder builder = new URIBuilder(ozoneRestUri);
- builder.setPath(PATH_SEPARATOR + volumeName +
- PATH_SEPARATOR + bucketName);
- builder.setParameter(Header.OZONE_INFO_QUERY_TAG,
- Header.OZONE_INFO_QUERY_BUCKET);
- HttpGet httpGet = new HttpGet(builder.build());
- addOzoneHeaders(httpGet);
- HttpEntity response = executeHttpRequest(httpGet);
- BucketInfo bucketInfo =
- BucketInfo.parse(EntityUtils.toString(response));
- OzoneBucket bucket = new OzoneBucket(conf,
- this,
- bucketInfo.getVolumeName(),
- bucketInfo.getBucketName(),
- bucketInfo.getAcls(),
- bucketInfo.getStorageType(),
- getBucketVersioningFlag(bucketInfo.getVersioning()),
- HddsClientUtils.formatDateTime(bucketInfo.getCreatedOn()));
- EntityUtils.consume(response);
- return bucket;
- } catch (URISyntaxException | ParseException e) {
- throw new IOException(e);
- }
- }
-
- @Override
- public List<OzoneBucket> listBuckets(String volumeName, String bucketPrefix,
- String prevBucket, int maxListResult)
- throws IOException {
- try {
- HddsClientUtils.verifyResourceName(volumeName);
- URIBuilder builder = new URIBuilder(ozoneRestUri);
- builder.setPath(PATH_SEPARATOR + volumeName);
- builder.addParameter(Header.OZONE_INFO_QUERY_TAG,
- Header.OZONE_INFO_QUERY_BUCKET);
- builder.addParameter(Header.OZONE_LIST_QUERY_MAXKEYS,
- String.valueOf(maxListResult));
- addQueryParamter(Header.OZONE_LIST_QUERY_PREFIX, bucketPrefix, builder);
- addQueryParamter(Header.OZONE_LIST_QUERY_PREVKEY, prevBucket, builder);
- HttpGet httpGet = new HttpGet(builder.build());
- addOzoneHeaders(httpGet);
- HttpEntity response = executeHttpRequest(httpGet);
- ListBuckets bucketList =
- ListBuckets.parse(EntityUtils.toString(response));
- EntityUtils.consume(response);
- return bucketList.getBuckets().stream().map(bucketInfo -> {
- long creationTime = 0;
- try {
- creationTime =
- HddsClientUtils.formatDateTime(bucketInfo.getCreatedOn());
- } catch (ParseException e) {
- LOG.warn("Parse exception in getting creation time for volume", e);
- }
- return new OzoneBucket(conf, this, volumeName,
- bucketInfo.getBucketName(), bucketInfo.getAcls(),
- bucketInfo.getStorageType(),
- getBucketVersioningFlag(bucketInfo.getVersioning()), creationTime);
- }).collect(Collectors.toList());
- } catch (URISyntaxException e) {
- throw new IOException(e);
- }
- }
-
- /**
- * Writes a key in an existing bucket.
- *
- * @param volumeName Name of the Volume
- * @param bucketName Name of the Bucket
- * @param keyName Name of the Key
- * @param size Size of the data
- * @param type
- * @param factor @return {@link OzoneOutputStream}
- */
- @Override
- public OzoneOutputStream createKey(
- String volumeName, String bucketName, String keyName, long size,
- ReplicationType type, ReplicationFactor factor)
- throws IOException {
- // TODO: Once ReplicationType and ReplicationFactor are supported in
- // OzoneHandler (in Datanode), set them in header.
- try {
- HddsClientUtils.verifyResourceName(volumeName, bucketName);
- HddsClientUtils.checkNotNull(keyName, type, factor);
- URIBuilder builder = new URIBuilder(ozoneRestUri);
- builder.setPath(PATH_SEPARATOR + volumeName +
- PATH_SEPARATOR + bucketName +
- PATH_SEPARATOR + keyName);
- HttpPut putRequest = new HttpPut(builder.build());
- addOzoneHeaders(putRequest);
- PipedInputStream in = new PipedInputStream();
- OutputStream out = new PipedOutputStream(in);
- putRequest.setEntity(new InputStreamEntity(in, size));
- FutureTask<HttpEntity> futureTask =
- new FutureTask<>(() -> executeHttpRequest(putRequest));
- new Thread(futureTask).start();
- OzoneOutputStream outputStream = new OzoneOutputStream(
- new OutputStream() {
- @Override
- public void write(int b) throws IOException {
- out.write(b);
- }
-
- @Override
- public void close() throws IOException {
- try {
- out.close();
- EntityUtils.consume(futureTask.get());
- } catch (ExecutionException | InterruptedException e) {
- throw new IOException(e);
- }
- }
- });
-
- return outputStream;
- } catch (URISyntaxException e) {
- throw new IOException(e);
- }
- }
-
- @Override
- public OzoneInputStream getKey(
- String volumeName, String bucketName, String keyName)
- throws IOException {
- try {
- HddsClientUtils.verifyResourceName(volumeName, bucketName);
- Preconditions.checkNotNull(keyName);
- URIBuilder builder = new URIBuilder(ozoneRestUri);
- builder.setPath(PATH_SEPARATOR + volumeName +
- PATH_SEPARATOR + bucketName +
- PATH_SEPARATOR + keyName);
- HttpGet getRequest = new HttpGet(builder.build());
- addOzoneHeaders(getRequest);
- HttpEntity entity = executeHttpRequest(getRequest);
- PipedInputStream in = new PipedInputStream();
- OutputStream out = new PipedOutputStream(in);
- FutureTask<Void> futureTask =
- new FutureTask<>(() -> {
- entity.writeTo(out);
- out.close();
- return null;
- });
- new Thread(futureTask).start();
- OzoneInputStream inputStream = new OzoneInputStream(
- new InputStream() {
-
- @Override
- public int read() throws IOException {
- return in.read();
- }
-
- @Override
- public void close() throws IOException {
- in.close();
- EntityUtils.consume(entity);
- }
- });
-
- return inputStream;
- } catch (URISyntaxException e) {
- throw new IOException(e);
- }
- }
-
- @Override
- public void deleteKey(String volumeName, String bucketName, String keyName)
- throws IOException {
- try {
- HddsClientUtils.verifyResourceName(volumeName, bucketName);
- Preconditions.checkNotNull(keyName);
- URIBuilder builder = new URIBuilder(ozoneRestUri);
- builder.setPath(PATH_SEPARATOR + volumeName +
- PATH_SEPARATOR + bucketName + PATH_SEPARATOR + keyName);
- HttpDelete httpDelete = new HttpDelete(builder.build());
- addOzoneHeaders(httpDelete);
- EntityUtils.consume(executeHttpRequest(httpDelete));
- } catch (URISyntaxException e) {
- throw new IOException(e);
- }
- }
-
- @Override
- public void renameKey(String volumeName, String bucketName,
- String fromKeyName, String toKeyName) throws IOException {
- try {
- HddsClientUtils.verifyResourceName(volumeName, bucketName);
- HddsClientUtils.checkNotNull(fromKeyName, toKeyName);
- URIBuilder builder = new URIBuilder(ozoneRestUri);
- builder.setPath(PATH_SEPARATOR + volumeName + PATH_SEPARATOR + bucketName
- + PATH_SEPARATOR + fromKeyName);
- builder.addParameter(Header.OZONE_RENAME_TO_KEY_PARAM_NAME, toKeyName);
- HttpPost httpPost = new HttpPost(builder.build());
- addOzoneHeaders(httpPost);
- EntityUtils.consume(executeHttpRequest(httpPost));
- } catch (URISyntaxException e) {
- throw new IOException(e);
- }
- }
-
- @Override
- public List<OzoneKey> listKeys(String volumeName, String bucketName,
- String keyPrefix, String prevKey,
- int maxListResult)
- throws IOException {
- try {
- HddsClientUtils.verifyResourceName(volumeName);
- URIBuilder builder = new URIBuilder(ozoneRestUri);
- builder
- .setPath(PATH_SEPARATOR + volumeName + PATH_SEPARATOR + bucketName);
- builder.addParameter(Header.OZONE_INFO_QUERY_TAG,
- Header.OZONE_INFO_QUERY_KEY);
- builder.addParameter(Header.OZONE_LIST_QUERY_MAXKEYS,
- String.valueOf(maxListResult));
- addQueryParamter(Header.OZONE_LIST_QUERY_PREFIX, keyPrefix, builder);
- addQueryParamter(Header.OZONE_LIST_QUERY_PREVKEY, prevKey, builder);
- HttpGet httpGet = new HttpGet(builder.build());
- addOzoneHeaders(httpGet);
- HttpEntity response = executeHttpRequest(httpGet);
- ListKeys keyList = ListKeys.parse(EntityUtils.toString(response));
- EntityUtils.consume(response);
- return keyList.getKeyList().stream().map(keyInfo -> {
- long creationTime = 0, modificationTime = 0;
- try {
- creationTime = HddsClientUtils.formatDateTime(keyInfo.getCreatedOn());
- modificationTime =
- HddsClientUtils.formatDateTime(keyInfo.getModifiedOn());
- } catch (ParseException e) {
- LOG.warn("Parse exception in getting creation time for volume", e);
- }
- return new OzoneKey(volumeName, bucketName, keyInfo.getKeyName(),
- keyInfo.getSize(), creationTime, modificationTime);
- }).collect(Collectors.toList());
- } catch (URISyntaxException e) {
- throw new IOException(e);
- }
- }
-
- @Override
- public OzoneKeyDetails getKeyDetails(
- String volumeName, String bucketName, String keyName)
- throws IOException {
- try {
- HddsClientUtils.verifyResourceName(volumeName, bucketName);
- Preconditions.checkNotNull(keyName);
- URIBuilder builder = new URIBuilder(ozoneRestUri);
- builder.setPath(PATH_SEPARATOR + volumeName +
- PATH_SEPARATOR + bucketName + PATH_SEPARATOR + keyName);
- builder.setParameter(Header.OZONE_INFO_QUERY_TAG,
- Header.OZONE_INFO_QUERY_KEY_DETAIL);
- HttpGet httpGet = new HttpGet(builder.build());
- addOzoneHeaders(httpGet);
- HttpEntity response = executeHttpRequest(httpGet);
- KeyInfoDetails keyInfo =
- KeyInfoDetails.parse(EntityUtils.toString(response));
-
- List<OzoneKeyLocation> ozoneKeyLocations = new ArrayList<>();
- keyInfo.getKeyLocations().forEach((a) -> ozoneKeyLocations.add(
- new OzoneKeyLocation(a.getContainerID(), a.getLocalID(),
- a.getLength(), a.getOffset())));
- OzoneKeyDetails key = new OzoneKeyDetails(volumeName,
- bucketName,
- keyInfo.getKeyName(),
- keyInfo.getSize(),
- HddsClientUtils.formatDateTime(keyInfo.getCreatedOn()),
- HddsClientUtils.formatDateTime(keyInfo.getModifiedOn()),
- ozoneKeyLocations);
- EntityUtils.consume(response);
- return key;
- } catch (URISyntaxException | ParseException e) {
- throw new IOException(e);
- }
- }
-
- /**
- * Adds Ozone headers to http request.
- *
- * @param httpRequest Http Request
- */
- private void addOzoneHeaders(HttpUriRequest httpRequest) {
- httpRequest.addHeader(HttpHeaders.AUTHORIZATION,
- Header.OZONE_SIMPLE_AUTHENTICATION_SCHEME + " " +
- ugi.getUserName());
- httpRequest.addHeader(HttpHeaders.DATE,
- HddsClientUtils.formatDateTime(Time.monotonicNow()));
- httpRequest.addHeader(Header.OZONE_VERSION_HEADER,
- Header.OZONE_V1_VERSION_HEADER);
- }
-
- /**
- * Sends the http request to server and returns the response HttpEntity.
- * It's responsibility of the caller to consume and close response HttpEntity
- * by calling {@code EntityUtils.consume}
- *
- * @param httpUriRequest http request
- * @throws IOException
- */
- private HttpEntity executeHttpRequest(HttpUriRequest httpUriRequest)
- throws IOException {
- HttpResponse response = httpClient.execute(httpUriRequest);
- int errorCode = response.getStatusLine().getStatusCode();
- HttpEntity entity = response.getEntity();
- if ((errorCode == HTTP_OK) || (errorCode == HTTP_CREATED)) {
- return entity;
- }
- if (entity != null) {
- throw new IOException(
- OzoneException.parse(EntityUtils.toString(entity)));
- } else {
- throw new IOException("Unexpected null in http payload," +
- " while processing request");
- }
- }
-
- /**
- * Converts OzoneConts.Versioning to boolean.
- *
- * @param version
- * @return corresponding boolean value
- */
- private Boolean getBucketVersioningFlag(
- OzoneConsts.Versioning version) {
- if(version != null) {
- switch(version) {
- case ENABLED:
- return true;
- case NOT_DEFINED:
- case DISABLED:
- default:
- return false;
- }
- }
- return false;
- }
-
- /**
- * Converts Bucket versioning flag into OzoneConts.Versioning.
- *
- * @param flag versioning flag
- * @return corresponding OzoneConts.Versionin
- */
- private OzoneConsts.Versioning getBucketVersioning(Boolean flag) {
- if(flag != null) {
- if(flag) {
- return OzoneConsts.Versioning.ENABLED;
- } else {
- return OzoneConsts.Versioning.DISABLED;
- }
- }
- return OzoneConsts.Versioning.NOT_DEFINED;
- }
-
- @Override
- public void close() throws IOException {
- httpClient.close();
- }
-
- private void addQueryParamter(String param, String value,
- URIBuilder builder) {
- if (!Strings.isNullOrEmpty(value)) {
- builder.addParameter(param, value);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestServerSelector.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestServerSelector.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestServerSelector.java
deleted file mode 100644
index fbd6eb8..0000000
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestServerSelector.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.client.rest;
-
-import org.apache.hadoop.ozone.om.helpers.ServiceInfo;
-
-import java.util.List;
-
-/**
- * The implementor of this interface should select the REST server which will
- * be used by the client to connect to Ozone Cluster, given list of
- * REST Servers/DataNodes (DataNodes are the ones which hosts REST Service).
- */
-public interface RestServerSelector {
-
- /**
- * Returns the REST Service which will be used by the client for connection.
- *
- * @param restServices list of available REST servers
- * @return ServiceInfo
- */
- ServiceInfo getRestServer(List<ServiceInfo> restServices);
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/exceptions/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/exceptions/package-info.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/exceptions/package-info.java
deleted file mode 100644
index 233e788..0000000
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/exceptions/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.client.rest.exceptions;
-
-/**
- * This package contains ozone rest client libraries.
- */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/headers/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/headers/package-info.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/headers/package-info.java
deleted file mode 100644
index 340709f..0000000
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/headers/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-@InterfaceAudience.Private
-package org.apache.hadoop.ozone.client.rest.headers;
-
-import org.apache.hadoop.classification.InterfaceAudience;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/package-info.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/package-info.java
deleted file mode 100644
index ebcc104..0000000
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.client.rest;
-
-/**
- * This package contains Ozone rest client library classes.
- */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
deleted file mode 100644
index 330eba8..0000000
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
+++ /dev/null
@@ -1,576 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.client.rpc;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.fs.StorageType;
-import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.io.retry.RetryPolicy;
-import org.apache.hadoop.ipc.Client;
-import org.apache.hadoop.ipc.ProtobufRpcEngine;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.ozone.OmUtils;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.client.*;
-import org.apache.hadoop.hdds.client.OzoneQuota;
-import org.apache.hadoop.hdds.client.ReplicationFactor;
-import org.apache.hadoop.hdds.client.ReplicationType;
-import org.apache.hadoop.ozone.client.VolumeArgs;
-import org.apache.hadoop.ozone.client.OzoneClientUtils;
-import org.apache.hadoop.ozone.client.io.ChunkGroupInputStream;
-import org.apache.hadoop.ozone.client.io.ChunkGroupOutputStream;
-import org.apache.hadoop.ozone.client.io.LengthInputStream;
-import org.apache.hadoop.ozone.client.io.OzoneInputStream;
-import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
-import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
-import org.apache.hadoop.ozone.om.helpers.OmBucketArgs;
-import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
-import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
-import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
-import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
-import org.apache.hadoop.ozone.om.helpers.OpenKeySession;
-import org.apache.hadoop.ozone.om.helpers.ServiceInfo;
-import org.apache.hadoop.ozone.om.protocolPB.OzoneManagerProtocolClientSideTranslatorPB;
-import org.apache.hadoop.ozone.om.protocolPB.OzoneManagerProtocolPB;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.ozone.OzoneAcl;
-import org.apache.hadoop.ozone.om.OMConfigKeys;
-import org.apache.hadoop.ozone.protocol.proto
- .OzoneManagerProtocolProtos.ServicePort;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.ozone.protocolPB.OMPBHelper;
-import org.apache.hadoop.hdds.scm.ScmConfigKeys;
-import org.apache.hadoop.hdds.scm.XceiverClientManager;
-import org.apache.hadoop.hdds.scm.protocolPB
- .StorageContainerLocationProtocolClientSideTranslatorPB;
-import org.apache.hadoop.hdds.scm.protocolPB
- .StorageContainerLocationProtocolPB;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.util.*;
-import java.util.stream.Collectors;
-
-/**
- * Ozone RPC Client Implementation, it connects to OM, SCM and DataNode
- * to execute client calls. This uses RPC protocol for communication
- * with the servers.
- */
-public class RpcClient implements ClientProtocol {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(RpcClient.class);
-
- private final OzoneConfiguration conf;
- private final StorageContainerLocationProtocolClientSideTranslatorPB
- storageContainerLocationClient;
- private final OzoneManagerProtocolClientSideTranslatorPB
- ozoneManagerClient;
- private final XceiverClientManager xceiverClientManager;
- private final int chunkSize;
- private final UserGroupInformation ugi;
- private final OzoneAcl.OzoneACLRights userRights;
- private final OzoneAcl.OzoneACLRights groupRights;
- private final RetryPolicy retryPolicy;
-
- /**
- * Creates RpcClient instance with the given configuration.
- * @param conf
- * @throws IOException
- */
- public RpcClient(Configuration conf) throws IOException {
- Preconditions.checkNotNull(conf);
- this.conf = new OzoneConfiguration(conf);
- this.ugi = UserGroupInformation.getCurrentUser();
- this.userRights = conf.getEnum(OMConfigKeys.OZONE_OM_USER_RIGHTS,
- OMConfigKeys.OZONE_OM_USER_RIGHTS_DEFAULT);
- this.groupRights = conf.getEnum(OMConfigKeys.OZONE_OM_GROUP_RIGHTS,
- OMConfigKeys.OZONE_OM_GROUP_RIGHTS_DEFAULT);
- long omVersion =
- RPC.getProtocolVersion(OzoneManagerProtocolPB.class);
- InetSocketAddress omAddress = OmUtils
- .getOmAddressForClients(conf);
- RPC.setProtocolEngine(conf, OzoneManagerProtocolPB.class,
- ProtobufRpcEngine.class);
- this.ozoneManagerClient =
- new OzoneManagerProtocolClientSideTranslatorPB(
- RPC.getProxy(OzoneManagerProtocolPB.class, omVersion,
- omAddress, UserGroupInformation.getCurrentUser(), conf,
- NetUtils.getDefaultSocketFactory(conf),
- Client.getRpcTimeout(conf)));
-
- long scmVersion =
- RPC.getProtocolVersion(StorageContainerLocationProtocolPB.class);
- InetSocketAddress scmAddress = getScmAddressForClient();
- RPC.setProtocolEngine(conf, StorageContainerLocationProtocolPB.class,
- ProtobufRpcEngine.class);
- this.storageContainerLocationClient =
- new StorageContainerLocationProtocolClientSideTranslatorPB(
- RPC.getProxy(StorageContainerLocationProtocolPB.class, scmVersion,
- scmAddress, UserGroupInformation.getCurrentUser(), conf,
- NetUtils.getDefaultSocketFactory(conf),
- Client.getRpcTimeout(conf)));
-
- this.xceiverClientManager = new XceiverClientManager(conf);
- retryPolicy = OzoneClientUtils.createRetryPolicy(conf);
-
- int configuredChunkSize = conf.getInt(
- ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY,
- ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_DEFAULT);
- if(configuredChunkSize > ScmConfigKeys.OZONE_SCM_CHUNK_MAX_SIZE) {
- LOG.warn("The chunk size ({}) is not allowed to be more than"
- + " the maximum size ({}),"
- + " resetting to the maximum size.",
- configuredChunkSize, ScmConfigKeys.OZONE_SCM_CHUNK_MAX_SIZE);
- chunkSize = ScmConfigKeys.OZONE_SCM_CHUNK_MAX_SIZE;
- } else {
- chunkSize = configuredChunkSize;
- }
- }
-
- private InetSocketAddress getScmAddressForClient() throws IOException {
- List<ServiceInfo> services = ozoneManagerClient.getServiceList();
- ServiceInfo scmInfo = services.stream().filter(
- a -> a.getNodeType().equals(HddsProtos.NodeType.SCM))
- .collect(Collectors.toList()).get(0);
- return NetUtils.createSocketAddr(scmInfo.getHostname()+ ":" +
- scmInfo.getPort(ServicePort.Type.RPC));
- }
-
- @Override
- public void createVolume(String volumeName) throws IOException {
- createVolume(volumeName, VolumeArgs.newBuilder().build());
- }
-
- @Override
- public void createVolume(String volumeName, VolumeArgs volArgs)
- throws IOException {
- HddsClientUtils.verifyResourceName(volumeName);
- Preconditions.checkNotNull(volArgs);
-
- String admin = volArgs.getAdmin() == null ?
- ugi.getUserName() : volArgs.getAdmin();
- String owner = volArgs.getOwner() == null ?
- ugi.getUserName() : volArgs.getOwner();
- long quota = volArgs.getQuota() == null ?
- OzoneConsts.MAX_QUOTA_IN_BYTES :
- OzoneQuota.parseQuota(volArgs.getQuota()).sizeInBytes();
- List<OzoneAcl> listOfAcls = new ArrayList<>();
- //User ACL
- listOfAcls.add(new OzoneAcl(OzoneAcl.OzoneACLType.USER,
- owner, userRights));
- //Group ACLs of the User
- List<String> userGroups = Arrays.asList(UserGroupInformation
- .createRemoteUser(owner).getGroupNames());
- userGroups.stream().forEach((group) -> listOfAcls.add(
- new OzoneAcl(OzoneAcl.OzoneACLType.GROUP, group, groupRights)));
- //ACLs from VolumeArgs
- if(volArgs.getAcls() != null) {
- listOfAcls.addAll(volArgs.getAcls());
- }
-
- OmVolumeArgs.Builder builder = OmVolumeArgs.newBuilder();
- builder.setVolume(volumeName);
- builder.setAdminName(admin);
- builder.setOwnerName(owner);
- builder.setQuotaInBytes(quota);
-
- //Remove duplicates and add ACLs
- for (OzoneAcl ozoneAcl :
- listOfAcls.stream().distinct().collect(Collectors.toList())) {
- builder.addOzoneAcls(OMPBHelper.convertOzoneAcl(ozoneAcl));
- }
-
- LOG.info("Creating Volume: {}, with {} as owner and quota set to {} bytes.",
- volumeName, owner, quota);
- ozoneManagerClient.createVolume(builder.build());
- }
-
- @Override
- public void setVolumeOwner(String volumeName, String owner)
- throws IOException {
- HddsClientUtils.verifyResourceName(volumeName);
- Preconditions.checkNotNull(owner);
- ozoneManagerClient.setOwner(volumeName, owner);
- }
-
- @Override
- public void setVolumeQuota(String volumeName, OzoneQuota quota)
- throws IOException {
- HddsClientUtils.verifyResourceName(volumeName);
- Preconditions.checkNotNull(quota);
- long quotaInBytes = quota.sizeInBytes();
- ozoneManagerClient.setQuota(volumeName, quotaInBytes);
- }
-
- @Override
- public OzoneVolume getVolumeDetails(String volumeName)
- throws IOException {
- HddsClientUtils.verifyResourceName(volumeName);
- OmVolumeArgs volume = ozoneManagerClient.getVolumeInfo(volumeName);
- return new OzoneVolume(
- conf,
- this,
- volume.getVolume(),
- volume.getAdminName(),
- volume.getOwnerName(),
- volume.getQuotaInBytes(),
- volume.getCreationTime(),
- volume.getAclMap().ozoneAclGetProtobuf().stream().
- map(OMPBHelper::convertOzoneAcl).collect(Collectors.toList()));
- }
-
- @Override
- public boolean checkVolumeAccess(String volumeName, OzoneAcl acl)
- throws IOException {
- throw new UnsupportedOperationException("Not yet implemented.");
- }
-
- @Override
- public void deleteVolume(String volumeName) throws IOException {
- HddsClientUtils.verifyResourceName(volumeName);
- ozoneManagerClient.deleteVolume(volumeName);
- }
-
- @Override
- public List<OzoneVolume> listVolumes(String volumePrefix, String prevVolume,
- int maxListResult)
- throws IOException {
- List<OmVolumeArgs> volumes = ozoneManagerClient.listAllVolumes(
- volumePrefix, prevVolume, maxListResult);
-
- return volumes.stream().map(volume -> new OzoneVolume(
- conf,
- this,
- volume.getVolume(),
- volume.getAdminName(),
- volume.getOwnerName(),
- volume.getQuotaInBytes(),
- volume.getCreationTime(),
- volume.getAclMap().ozoneAclGetProtobuf().stream().
- map(OMPBHelper::convertOzoneAcl).collect(Collectors.toList())))
- .collect(Collectors.toList());
- }
-
- @Override
- public List<OzoneVolume> listVolumes(String user, String volumePrefix,
- String prevVolume, int maxListResult)
- throws IOException {
- List<OmVolumeArgs> volumes = ozoneManagerClient.listVolumeByUser(
- user, volumePrefix, prevVolume, maxListResult);
-
- return volumes.stream().map(volume -> new OzoneVolume(
- conf,
- this,
- volume.getVolume(),
- volume.getAdminName(),
- volume.getOwnerName(),
- volume.getQuotaInBytes(),
- volume.getCreationTime(),
- volume.getAclMap().ozoneAclGetProtobuf().stream().
- map(OMPBHelper::convertOzoneAcl).collect(Collectors.toList())))
- .collect(Collectors.toList());
- }
-
- @Override
- public void createBucket(String volumeName, String bucketName)
- throws IOException {
- createBucket(volumeName, bucketName, BucketArgs.newBuilder().build());
- }
-
- @Override
- public void createBucket(
- String volumeName, String bucketName, BucketArgs bucketArgs)
- throws IOException {
- HddsClientUtils.verifyResourceName(volumeName, bucketName);
- Preconditions.checkNotNull(bucketArgs);
-
- Boolean isVersionEnabled = bucketArgs.getVersioning() == null ?
- Boolean.FALSE : bucketArgs.getVersioning();
- StorageType storageType = bucketArgs.getStorageType() == null ?
- StorageType.DEFAULT : bucketArgs.getStorageType();
- List<OzoneAcl> listOfAcls = new ArrayList<>();
- //User ACL
- listOfAcls.add(new OzoneAcl(OzoneAcl.OzoneACLType.USER,
- ugi.getUserName(), userRights));
- //Group ACLs of the User
- List<String> userGroups = Arrays.asList(UserGroupInformation
- .createRemoteUser(ugi.getUserName()).getGroupNames());
- userGroups.stream().forEach((group) -> listOfAcls.add(
- new OzoneAcl(OzoneAcl.OzoneACLType.GROUP, group, groupRights)));
- //ACLs from BucketArgs
- if(bucketArgs.getAcls() != null) {
- listOfAcls.addAll(bucketArgs.getAcls());
- }
-
- OmBucketInfo.Builder builder = OmBucketInfo.newBuilder();
- builder.setVolumeName(volumeName)
- .setBucketName(bucketName)
- .setIsVersionEnabled(isVersionEnabled)
- .setStorageType(storageType)
- .setAcls(listOfAcls.stream().distinct().collect(Collectors.toList()));
-
- LOG.info("Creating Bucket: {}/{}, with Versioning {} and " +
- "Storage Type set to {}", volumeName, bucketName, isVersionEnabled,
- storageType);
- ozoneManagerClient.createBucket(builder.build());
- }
-
- @Override
- public void addBucketAcls(
- String volumeName, String bucketName, List<OzoneAcl> addAcls)
- throws IOException {
- HddsClientUtils.verifyResourceName(volumeName, bucketName);
- Preconditions.checkNotNull(addAcls);
- OmBucketArgs.Builder builder = OmBucketArgs.newBuilder();
- builder.setVolumeName(volumeName)
- .setBucketName(bucketName)
- .setAddAcls(addAcls);
- ozoneManagerClient.setBucketProperty(builder.build());
- }
-
- @Override
- public void removeBucketAcls(
- String volumeName, String bucketName, List<OzoneAcl> removeAcls)
- throws IOException {
- HddsClientUtils.verifyResourceName(volumeName, bucketName);
- Preconditions.checkNotNull(removeAcls);
- OmBucketArgs.Builder builder = OmBucketArgs.newBuilder();
- builder.setVolumeName(volumeName)
- .setBucketName(bucketName)
- .setRemoveAcls(removeAcls);
- ozoneManagerClient.setBucketProperty(builder.build());
- }
-
- @Override
- public void setBucketVersioning(
- String volumeName, String bucketName, Boolean versioning)
- throws IOException {
- HddsClientUtils.verifyResourceName(volumeName, bucketName);
- Preconditions.checkNotNull(versioning);
- OmBucketArgs.Builder builder = OmBucketArgs.newBuilder();
- builder.setVolumeName(volumeName)
- .setBucketName(bucketName)
- .setIsVersionEnabled(versioning);
- ozoneManagerClient.setBucketProperty(builder.build());
- }
-
- @Override
- public void setBucketStorageType(
- String volumeName, String bucketName, StorageType storageType)
- throws IOException {
- HddsClientUtils.verifyResourceName(volumeName, bucketName);
- Preconditions.checkNotNull(storageType);
- OmBucketArgs.Builder builder = OmBucketArgs.newBuilder();
- builder.setVolumeName(volumeName)
- .setBucketName(bucketName)
- .setStorageType(storageType);
- ozoneManagerClient.setBucketProperty(builder.build());
- }
-
- @Override
- public void deleteBucket(
- String volumeName, String bucketName) throws IOException {
- HddsClientUtils.verifyResourceName(volumeName, bucketName);
- ozoneManagerClient.deleteBucket(volumeName, bucketName);
- }
-
- @Override
- public void checkBucketAccess(
- String volumeName, String bucketName) throws IOException {
-
- }
-
- @Override
- public OzoneBucket getBucketDetails(
- String volumeName, String bucketName) throws IOException {
- HddsClientUtils.verifyResourceName(volumeName, bucketName);
- OmBucketInfo bucketArgs =
- ozoneManagerClient.getBucketInfo(volumeName, bucketName);
- return new OzoneBucket(
- conf,
- this,
- bucketArgs.getVolumeName(),
- bucketArgs.getBucketName(),
- bucketArgs.getAcls(),
- bucketArgs.getStorageType(),
- bucketArgs.getIsVersionEnabled(),
- bucketArgs.getCreationTime());
- }
-
- @Override
- public List<OzoneBucket> listBuckets(String volumeName, String bucketPrefix,
- String prevBucket, int maxListResult)
- throws IOException {
- List<OmBucketInfo> buckets = ozoneManagerClient.listBuckets(
- volumeName, prevBucket, bucketPrefix, maxListResult);
-
- return buckets.stream().map(bucket -> new OzoneBucket(
- conf,
- this,
- bucket.getVolumeName(),
- bucket.getBucketName(),
- bucket.getAcls(),
- bucket.getStorageType(),
- bucket.getIsVersionEnabled(),
- bucket.getCreationTime()))
- .collect(Collectors.toList());
- }
-
- @Override
- public OzoneOutputStream createKey(
- String volumeName, String bucketName, String keyName, long size,
- ReplicationType type, ReplicationFactor factor)
- throws IOException {
- HddsClientUtils.verifyResourceName(volumeName, bucketName);
- HddsClientUtils.checkNotNull(keyName, type, factor);
- String requestId = UUID.randomUUID().toString();
- OmKeyArgs keyArgs = new OmKeyArgs.Builder()
- .setVolumeName(volumeName)
- .setBucketName(bucketName)
- .setKeyName(keyName)
- .setDataSize(size)
- .setType(HddsProtos.ReplicationType.valueOf(type.toString()))
- .setFactor(HddsProtos.ReplicationFactor.valueOf(factor.getValue()))
- .build();
-
- OpenKeySession openKey = ozoneManagerClient.openKey(keyArgs);
- ChunkGroupOutputStream groupOutputStream =
- new ChunkGroupOutputStream.Builder()
- .setHandler(openKey)
- .setXceiverClientManager(xceiverClientManager)
- .setScmClient(storageContainerLocationClient)
- .setOmClient(ozoneManagerClient)
- .setChunkSize(chunkSize)
- .setRequestID(requestId)
- .setType(HddsProtos.ReplicationType.valueOf(type.toString()))
- .setFactor(HddsProtos.ReplicationFactor.valueOf(factor.getValue()))
- .setRetryPolicy(retryPolicy)
- .build();
- groupOutputStream.addPreallocateBlocks(
- openKey.getKeyInfo().getLatestVersionLocations(),
- openKey.getOpenVersion());
- return new OzoneOutputStream(groupOutputStream);
- }
-
- @Override
- public OzoneInputStream getKey(
- String volumeName, String bucketName, String keyName)
- throws IOException {
- HddsClientUtils.verifyResourceName(volumeName, bucketName);
- Preconditions.checkNotNull(keyName);
- String requestId = UUID.randomUUID().toString();
- OmKeyArgs keyArgs = new OmKeyArgs.Builder()
- .setVolumeName(volumeName)
- .setBucketName(bucketName)
- .setKeyName(keyName)
- .build();
- OmKeyInfo keyInfo = ozoneManagerClient.lookupKey(keyArgs);
- LengthInputStream lengthInputStream =
- ChunkGroupInputStream.getFromOmKeyInfo(
- keyInfo, xceiverClientManager, storageContainerLocationClient,
- requestId);
- return new OzoneInputStream(lengthInputStream.getWrappedStream());
- }
-
- @Override
- public void deleteKey(
- String volumeName, String bucketName, String keyName)
- throws IOException {
- HddsClientUtils.verifyResourceName(volumeName, bucketName);
- Preconditions.checkNotNull(keyName);
- OmKeyArgs keyArgs = new OmKeyArgs.Builder()
- .setVolumeName(volumeName)
- .setBucketName(bucketName)
- .setKeyName(keyName)
- .build();
- ozoneManagerClient.deleteKey(keyArgs);
- }
-
- @Override
- public void renameKey(String volumeName, String bucketName,
- String fromKeyName, String toKeyName) throws IOException {
- HddsClientUtils.verifyResourceName(volumeName, bucketName);
- HddsClientUtils.checkNotNull(fromKeyName, toKeyName);
- OmKeyArgs keyArgs = new OmKeyArgs.Builder()
- .setVolumeName(volumeName)
- .setBucketName(bucketName)
- .setKeyName(fromKeyName)
- .build();
- ozoneManagerClient.renameKey(keyArgs, toKeyName);
- }
-
- @Override
- public List<OzoneKey> listKeys(String volumeName, String bucketName,
- String keyPrefix, String prevKey,
- int maxListResult)
- throws IOException {
- List<OmKeyInfo> keys = ozoneManagerClient.listKeys(
- volumeName, bucketName, prevKey, keyPrefix, maxListResult);
-
- return keys.stream().map(key -> new OzoneKey(
- key.getVolumeName(),
- key.getBucketName(),
- key.getKeyName(),
- key.getDataSize(),
- key.getCreationTime(),
- key.getModificationTime()))
- .collect(Collectors.toList());
- }
-
- @Override
- public OzoneKeyDetails getKeyDetails(
- String volumeName, String bucketName, String keyName)
- throws IOException {
- Preconditions.checkNotNull(volumeName);
- Preconditions.checkNotNull(bucketName);
- Preconditions.checkNotNull(keyName);
- OmKeyArgs keyArgs = new OmKeyArgs.Builder()
- .setVolumeName(volumeName)
- .setBucketName(bucketName)
- .setKeyName(keyName)
- .build();
- OmKeyInfo keyInfo = ozoneManagerClient.lookupKey(keyArgs);
-
- List<OzoneKeyLocation> ozoneKeyLocations = new ArrayList<>();
- keyInfo.getLatestVersionLocations().getBlocksLatestVersionOnly().forEach(
- (a) -> ozoneKeyLocations.add(new OzoneKeyLocation(a.getContainerID(),
- a.getLocalID(), a.getLength(), a.getOffset())));
- return new OzoneKeyDetails(keyInfo.getVolumeName(),
- keyInfo.getBucketName(),
- keyInfo.getKeyName(),
- keyInfo.getDataSize(),
- keyInfo.getCreationTime(),
- keyInfo.getModificationTime(),
- ozoneKeyLocations);
- }
-
- @Override
- public void close() throws IOException {
- IOUtils.cleanupWithLogger(LOG, storageContainerLocationClient);
- IOUtils.cleanupWithLogger(LOG, ozoneManagerClient);
- IOUtils.cleanupWithLogger(LOG, xceiverClientManager);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/package-info.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/package-info.java
deleted file mode 100644
index 0fcc3fc..0000000
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.client.rpc;
-
-/**
- * This package contains Ozone rpc client library classes.
- */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestHddsClientUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestHddsClientUtils.java b/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestHddsClientUtils.java
deleted file mode 100644
index 3aefe8a..0000000
--- a/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestHddsClientUtils.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.client;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.om.OMConfigKeys;
-import org.apache.hadoop.hdds.scm.ScmConfigKeys;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
-import org.junit.rules.Timeout;
-
-import java.net.InetSocketAddress;
-
-import static org.apache.hadoop.hdds.HddsUtils.getScmAddressForClients;
-import static org.apache.hadoop.ozone.OmUtils.getOmAddress;
-import static org.hamcrest.core.Is.is;
-import static org.junit.Assert.assertThat;
-
-/**
- * This test class verifies the parsing of SCM endpoint config settings. The
- * parsing logic is in {@link org.apache.hadoop.hdds.scm.client.HddsClientUtils}.
- */
-public class TestHddsClientUtils {
- @Rule
- public Timeout timeout = new Timeout(300000);
-
- @Rule
- public ExpectedException thrown= ExpectedException.none();
-
- /**
- * Verify client endpoint lookup failure if it is not configured.
- */
- @Test
- public void testMissingScmClientAddress() {
- final Configuration conf = new OzoneConfiguration();
- thrown.expect(IllegalArgumentException.class);
- getScmAddressForClients(conf);
- }
-
- /**
- * Verify that the client endpoint can be correctly parsed from
- * configuration.
- */
- @Test
- public void testGetScmClientAddress() {
- final Configuration conf = new OzoneConfiguration();
-
- // First try a client address with just a host name. Verify it falls
- // back to the default port.
- conf.set(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY, "1.2.3.4");
- InetSocketAddress addr = getScmAddressForClients(conf);
- assertThat(addr.getHostString(), is("1.2.3.4"));
- assertThat(addr.getPort(), is(ScmConfigKeys.OZONE_SCM_CLIENT_PORT_DEFAULT));
-
- // Next try a client address with a host name and port. Verify both
- // are used correctly.
- conf.set(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY, "1.2.3.4:100");
- addr = getScmAddressForClients(conf);
- assertThat(addr.getHostString(), is("1.2.3.4"));
- assertThat(addr.getPort(), is(100));
- }
-
- @Test
- public void testGetOmAddress() {
- final Configuration conf = new OzoneConfiguration();
-
- // First try a client address with just a host name. Verify it falls
- // back to the default port.
- conf.set(OMConfigKeys.OZONE_OM_ADDRESS_KEY, "1.2.3.4");
- InetSocketAddress addr = getOmAddress(conf);
- assertThat(addr.getHostString(), is("1.2.3.4"));
- assertThat(addr.getPort(), is(OMConfigKeys.OZONE_OM_PORT_DEFAULT));
-
- // Next try a client address with just a host name and port. Verify the port
- // is ignored and the default OM port is used.
- conf.set(OMConfigKeys.OZONE_OM_ADDRESS_KEY, "1.2.3.4:100");
- addr = getOmAddress(conf);
- assertThat(addr.getHostString(), is("1.2.3.4"));
- assertThat(addr.getPort(), is(100));
-
- // Assert the we are able to use default configs if no value is specified.
- conf.set(OMConfigKeys.OZONE_OM_ADDRESS_KEY, "");
- addr = getOmAddress(conf);
- assertThat(addr.getHostString(), is("0.0.0.0"));
- assertThat(addr.getPort(), is(OMConfigKeys.OZONE_OM_PORT_DEFAULT));
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/package-info.java b/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/package-info.java
deleted file mode 100644
index be63eab..0000000
--- a/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.client;
-
-/**
- * This package contains test classes for Ozone Client.
- */
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-ozone/common/dev-support/findbugsExcludeFile.xml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/dev-support/findbugsExcludeFile.xml b/hadoop-ozone/common/dev-support/findbugsExcludeFile.xml
deleted file mode 100644
index df58f36..0000000
--- a/hadoop-ozone/common/dev-support/findbugsExcludeFile.xml
+++ /dev/null
@@ -1,22 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<FindBugsFilter>
- <Match>
- <Package name="org.apache.hadoop.ozone.protocol.proto"/>
- </Match>
-</FindBugsFilter>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-ozone/common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/pom.xml b/hadoop-ozone/common/pom.xml
deleted file mode 100644
index 942576b..0000000
--- a/hadoop-ozone/common/pom.xml
+++ /dev/null
@@ -1,115 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License. See accompanying LICENSE file.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
-http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-ozone</artifactId>
- <version>0.3.0-SNAPSHOT</version>
- </parent>
- <artifactId>hadoop-ozone-common</artifactId>
- <version>0.3.0-SNAPSHOT</version>
- <description>Apache Hadoop Ozone Common</description>
- <name>Apache Hadoop Ozone Common</name>
- <packaging>jar</packaging>
-
- <dependencies>
-
- </dependencies>
-
- <build>
- <resources>
- <resource>
- <directory>${basedir}/src/main/resources</directory>
- <excludes>
- <exclude>ozone-version-info.properties</exclude>
- </excludes>
- <filtering>false</filtering>
- </resource>
- <resource>
- <directory>${basedir}/src/main/resources</directory>
- <includes>
- <include>ozone-version-info.properties</include>
- </includes>
- <filtering>true</filtering>
- </resource>
- </resources>
- <plugins>
- <plugin>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-maven-plugins</artifactId>
- <executions>
- <execution>
- <id>version-info</id>
- <phase>generate-resources</phase>
- <goals>
- <goal>version-info</goal>
- </goals>
- <configuration>
- <source>
- <directory>${basedir}/../</directory>
- <includes>
- <include>*/src/main/java/**/*.java</include>
- <include>*/src/main/proto/*.proto</include>
- </includes>
- </source>
- </configuration>
- </execution>
- <execution>
- <id>compile-protoc</id>
- <goals>
- <goal>protoc</goal>
- </goals>
- <configuration>
- <protocVersion>${protobuf.version}</protocVersion>
- <protocCommand>${protoc.path}</protocCommand>
- <imports>
- <param>
- ${basedir}/../../hadoop-common-project/hadoop-common/src/main/proto
- </param>
- <param>
- ${basedir}/../../hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/
- </param>
- <param>
- ${basedir}/../../hadoop-hdfs-project/hadoop-hdfs/src/main/proto/
- </param>
- <param>
- ${basedir}/../../hadoop-hdds/common/src/main/proto/
- </param>
- <param>${basedir}/src/main/proto</param>
- </imports>
- <source>
- <directory>${basedir}/src/main/proto</directory>
- <includes>
- <include>OzoneManagerProtocol.proto</include>
- </includes>
- </source>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>findbugs-maven-plugin</artifactId>
- <configuration>
- <excludeFilterFile>${basedir}/dev-support/findbugsExcludeFile.xml</excludeFilterFile>
- </configuration>
- </plugin>
- </plugins>
- </build>
-</project>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-ozone/common/src/main/bin/ozone
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/bin/ozone b/hadoop-ozone/common/src/main/bin/ozone
deleted file mode 100755
index 4b50771..0000000
--- a/hadoop-ozone/common/src/main/bin/ozone
+++ /dev/null
@@ -1,242 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# The name of the script being executed.
-HADOOP_SHELL_EXECNAME="ozone"
-MYNAME="${BASH_SOURCE-$0}"
-
-## @description build up the hdfs command's usage text.
-## @audience public
-## @stability stable
-## @replaceable no
-function hadoop_usage
-{
- hadoop_add_option "--buildpaths" "attempt to add class files from build tree"
- hadoop_add_option "--daemon (start|status|stop)" "operate on a daemon"
- hadoop_add_option "--hostnames list[,of,host,names]" "hosts to use in worker mode"
- hadoop_add_option "--hosts filename" "list of hosts to use in worker mode"
- hadoop_add_option "--loglevel level" "set the log4j level for this command"
- hadoop_add_option "--workers" "turn on worker mode"
-
- hadoop_add_subcommand "classpath" client "prints the class path needed to get the hadoop jar and the required libraries"
- hadoop_add_subcommand "datanode" daemon "run a HDDS datanode"
- hadoop_add_subcommand "envvars" client "display computed Hadoop environment variables"
- hadoop_add_subcommand "freon" client "runs an ozone data generator"
- hadoop_add_subcommand "fs" client "run a filesystem command on Ozone file system. Equivalent to 'hadoop fs'"
- hadoop_add_subcommand "genconf" client "generate minimally required ozone configs and output to ozone-site.xml in specified path"
- hadoop_add_subcommand "genesis" client "runs a collection of ozone benchmarks to help with tuning."
- hadoop_add_subcommand "getozoneconf" client "get ozone config values from configuration"
- hadoop_add_subcommand "jmxget" admin "get JMX exported values from NameNode or DataNode."
- hadoop_add_subcommand "noz" client "ozone debug tool, convert ozone metadata into relational data"
- hadoop_add_subcommand "om" daemon "Ozone Manager"
- hadoop_add_subcommand "scm" daemon "run the Storage Container Manager service"
- hadoop_add_subcommand "s3g" daemon "run the S3 compatible REST gateway"
- hadoop_add_subcommand "scmcli" client "run the CLI of the Storage Container Manager"
- hadoop_add_subcommand "sh" client "command line interface for object store operations"
- hadoop_add_subcommand "version" client "print the version"
-
- hadoop_generate_usage "${HADOOP_SHELL_EXECNAME}" false
-}
-
-## @description Default command handler for hadoop command
-## @audience public
-## @stability stable
-## @replaceable no
-## @param CLI arguments
-function ozonecmd_case
-{
- subcmd=$1
- shift
-
- case ${subcmd} in
- classpath)
- hadoop_do_classpath_subcommand HADOOP_CLASSNAME "$@"
- ;;
- datanode)
- HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
- HADOOP_CLASSNAME=org.apache.hadoop.ozone.HddsDatanodeService
- OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-datanode"
- ;;
- envvars)
- echo "JAVA_HOME='${JAVA_HOME}'"
- echo "HADOOP_HDFS_HOME='${HADOOP_HDFS_HOME}'"
- echo "HDFS_DIR='${HDFS_DIR}'"
- echo "HDFS_LIB_JARS_DIR='${HDFS_LIB_JARS_DIR}'"
- echo "HADOOP_CONF_DIR='${HADOOP_CONF_DIR}'"
- echo "HADOOP_TOOLS_HOME='${HADOOP_TOOLS_HOME}'"
- echo "HADOOP_TOOLS_DIR='${HADOOP_TOOLS_DIR}'"
- echo "HADOOP_TOOLS_LIB_JARS_DIR='${HADOOP_TOOLS_LIB_JARS_DIR}'"
- if [[ -n "${QATESTMODE}" ]]; then
- echo "MYNAME=${MYNAME}"
- echo "HADOOP_SHELL_EXECNAME=${HADOOP_SHELL_EXECNAME}"
- fi
- exit 0
- ;;
- freon)
- HADOOP_CLASSNAME=org.apache.hadoop.ozone.freon.Freon
- OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-tools"
- ;;
- genesis)
- HADOOP_CLASSNAME=org.apache.hadoop.ozone.genesis.Genesis
- OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-tools"
- ;;
- getozoneconf)
- HADOOP_CLASSNAME=org.apache.hadoop.ozone.freon.OzoneGetConf;
- OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-tools"
- ;;
- om)
- HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
- HADOOP_CLASSNAME=org.apache.hadoop.ozone.om.OzoneManager
- HDFS_OM_OPTS="${HDFS_OM_OPTS} -Dlog4j.configurationFile=${HADOOP_CONF_DIR}/om-audit-log4j2.properties"
- HADOOP_OPTS="${HADOOP_OPTS} ${HDFS_OM_OPTS}"
- OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-ozone-manager"
- ;;
- sh | shell)
- HADOOP_CLASSNAME=org.apache.hadoop.ozone.web.ozShell.Shell
- OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-ozone-manager"
- ;;
- scm)
- HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
- HADOOP_CLASSNAME='org.apache.hadoop.hdds.scm.server.StorageContainerManager'
- hadoop_debug "Appending HDFS_STORAGECONTAINERMANAGER_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${HDFS_STORAGECONTAINERMANAGER_OPTS}"
- OZONE_RUN_ARTIFACT_NAME="hadoop-hdds-server-scm"
- ;;
- s3g)
- HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
- HADOOP_CLASSNAME='org.apache.hadoop.ozone.s3.Gateway'
- OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-s3gateway"
- ;;
- fs)
- HADOOP_CLASSNAME=org.apache.hadoop.fs.FsShell
- OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-tools"
- ;;
- scmcli)
- HADOOP_CLASSNAME=org.apache.hadoop.hdds.scm.cli.SCMCLI
- OZONE_RUN_ARTIFACT_NAME="hadoop-hdds-tools"
- ;;
- version)
- HADOOP_CLASSNAME=org.apache.hadoop.ozone.util.OzoneVersionInfo
- OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-common"
- ;;
- genconf)
- HADOOP_CLASSNAME=org.apache.hadoop.ozone.genconf.GenerateOzoneRequiredConfigurations
- OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-tools"
- ;;
- *)
- HADOOP_CLASSNAME="${subcmd}"
- if ! hadoop_validate_classname "${HADOOP_CLASSNAME}"; then
- hadoop_exit_with_usage 1
- fi
- ;;
- esac
-}
-
-# let's locate libexec...
-if [[ -n "${HADOOP_HOME}" ]]; then
- HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec"
-else
- bin=$(cd -P -- "$(dirname -- "${MYNAME}")" >/dev/null && pwd -P)
- HADOOP_DEFAULT_LIBEXEC_DIR="${bin}/../libexec"
-fi
-
-HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$HADOOP_DEFAULT_LIBEXEC_DIR}"
-# shellcheck disable=SC2034
-HADOOP_NEW_CONFIG=true
-if [[ -f "${HADOOP_LIBEXEC_DIR}/ozone-config.sh" ]]; then
- # shellcheck source=./hadoop-ozone/common/src/main/bin/ozone-config.sh
- . "${HADOOP_LIBEXEC_DIR}/ozone-config.sh"
-else
- echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/ozone-config.sh." 2>&1
- exit 1
-fi
-
-# now that we have support code, let's abs MYNAME so we can use it later
-MYNAME=$(hadoop_abs "${MYNAME}")
-
-if [[ $# = 0 ]]; then
- hadoop_exit_with_usage 1
-fi
-
-HADOOP_SUBCMD=$1
-shift
-
-
-if hadoop_need_reexec ozone "${HADOOP_SUBCMD}"; then
- hadoop_uservar_su ozone "${HADOOP_SUBCMD}" \
- "${MYNAME}" \
- "--reexec" \
- "${HADOOP_USER_PARAMS[@]}"
- exit $?
-fi
-
-hadoop_verify_user_perm "${HADOOP_SHELL_EXECNAME}" "${HADOOP_SUBCMD}"
-
-HADOOP_SUBCMD_ARGS=("$@")
-
-if declare -f ozone_subcommand_"${HADOOP_SUBCMD}" >/dev/null 2>&1; then
- hadoop_debug "Calling dynamically: ozone_subcommand_${HADOOP_SUBCMD} ${HADOOP_SUBCMD_ARGS[*]}"
- "ozone_subcommand_${HADOOP_SUBCMD}" "${HADOOP_SUBCMD_ARGS[@]}"
-else
- ozonecmd_case "${HADOOP_SUBCMD}" "${HADOOP_SUBCMD_ARGS[@]}"
-fi
-
-
-#
-# Setting up classpath based on the generate classpath descriptors
-#
-if [ ! "$OZONE_RUN_ARTIFACT_NAME" ]; then
- echo "ERROR: Ozone components require to set OZONE_RUN_ARTIFACT_NAME to set the classpath"
- exit -1
-fi
-export HDDS_LIB_JARS_DIR="${HADOOP_HDFS_HOME}/share/ozone/lib"
-CLASSPATH_FILE="${HADOOP_HDFS_HOME}/share/ozone/classpath/${OZONE_RUN_ARTIFACT_NAME}.classpath"
-if [ ! "$CLASSPATH_FILE" ]; then
- echo "ERROR: Classpath file descriptor $CLASSPATH_FILE is missing"
- exit -1
-fi
-# shellcheck disable=SC1090,SC2086
-source $CLASSPATH_FILE
-OIFS=$IFS
-IFS=':'
-# shellcheck disable=SC2154
-for jar in $classpath; do
- hadoop_add_classpath "$jar"
-done
-hadoop_add_classpath "${HADOOP_HDFS_HOME}/share/ozone/web"
-
-#We need to add the artifact manually as it's not part the generated classpath desciptor
-ARTIFACT_LIB_DIR="${HADOOP_HDFS_HOME}/share/ozone/lib"
-MAIN_ARTIFACT=$(find "$ARTIFACT_LIB_DIR" -name "${OZONE_RUN_ARTIFACT_NAME}-*.jar")
-if [ ! "$MAIN_ARTIFACT" ]; then
- echo "ERROR: Component jar file $MAIN_ARTIFACT is missing from ${HADOOP_HDFS_HOME}/share/ozone/lib"
-fi
-hadoop_add_classpath "${MAIN_ARTIFACT}"
-IFS=$OIFS
-
-
-hadoop_add_client_opts
-
-if [[ ${HADOOP_WORKER_MODE} = true ]]; then
- hadoop_common_worker_mode_execute "${HADOOP_HDFS_HOME}/bin/ozone" "${HADOOP_USER_PARAMS[@]}"
- exit $?
-fi
-
-hadoop_subcommand_opts "${HADOOP_SHELL_EXECNAME}" "${HADOOP_SUBCMD}"
-
-# everything is in globals at this point, so call the generic handler
-hadoop_generic_java_subcmd_handler
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org