You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ae...@apache.org on 2018/03/15 16:32:54 UTC
[39/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure
Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton,
Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain,
Nanda Kumar.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/client/ContainerOperationClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/client/ContainerOperationClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/client/ContainerOperationClient.java
deleted file mode 100644
index d411a6b..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/client/ContainerOperationClient.java
+++ /dev/null
@@ -1,403 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.scm.client;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.ContainerData;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.ReadContainerResponseProto;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.ObjectStageChangeRequestProto;
-import org.apache.hadoop.scm.XceiverClientManager;
-import org.apache.hadoop.scm.XceiverClientSpi;
-import org.apache.hadoop.scm.container.common.helpers.ContainerInfo;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.scm.protocolPB
- .StorageContainerLocationProtocolClientSideTranslatorPB;
-import org.apache.hadoop.scm.storage.ContainerProtocolCalls;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.EnumSet;
-import java.util.List;
-import java.util.UUID;
-
-import static org.apache.hadoop.ozone.protocol.proto.OzoneProtos.LifeCycleState.ALLOCATED;
-import static org.apache.hadoop.ozone.protocol.proto.OzoneProtos.LifeCycleState.OPEN;
-
-/**
- * This class provides the client-facing APIs of container operations.
- */
-public class ContainerOperationClient implements ScmClient {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(ContainerOperationClient.class);
- private static long containerSizeB = -1;
- private final StorageContainerLocationProtocolClientSideTranslatorPB
- storageContainerLocationClient;
- private final XceiverClientManager xceiverClientManager;
-
- public ContainerOperationClient(
- StorageContainerLocationProtocolClientSideTranslatorPB
- storageContainerLocationClient,
- XceiverClientManager xceiverClientManager) {
- this.storageContainerLocationClient = storageContainerLocationClient;
- this.xceiverClientManager = xceiverClientManager;
- }
-
- /**
- * Return the capacity of containers. The current assumption is that all
- * containers have the same capacity. Therefore one static is sufficient for
- * any container.
- * @return The capacity of one container in number of bytes.
- */
- public static long getContainerSizeB() {
- return containerSizeB;
- }
-
- /**
- * Set the capacity of container. Should be exactly once on system start.
- * @param size Capacity of one container in number of bytes.
- */
- public static void setContainerSizeB(long size) {
- containerSizeB = size;
- }
-
- /**
- * @inheritDoc
- */
- @Override
- public Pipeline createContainer(String containerId, String owner)
- throws IOException {
- XceiverClientSpi client = null;
- try {
- Pipeline pipeline =
- storageContainerLocationClient.allocateContainer(
- xceiverClientManager.getType(),
- xceiverClientManager.getFactor(), containerId, owner);
- client = xceiverClientManager.acquireClient(pipeline);
-
- // Allocated State means that SCM has allocated this pipeline in its
- // namespace. The client needs to create the pipeline on the machines
- // which was choosen by the SCM.
- Preconditions.checkState(pipeline.getLifeCycleState() == ALLOCATED ||
- pipeline.getLifeCycleState() == OPEN, "Unexpected pipeline state");
- if (pipeline.getLifeCycleState() == ALLOCATED) {
- createPipeline(client, pipeline);
- }
- // TODO : Container Client State needs to be updated.
- // TODO : Return ContainerInfo instead of Pipeline
- createContainer(containerId, client, pipeline);
- return pipeline;
- } finally {
- if (client != null) {
- xceiverClientManager.releaseClient(client);
- }
- }
- }
-
- /**
- * Create a container over pipeline specified by the SCM.
- *
- * @param containerId - Container ID
- * @param client - Client to communicate with Datanodes
- * @param pipeline - A pipeline that is already created.
- * @throws IOException
- */
- public void createContainer(String containerId, XceiverClientSpi client,
- Pipeline pipeline) throws IOException {
- String traceID = UUID.randomUUID().toString();
- storageContainerLocationClient.notifyObjectStageChange(
- ObjectStageChangeRequestProto.Type.container,
- containerId,
- ObjectStageChangeRequestProto.Op.create,
- ObjectStageChangeRequestProto.Stage.begin);
- ContainerProtocolCalls.createContainer(client, traceID);
- storageContainerLocationClient.notifyObjectStageChange(
- ObjectStageChangeRequestProto.Type.container,
- containerId,
- ObjectStageChangeRequestProto.Op.create,
- ObjectStageChangeRequestProto.Stage.complete);
-
- // Let us log this info after we let SCM know that we have completed the
- // creation state.
- if (LOG.isDebugEnabled()) {
- LOG.debug("Created container " + containerId
- + " leader:" + pipeline.getLeader()
- + " machines:" + pipeline.getMachines());
- }
- }
-
- /**
- * Creates a pipeline over the machines choosen by the SCM.
- *
- * @param client - Client
- * @param pipeline - pipeline to be createdon Datanodes.
- * @throws IOException
- */
- private void createPipeline(XceiverClientSpi client, Pipeline pipeline)
- throws IOException {
-
- Preconditions.checkNotNull(pipeline.getPipelineName(), "Pipeline " +
- "name cannot be null when client create flag is set.");
-
- // Pipeline creation is a three step process.
- //
- // 1. Notify SCM that this client is doing a create pipeline on
- // datanodes.
- //
- // 2. Talk to Datanodes to create the pipeline.
- //
- // 3. update SCM that pipeline creation was successful.
- storageContainerLocationClient.notifyObjectStageChange(
- ObjectStageChangeRequestProto.Type.pipeline,
- pipeline.getPipelineName(),
- ObjectStageChangeRequestProto.Op.create,
- ObjectStageChangeRequestProto.Stage.begin);
-
- client.createPipeline(pipeline.getPipelineName(),
- pipeline.getMachines());
-
- storageContainerLocationClient.notifyObjectStageChange(
- ObjectStageChangeRequestProto.Type.pipeline,
- pipeline.getPipelineName(),
- ObjectStageChangeRequestProto.Op.create,
- ObjectStageChangeRequestProto.Stage.complete);
-
- // TODO : Should we change the state on the client side ??
- // That makes sense, but it is not needed for the client to work.
- LOG.debug("Pipeline creation successful. Pipeline: {}",
- pipeline.toString());
- }
-
- /**
- * @inheritDoc
- */
- @Override
- public Pipeline createContainer(OzoneProtos.ReplicationType type,
- OzoneProtos.ReplicationFactor factor,
- String containerId, String owner) throws IOException {
- XceiverClientSpi client = null;
- try {
- // allocate container on SCM.
- Pipeline pipeline =
- storageContainerLocationClient.allocateContainer(type, factor,
- containerId, owner);
- client = xceiverClientManager.acquireClient(pipeline);
-
- // Allocated State means that SCM has allocated this pipeline in its
- // namespace. The client needs to create the pipeline on the machines
- // which was choosen by the SCM.
- if (pipeline.getLifeCycleState() == ALLOCATED) {
- createPipeline(client, pipeline);
- }
-
- // TODO : Return ContainerInfo instead of Pipeline
- // connect to pipeline leader and allocate container on leader datanode.
- client = xceiverClientManager.acquireClient(pipeline);
- createContainer(containerId, client, pipeline);
- return pipeline;
- } finally {
- if (client != null) {
- xceiverClientManager.releaseClient(client);
- }
- }
- }
-
- /**
- * Returns a set of Nodes that meet a query criteria.
- *
- * @param nodeStatuses - A set of criteria that we want the node to have.
- * @param queryScope - Query scope - Cluster or pool.
- * @param poolName - if it is pool, a pool name is required.
- * @return A set of nodes that meet the requested criteria.
- * @throws IOException
- */
- @Override
- public OzoneProtos.NodePool queryNode(EnumSet<OzoneProtos.NodeState>
- nodeStatuses, OzoneProtos.QueryScope queryScope, String poolName)
- throws IOException {
- return storageContainerLocationClient.queryNode(nodeStatuses, queryScope,
- poolName);
- }
-
- /**
- * Creates a specified replication pipeline.
- */
- @Override
- public Pipeline createReplicationPipeline(OzoneProtos.ReplicationType type,
- OzoneProtos.ReplicationFactor factor, OzoneProtos.NodePool nodePool)
- throws IOException {
- return storageContainerLocationClient.createReplicationPipeline(type,
- factor, nodePool);
- }
-
- /**
- * Delete the container, this will release any resource it uses.
- * @param pipeline - Pipeline that represents the container.
- * @param force - True to forcibly delete the container.
- * @throws IOException
- */
- @Override
- public void deleteContainer(Pipeline pipeline, boolean force)
- throws IOException {
- XceiverClientSpi client = null;
- try {
- client = xceiverClientManager.acquireClient(pipeline);
- String traceID = UUID.randomUUID().toString();
- ContainerProtocolCalls.deleteContainer(client, force, traceID);
- storageContainerLocationClient
- .deleteContainer(pipeline.getContainerName());
- if (LOG.isDebugEnabled()) {
- LOG.debug("Deleted container {}, leader: {}, machines: {} ",
- pipeline.getContainerName(),
- pipeline.getLeader(),
- pipeline.getMachines());
- }
- } finally {
- if (client != null) {
- xceiverClientManager.releaseClient(client);
- }
- }
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public List<ContainerInfo> listContainer(String startName,
- String prefixName, int count)
- throws IOException {
- return storageContainerLocationClient.listContainer(
- startName, prefixName, count);
- }
-
- /**
- * Get meta data from an existing container.
- *
- * @param pipeline - pipeline that represents the container.
- * @return ContainerInfo - a message of protobuf which has basic info
- * of a container.
- * @throws IOException
- */
- @Override
- public ContainerData readContainer(Pipeline pipeline) throws IOException {
- XceiverClientSpi client = null;
- try {
- client = xceiverClientManager.acquireClient(pipeline);
- String traceID = UUID.randomUUID().toString();
- ReadContainerResponseProto response =
- ContainerProtocolCalls.readContainer(client,
- pipeline.getContainerName(), traceID);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Read container {}, leader: {}, machines: {} ",
- pipeline.getContainerName(),
- pipeline.getLeader(),
- pipeline.getMachines());
- }
- return response.getContainerData();
- } finally {
- if (client != null) {
- xceiverClientManager.releaseClient(client);
- }
- }
- }
-
- /**
- * Given an id, return the pipeline associated with the container.
- * @param containerId - String Container ID
- * @return Pipeline of the existing container, corresponding to the given id.
- * @throws IOException
- */
- @Override
- public Pipeline getContainer(String containerId) throws
- IOException {
- return storageContainerLocationClient.getContainer(containerId);
- }
-
- /**
- * Close a container.
- *
- * @param pipeline the container to be closed.
- * @throws IOException
- */
- @Override
- public void closeContainer(Pipeline pipeline) throws IOException {
- XceiverClientSpi client = null;
- try {
- LOG.debug("Close container {}", pipeline);
- /*
- TODO: two orders here, revisit this later:
- 1. close on SCM first, then on data node
- 2. close on data node first, then on SCM
-
- with 1: if client failed after closing on SCM, then there is a
- container SCM thinks as closed, but is actually open. Then SCM will no
- longer allocate block to it, which is fine. But SCM may later try to
- replicate this "closed" container, which I'm not sure is safe.
-
- with 2: if client failed after close on datanode, then there is a
- container SCM thinks as open, but is actually closed. Then SCM will still
- try to allocate block to it. Which will fail when actually doing the
- write. No more data can be written, but at least the correctness and
- consistency of existing data will maintain.
-
- For now, take the #2 way.
- */
- // Actually close the container on Datanode
- client = xceiverClientManager.acquireClient(pipeline);
- String traceID = UUID.randomUUID().toString();
-
- String containerId = pipeline.getContainerName();
-
- storageContainerLocationClient.notifyObjectStageChange(
- ObjectStageChangeRequestProto.Type.container,
- containerId,
- ObjectStageChangeRequestProto.Op.close,
- ObjectStageChangeRequestProto.Stage.begin);
-
- ContainerProtocolCalls.closeContainer(client, traceID);
- // Notify SCM to close the container
- storageContainerLocationClient.notifyObjectStageChange(
- ObjectStageChangeRequestProto.Type.container,
- containerId,
- ObjectStageChangeRequestProto.Op.close,
- ObjectStageChangeRequestProto.Stage.complete);
- } finally {
- if (client != null) {
- xceiverClientManager.releaseClient(client);
- }
- }
- }
-
- /**
- * Get the the current usage information.
- * @param pipeline - Pipeline
- * @return the size of the given container.
- * @throws IOException
- */
- @Override
- public long getContainerSize(Pipeline pipeline) throws IOException {
- // TODO : Pipeline can be null, handle it correctly.
- long size = getContainerSizeB();
- if (size == -1) {
- throw new IOException("Container size unknown!");
- }
- return size;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/client/ScmClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/client/ScmClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/client/ScmClient.java
deleted file mode 100644
index ef2f1e0..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/client/ScmClient.java
+++ /dev/null
@@ -1,139 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.scm.client;
-
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.ContainerData;
-import org.apache.hadoop.scm.container.common.helpers.ContainerInfo;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-
-import java.io.IOException;
-import java.util.EnumSet;
-import java.util.List;
-
-/**
- * The interface to call into underlying container layer.
- *
- * Written as interface to allow easy testing: implement a mock container layer
- * for standalone testing of CBlock API without actually calling into remote
- * containers. Actual container layer can simply re-implement this.
- *
- * NOTE this is temporarily needed class. When SCM containers are full-fledged,
- * this interface will likely be removed.
- */
-@InterfaceStability.Unstable
-public interface ScmClient {
- /**
- * Creates a Container on SCM and returns the pipeline.
- * @param containerId - String container ID
- * @return Pipeline
- * @throws IOException
- */
- Pipeline createContainer(String containerId, String owner) throws IOException;
-
- /**
- * Gets a container by Name -- Throws if the container does not exist.
- * @param containerId - String Container ID
- * @return Pipeline
- * @throws IOException
- */
- Pipeline getContainer(String containerId) throws IOException;
-
- /**
- * Close a container by name.
- *
- * @param pipeline the container to be closed.
- * @throws IOException
- */
- void closeContainer(Pipeline pipeline) throws IOException;
-
- /**
- * Deletes an existing container.
- * @param pipeline - Pipeline that represents the container.
- * @param force - true to forcibly delete the container.
- * @throws IOException
- */
- void deleteContainer(Pipeline pipeline, boolean force) throws IOException;
-
- /**
- * Lists a range of containers and get their info.
- *
- * @param startName start name, if null, start searching at the head.
- * @param prefixName prefix name, if null, then filter is disabled.
- * @param count count, if count < 0, the max size is unlimited.(
- * Usually the count will be replace with a very big
- * value instead of being unlimited in case the db is very big)
- *
- * @return a list of pipeline.
- * @throws IOException
- */
- List<ContainerInfo> listContainer(String startName, String prefixName,
- int count) throws IOException;
-
- /**
- * Read meta data from an existing container.
- * @param pipeline - Pipeline that represents the container.
- * @return ContainerInfo
- * @throws IOException
- */
- ContainerData readContainer(Pipeline pipeline) throws IOException;
-
-
- /**
- * Gets the container size -- Computed by SCM from Container Reports.
- * @param pipeline - Pipeline
- * @return number of bytes used by this container.
- * @throws IOException
- */
- long getContainerSize(Pipeline pipeline) throws IOException;
-
- /**
- * Creates a Container on SCM and returns the pipeline.
- * @param type - Replication Type.
- * @param replicationFactor - Replication Factor
- * @param containerId - Container ID
- * @return Pipeline
- * @throws IOException - in case of error.
- */
- Pipeline createContainer(OzoneProtos.ReplicationType type,
- OzoneProtos.ReplicationFactor replicationFactor, String containerId,
- String owner) throws IOException;
-
- /**
- * Returns a set of Nodes that meet a query criteria.
- * @param nodeStatuses - A set of criteria that we want the node to have.
- * @param queryScope - Query scope - Cluster or pool.
- * @param poolName - if it is pool, a pool name is required.
- * @return A set of nodes that meet the requested criteria.
- * @throws IOException
- */
- OzoneProtos.NodePool queryNode(EnumSet<OzoneProtos.NodeState> nodeStatuses,
- OzoneProtos.QueryScope queryScope, String poolName) throws IOException;
-
- /**
- * Creates a specified replication pipeline.
- * @param type - Type
- * @param factor - Replication factor
- * @param nodePool - Set of machines.
- * @throws IOException
- */
- Pipeline createReplicationPipeline(OzoneProtos.ReplicationType type,
- OzoneProtos.ReplicationFactor factor, OzoneProtos.NodePool nodePool)
- throws IOException;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/client/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/client/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/client/package-info.java
deleted file mode 100644
index e85e542..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/client/package-info.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.scm.client;
-
-/**
- * This package contains classes for the client of the storage container
- * protocol.
- */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/AllocatedBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/AllocatedBlock.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/AllocatedBlock.java
deleted file mode 100644
index f51336f..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/AllocatedBlock.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.scm.container.common.helpers;
-
-/**
- * Allocated block wraps the result returned from SCM#allocateBlock which
- * contains a Pipeline and the key.
- */
-public final class AllocatedBlock {
- private Pipeline pipeline;
- private String key;
- // Indicates whether the client should create container before writing block.
- private boolean shouldCreateContainer;
-
- /**
- * Builder for AllocatedBlock.
- */
- public static class Builder {
- private Pipeline pipeline;
- private String key;
- private boolean shouldCreateContainer;
-
- public Builder setPipeline(Pipeline p) {
- this.pipeline = p;
- return this;
- }
-
- public Builder setKey(String k) {
- this.key = k;
- return this;
- }
-
- public Builder setShouldCreateContainer(boolean shouldCreate) {
- this.shouldCreateContainer = shouldCreate;
- return this;
- }
-
- public AllocatedBlock build() {
- return new AllocatedBlock(pipeline, key, shouldCreateContainer);
- }
- }
-
- private AllocatedBlock(Pipeline pipeline, String key,
- boolean shouldCreateContainer) {
- this.pipeline = pipeline;
- this.key = key;
- this.shouldCreateContainer = shouldCreateContainer;
- }
-
- public Pipeline getPipeline() {
- return pipeline;
- }
-
- public String getKey() {
- return key;
- }
-
- public boolean getCreateContainer() {
- return shouldCreateContainer;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/ContainerInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/ContainerInfo.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/ContainerInfo.java
deleted file mode 100644
index 11d4438..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/ContainerInfo.java
+++ /dev/null
@@ -1,333 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.scm.container.common.helpers;
-
-import com.google.common.base.Preconditions;
-import org.apache.commons.lang3.builder.EqualsBuilder;
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-import org.apache.hadoop.ozone.scm.container.ContainerStates.ContainerID;
-import org.apache.hadoop.util.Time;
-
-import java.util.Comparator;
-
-/**
- * Class wraps ozone container info.
- */
-public class ContainerInfo
- implements Comparator<ContainerInfo>, Comparable<ContainerInfo> {
- private OzoneProtos.LifeCycleState state;
- private Pipeline pipeline;
- // Bytes allocated by SCM for clients.
- private long allocatedBytes;
- // Actual container usage, updated through heartbeat.
- private long usedBytes;
- private long numberOfKeys;
- private long lastUsed;
- // The wall-clock ms since the epoch at which the current state enters.
- private long stateEnterTime;
- private String owner;
- private String containerName;
- private long containerID;
- ContainerInfo(
- long containerID,
- final String containerName,
- OzoneProtos.LifeCycleState state,
- Pipeline pipeline,
- long allocatedBytes,
- long usedBytes,
- long numberOfKeys,
- long stateEnterTime,
- String owner) {
- this.containerID = containerID;
- this.containerName = containerName;
- this.pipeline = pipeline;
- this.allocatedBytes = allocatedBytes;
- this.usedBytes = usedBytes;
- this.numberOfKeys = numberOfKeys;
- this.lastUsed = Time.monotonicNow();
- this.state = state;
- this.stateEnterTime = stateEnterTime;
- this.owner = owner;
- }
-
- /**
- * Needed for serialization findbugs.
- */
- public ContainerInfo() {
- }
-
- public static ContainerInfo fromProtobuf(OzoneProtos.SCMContainerInfo info) {
- ContainerInfo.Builder builder = new ContainerInfo.Builder();
- builder.setPipeline(Pipeline.getFromProtoBuf(info.getPipeline()));
- builder.setAllocatedBytes(info.getAllocatedBytes());
- builder.setUsedBytes(info.getUsedBytes());
- builder.setNumberOfKeys(info.getNumberOfKeys());
- builder.setState(info.getState());
- builder.setStateEnterTime(info.getStateEnterTime());
- builder.setOwner(info.getOwner());
- builder.setContainerName(info.getContainerName());
- builder.setContainerID(info.getContainerID());
- return builder.build();
- }
-
- public long getContainerID() {
- return containerID;
- }
-
- public String getContainerName() {
- return containerName;
- }
-
- public OzoneProtos.LifeCycleState getState() {
- return state;
- }
-
- public void setState(OzoneProtos.LifeCycleState state) {
- this.state = state;
- }
-
- public long getStateEnterTime() {
- return stateEnterTime;
- }
-
- public Pipeline getPipeline() {
- return pipeline;
- }
-
- public long getAllocatedBytes() {
- return allocatedBytes;
- }
-
- /**
- * Set Allocated bytes.
- *
- * @param size - newly allocated bytes -- negative size is case of deletes
- * can be used.
- */
- public void updateAllocatedBytes(long size) {
- this.allocatedBytes += size;
- }
-
- public long getUsedBytes() {
- return usedBytes;
- }
-
- public long getNumberOfKeys() {
- return numberOfKeys;
- }
-
- public ContainerID containerID() {
- return new ContainerID(getContainerID());
- }
-
- /**
- * Gets the last used time from SCM's perspective.
- *
- * @return time in milliseconds.
- */
- public long getLastUsed() {
- return lastUsed;
- }
-
- public void updateLastUsedTime() {
- lastUsed = Time.monotonicNow();
- }
-
- public void allocate(long size) {
- // should we also have total container size in ContainerInfo
- // and check before allocating?
- allocatedBytes += size;
- }
-
- public OzoneProtos.SCMContainerInfo getProtobuf() {
- OzoneProtos.SCMContainerInfo.Builder builder =
- OzoneProtos.SCMContainerInfo.newBuilder();
- builder.setPipeline(getPipeline().getProtobufMessage());
- builder.setAllocatedBytes(getAllocatedBytes());
- builder.setUsedBytes(getUsedBytes());
- builder.setNumberOfKeys(getNumberOfKeys());
- builder.setState(state);
- builder.setStateEnterTime(stateEnterTime);
- builder.setContainerID(getContainerID());
-
- if (getOwner() != null) {
- builder.setOwner(getOwner());
- }
- builder.setContainerName(getContainerName());
- return builder.build();
- }
-
- public String getOwner() {
- return owner;
- }
-
- public void setOwner(String owner) {
- this.owner = owner;
- }
-
- @Override
- public String toString() {
- return "ContainerInfo{"
- + "state=" + state
- + ", pipeline=" + pipeline
- + ", stateEnterTime=" + stateEnterTime
- + ", owner=" + owner
- + ", containerName='" + containerName
- + '}';
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
-
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
-
- ContainerInfo that = (ContainerInfo) o;
-
- return new EqualsBuilder()
- .append(pipeline.getContainerName(), that.pipeline.getContainerName())
-
- // TODO : Fix this later. If we add these factors some tests fail.
- // So Commenting this to continue and will enforce this with
- // Changes in pipeline where we remove Container Name to
- // SCMContainerinfo from Pipeline.
- // .append(pipeline.getFactor(), that.pipeline.getFactor())
- // .append(pipeline.getType(), that.pipeline.getType())
- .append(owner, that.owner)
- .isEquals();
- }
-
- @Override
- public int hashCode() {
- return new HashCodeBuilder(11, 811)
- .append(pipeline.getContainerName())
- .append(pipeline.getFactor())
- .append(pipeline.getType())
- .append(owner)
- .toHashCode();
- }
-
- /**
- * Compares its two arguments for order. Returns a negative integer, zero, or
- * a positive integer as the first argument is less than, equal to, or greater
- * than the second.<p>
- *
- * @param o1 the first object to be compared.
- * @param o2 the second object to be compared.
- * @return a negative integer, zero, or a positive integer as the first
- * argument is less than, equal to, or greater than the second.
- * @throws NullPointerException if an argument is null and this comparator
- * does not permit null arguments
- * @throws ClassCastException if the arguments' types prevent them from
- * being compared by this comparator.
- */
- @Override
- public int compare(ContainerInfo o1, ContainerInfo o2) {
- return Long.compare(o1.getLastUsed(), o2.getLastUsed());
- }
-
- /**
- * Compares this object with the specified object for order. Returns a
- * negative integer, zero, or a positive integer as this object is less than,
- * equal to, or greater than the specified object.
- *
- * @param o the object to be compared.
- * @return a negative integer, zero, or a positive integer as this object is
- * less than, equal to, or greater than the specified object.
- * @throws NullPointerException if the specified object is null
- * @throws ClassCastException if the specified object's type prevents it
- * from being compared to this object.
- */
- @Override
- public int compareTo(ContainerInfo o) {
- return this.compare(this, o);
- }
-
- /**
- * Builder class for ContainerInfo.
- */
- public static class Builder {
- private OzoneProtos.LifeCycleState state;
- private Pipeline pipeline;
- private long allocated;
- private long used;
- private long keys;
- private long stateEnterTime;
- private String owner;
- private String containerName;
- private long containerID;
-
- public Builder setContainerID(long id) {
- Preconditions.checkState(id >= 0);
- this.containerID = id;
- return this;
- }
-
- public Builder setState(OzoneProtos.LifeCycleState lifeCycleState) {
- this.state = lifeCycleState;
- return this;
- }
-
- public Builder setPipeline(Pipeline containerPipeline) {
- this.pipeline = containerPipeline;
- return this;
- }
-
- public Builder setAllocatedBytes(long bytesAllocated) {
- this.allocated = bytesAllocated;
- return this;
- }
-
- public Builder setUsedBytes(long bytesUsed) {
- this.used = bytesUsed;
- return this;
- }
-
- public Builder setNumberOfKeys(long keyCount) {
- this.keys = keyCount;
- return this;
- }
-
- public Builder setStateEnterTime(long time) {
- this.stateEnterTime = time;
- return this;
- }
-
- public Builder setOwner(String containerOwner) {
- this.owner = containerOwner;
- return this;
- }
-
- public Builder setContainerName(String container) {
- this.containerName = container;
- return this;
- }
-
- public ContainerInfo build() {
- return new
- ContainerInfo(containerID, containerName, state, pipeline,
- allocated, used, keys, stateEnterTime, owner);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/DeleteBlockResult.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/DeleteBlockResult.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/DeleteBlockResult.java
deleted file mode 100644
index 5dbb853..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/DeleteBlockResult.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.scm.container.common.helpers;
-
-
-import static org.apache.hadoop.ozone.protocol.proto
- .ScmBlockLocationProtocolProtos.DeleteScmBlockResult;
-
-/**
- * Class wraps storage container manager block deletion results.
- */
-public class DeleteBlockResult {
- private String key;
- private DeleteScmBlockResult.Result result;
-
- public DeleteBlockResult(final String key,
- final DeleteScmBlockResult.Result result) {
- this.key = key;
- this.result = result;
- }
-
- /**
- * Get key deleted.
- * @return key name.
- */
- public String getKey() {
- return key;
- }
-
- /**
- * Get key deletion result.
- * @return key deletion result.
- */
- public DeleteScmBlockResult.Result getResult() {
- return result;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/Pipeline.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/Pipeline.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/Pipeline.java
deleted file mode 100644
index 9d47646..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/Pipeline.java
+++ /dev/null
@@ -1,253 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.scm.container.common.helpers;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonFilter;
-import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.fasterxml.jackson.annotation.PropertyAccessor;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.ObjectWriter;
-import com.fasterxml.jackson.databind.ser.FilterProvider;
-import com.fasterxml.jackson.databind.ser.impl.SimpleBeanPropertyFilter;
-import com.fasterxml.jackson.databind.ser.impl.SimpleFilterProvider;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-/**
- * A pipeline represents the group of machines over which a container lives.
- */
-public class Pipeline {
- static final String PIPELINE_INFO = "PIPELINE_INFO_FILTER";
- private static final ObjectWriter WRITER;
-
- static {
- ObjectMapper mapper = new ObjectMapper();
- String[] ignorableFieldNames = {"data"};
- FilterProvider filters = new SimpleFilterProvider()
- .addFilter(PIPELINE_INFO, SimpleBeanPropertyFilter
- .serializeAllExcept(ignorableFieldNames));
- mapper.setVisibility(PropertyAccessor.FIELD,
- JsonAutoDetect.Visibility.ANY);
- mapper.addMixIn(Object.class, MixIn.class);
-
- WRITER = mapper.writer(filters);
- }
-
- private String containerName;
- private PipelineChannel pipelineChannel;
- /**
- * Allows you to maintain private data on pipelines. This is not serialized
- * via protobuf, just allows us to maintain some private data.
- */
- @JsonIgnore
- private byte[] data;
- /**
- * Constructs a new pipeline data structure.
- *
- * @param containerName - Container
- * @param pipelineChannel - transport information for this container
- */
- public Pipeline(String containerName, PipelineChannel pipelineChannel) {
- this.containerName = containerName;
- this.pipelineChannel = pipelineChannel;
- data = null;
- }
-
- /**
- * Gets pipeline object from protobuf.
- *
- * @param pipeline - ProtoBuf definition for the pipeline.
- * @return Pipeline Object
- */
- public static Pipeline getFromProtoBuf(OzoneProtos.Pipeline pipeline) {
- Preconditions.checkNotNull(pipeline);
- PipelineChannel pipelineChannel =
- PipelineChannel.getFromProtoBuf(pipeline.getPipelineChannel());
- return new Pipeline(pipeline.getContainerName(), pipelineChannel);
- }
-
- public OzoneProtos.ReplicationFactor getFactor() {
- return pipelineChannel.getFactor();
- }
-
- /**
- * Returns the first machine in the set of datanodes.
- *
- * @return First Machine.
- */
- @JsonIgnore
- public DatanodeID getLeader() {
- return pipelineChannel.getDatanodes().get(pipelineChannel.getLeaderID());
- }
-
- /**
- * Returns the leader host.
- *
- * @return First Machine.
- */
- public String getLeaderHost() {
- return pipelineChannel.getDatanodes()
- .get(pipelineChannel.getLeaderID()).getHostName();
- }
-
- /**
- * Returns all machines that make up this pipeline.
- *
- * @return List of Machines.
- */
- @JsonIgnore
- public List<DatanodeID> getMachines() {
- return new ArrayList<>(pipelineChannel.getDatanodes().values());
- }
-
- /**
- * Returns all machines that make up this pipeline.
- *
- * @return List of Machines.
- */
- public List<String> getDatanodeHosts() {
- List<String> dataHosts = new ArrayList<>();
- for (DatanodeID id : pipelineChannel.getDatanodes().values()) {
- dataHosts.add(id.getHostName());
- }
- return dataHosts;
- }
-
- /**
- * Return a Protobuf Pipeline message from pipeline.
- *
- * @return Protobuf message
- */
- @JsonIgnore
- public OzoneProtos.Pipeline getProtobufMessage() {
- OzoneProtos.Pipeline.Builder builder =
- OzoneProtos.Pipeline.newBuilder();
- builder.setContainerName(this.containerName);
- builder.setPipelineChannel(this.pipelineChannel.getProtobufMessage());
- return builder.build();
- }
-
- /**
- * Returns containerName if available.
- *
- * @return String.
- */
- public String getContainerName() {
- return containerName;
- }
-
- /**
- * Returns private data that is set on this pipeline.
- *
- * @return blob, the user can interpret it any way they like.
- */
- public byte[] getData() {
- if (this.data != null) {
- return Arrays.copyOf(this.data, this.data.length);
- } else {
- return null;
- }
- }
-
- @VisibleForTesting
- public PipelineChannel getPipelineChannel() {
- return pipelineChannel;
- }
-
- /**
- * Set private data on pipeline.
- *
- * @param data -- private data.
- */
- public void setData(byte[] data) {
- if (data != null) {
- this.data = Arrays.copyOf(data, data.length);
- }
- }
-
- /**
- * Gets the State of the pipeline.
- *
- * @return - LifeCycleStates.
- */
- public OzoneProtos.LifeCycleState getLifeCycleState() {
- return pipelineChannel.getLifeCycleState();
- }
-
- /**
- * Gets the pipeline Name.
- *
- * @return - Name of the pipeline
- */
- public String getPipelineName() {
- return pipelineChannel.getName();
- }
-
- /**
- * Returns the type.
- *
- * @return type - Standalone, Ratis, Chained.
- */
- public OzoneProtos.ReplicationType getType() {
- return pipelineChannel.getType();
- }
-
- @Override
- public String toString() {
- final StringBuilder b = new StringBuilder(getClass().getSimpleName())
- .append("[");
- pipelineChannel.getDatanodes().keySet().stream()
- .forEach(id -> b.
- append(id.endsWith(pipelineChannel.getLeaderID()) ? "*" + id : id));
- b.append("] container:").append(containerName);
- b.append(" name:").append(getPipelineName());
- if (getType() != null) {
- b.append(" type:").append(getType().toString());
- }
- if (getFactor() != null) {
- b.append(" factor:").append(getFactor().toString());
- }
- if (getLifeCycleState() != null) {
- b.append(" State:").append(getLifeCycleState().toString());
- }
- return b.toString();
- }
-
- /**
- * Returns a JSON string of this object.
- *
- * @return String - json string
- * @throws IOException
- */
- public String toJsonString() throws IOException {
- return WRITER.writeValueAsString(this);
- }
-
- @JsonFilter(PIPELINE_INFO)
- class MixIn {
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/PipelineChannel.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/PipelineChannel.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/PipelineChannel.java
deleted file mode 100644
index 0af0534..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/PipelineChannel.java
+++ /dev/null
@@ -1,121 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.scm.container.common.helpers;
-
-import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.LifeCycleState;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.ReplicationType;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.ReplicationFactor;
-
-import java.util.Map;
-import java.util.TreeMap;
-
-/**
- * PipelineChannel information for a {@link Pipeline}.
- */
-public class PipelineChannel {
- @JsonIgnore
- private String leaderID;
- @JsonIgnore
- private Map<String, DatanodeID> datanodes;
- private LifeCycleState lifeCycleState;
- private ReplicationType type;
- private ReplicationFactor factor;
- private String name;
-
- public PipelineChannel(String leaderID, LifeCycleState lifeCycleState,
- ReplicationType replicationType, ReplicationFactor replicationFactor,
- String name) {
- this.leaderID = leaderID;
- this.lifeCycleState = lifeCycleState;
- this.type = replicationType;
- this.factor = replicationFactor;
- this.name = name;
- datanodes = new TreeMap<>();
- }
-
- public String getLeaderID() {
- return leaderID;
- }
-
- public Map<String, DatanodeID> getDatanodes() {
- return datanodes;
- }
-
- public LifeCycleState getLifeCycleState() {
- return lifeCycleState;
- }
-
- public ReplicationType getType() {
- return type;
- }
-
- public ReplicationFactor getFactor() {
- return factor;
- }
-
- public String getName() {
- return name;
- }
-
- public void addMember(DatanodeID dataNodeId) {
- datanodes.put(dataNodeId.getDatanodeUuid(), dataNodeId);
- }
-
- @JsonIgnore
- public OzoneProtos.PipelineChannel getProtobufMessage() {
- OzoneProtos.PipelineChannel.Builder builder =
- OzoneProtos.PipelineChannel.newBuilder();
- for (DatanodeID datanode : datanodes.values()) {
- builder.addMembers(datanode.getProtoBufMessage());
- }
- builder.setLeaderID(leaderID);
-
- if (this.getLifeCycleState() != null) {
- builder.setState(this.getLifeCycleState());
- }
- if (this.getType() != null) {
- builder.setType(this.getType());
- }
-
- if (this.getFactor() != null) {
- builder.setFactor(this.getFactor());
- }
- return builder.build();
- }
-
- public static PipelineChannel getFromProtoBuf(
- OzoneProtos.PipelineChannel transportProtos) {
- Preconditions.checkNotNull(transportProtos);
- PipelineChannel pipelineChannel =
- new PipelineChannel(transportProtos.getLeaderID(),
- transportProtos.getState(),
- transportProtos.getType(),
- transportProtos.getFactor(),
- transportProtos.getName());
-
- for (HdfsProtos.DatanodeIDProto dataID : transportProtos.getMembersList()) {
- pipelineChannel.addMember(DatanodeID.getFromProtoBuf(dataID));
- }
- return pipelineChannel;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/StorageContainerException.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/StorageContainerException.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/StorageContainerException.java
deleted file mode 100644
index 549fced..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/StorageContainerException.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.scm.container.common.helpers;
-
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos;
-
-import java.io.IOException;
-
-/**
- * Exceptions thrown from the Storage Container.
- */
-public class StorageContainerException extends IOException {
- private ContainerProtos.Result result;
-
- /**
- * Constructs an {@code IOException} with {@code null}
- * as its error detail message.
- */
- public StorageContainerException(ContainerProtos.Result result) {
- this.result = result;
- }
-
- /**
- * Constructs an {@code IOException} with the specified detail message.
- *
- * @param message The detail message (which is saved for later retrieval by
- * the {@link #getMessage()} method)
- * @param result - The result code
- */
- public StorageContainerException(String message,
- ContainerProtos.Result result) {
- super(message);
- this.result = result;
- }
-
- /**
- * Constructs an {@code IOException} with the specified detail message
- * and cause.
- * <p>
- * <p> Note that the detail message associated with {@code cause} is
- * <i>not</i> automatically incorporated into this exception's detail
- * message.
- *
- * @param message The detail message (which is saved for later retrieval by
- * the {@link #getMessage()} method)
- *
- * @param cause The cause (which is saved for later retrieval by the {@link
- * #getCause()} method). (A null value is permitted, and indicates that the
- * cause is nonexistent or unknown.)
- *
- * @param result - The result code
- * @since 1.6
- */
- public StorageContainerException(String message, Throwable cause,
- ContainerProtos.Result result) {
- super(message, cause);
- this.result = result;
- }
-
- /**
- * Constructs an {@code IOException} with the specified cause and a
- * detail message of {@code (cause==null ? null : cause.toString())}
- * (which typically contains the class and detail message of {@code cause}).
- * This constructor is useful for IO exceptions that are little more
- * than wrappers for other throwables.
- *
- * @param cause The cause (which is saved for later retrieval by the {@link
- * #getCause()} method). (A null value is permitted, and indicates that the
- * cause is nonexistent or unknown.)
- * @param result - The result code
- * @since 1.6
- */
- public StorageContainerException(Throwable cause, ContainerProtos.Result
- result) {
- super(cause);
- this.result = result;
- }
-
- /**
- * Returns Result.
- *
- * @return Result.
- */
- public ContainerProtos.Result getResult() {
- return result;
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/package-info.java
deleted file mode 100644
index 3fa9663..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.scm.container.common.helpers;
-/**
- Contains protocol buffer helper classes and utilites used in
- impl.
- **/
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/package-info.java
deleted file mode 100644
index ad24f98..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/package-info.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.scm;
-
-/**
- * This package contains classes for the client of the storage container
- * protocol.
- */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/LocatedContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/LocatedContainer.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/LocatedContainer.java
deleted file mode 100644
index 469dab5..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/LocatedContainer.java
+++ /dev/null
@@ -1,127 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.scm.protocol;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
-
-import java.util.Set;
-
-/**
- * Holds the nodes that currently host the container for an object key hash.
- */
-@InterfaceAudience.Private
-public final class LocatedContainer {
- private final String key;
- private final String matchedKeyPrefix;
- private final String containerName;
- private final Set<DatanodeInfo> locations;
- private final DatanodeInfo leader;
-
- /**
- * Creates a LocatedContainer.
- *
- * @param key object key
- * @param matchedKeyPrefix prefix of key that was used to find the location
- * @param containerName container name
- * @param locations nodes that currently host the container
- * @param leader node that currently acts as pipeline leader
- */
- public LocatedContainer(String key, String matchedKeyPrefix,
- String containerName, Set<DatanodeInfo> locations, DatanodeInfo leader) {
- this.key = key;
- this.matchedKeyPrefix = matchedKeyPrefix;
- this.containerName = containerName;
- this.locations = locations;
- this.leader = leader;
- }
-
- /**
- * Returns the container name.
- *
- * @return container name
- */
- public String getContainerName() {
- return this.containerName;
- }
-
- /**
- * Returns the object key.
- *
- * @return object key
- */
- public String getKey() {
- return this.key;
- }
-
- /**
- * Returns the node that currently acts as pipeline leader.
- *
- * @return node that currently acts as pipeline leader
- */
- public DatanodeInfo getLeader() {
- return this.leader;
- }
-
- /**
- * Returns the nodes that currently host the container.
- *
- * @return Set<DatanodeInfo> nodes that currently host the container
- */
- public Set<DatanodeInfo> getLocations() {
- return this.locations;
- }
-
- /**
- * Returns the prefix of the key that was used to find the location.
- *
- * @return prefix of the key that was used to find the location
- */
- public String getMatchedKeyPrefix() {
- return this.matchedKeyPrefix;
- }
-
- @Override
- public boolean equals(Object otherObj) {
- if (otherObj == null) {
- return false;
- }
- if (!(otherObj instanceof LocatedContainer)) {
- return false;
- }
- LocatedContainer other = (LocatedContainer)otherObj;
- return this.key == null ? other.key == null : this.key.equals(other.key);
- }
-
- @Override
- public int hashCode() {
- return key.hashCode();
- }
-
- @Override
- public String toString() {
- return getClass().getSimpleName()
- + "{key=" + key
- + "; matchedKeyPrefix=" + matchedKeyPrefix
- + "; containerName=" + containerName
- + "; locations=" + locations
- + "; leader=" + leader
- + "}";
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/ScmBlockLocationProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/ScmBlockLocationProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/ScmBlockLocationProtocol.java
deleted file mode 100644
index 9728554..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/ScmBlockLocationProtocol.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.scm.protocol;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.hadoop.ozone.common.DeleteBlockGroupResult;
-import org.apache.hadoop.ozone.common.BlockGroup;
-import org.apache.hadoop.scm.container.common.helpers.AllocatedBlock;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.ReplicationType;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.ReplicationFactor;
-import org.apache.hadoop.scm.ScmInfo;
-
-/**
- * ScmBlockLocationProtocol is used by an HDFS node to find the set of nodes
- * to read/write a block.
- */
-public interface ScmBlockLocationProtocol {
-
- /**
- * Find the set of nodes to read/write a block, as
- * identified by the block key. This method supports batch lookup by
- * passing multiple keys.
- *
- * @param keys batch of block keys to find
- * @return allocated blocks for each block key
- * @throws IOException if there is any failure
- */
- Set<AllocatedBlock> getBlockLocations(Set<String> keys) throws IOException;
-
- /**
- * Asks SCM where a block should be allocated. SCM responds with the
- * set of datanodes that should be used creating this block.
- * @param size - size of the block.
- * @return allocated block accessing info (key, pipeline).
- * @throws IOException
- */
- AllocatedBlock allocateBlock(long size, ReplicationType type,
- ReplicationFactor factor, String owner) throws IOException;
-
- /**
- * Delete blocks for a set of object keys.
- *
- * @param keyBlocksInfoList Map of object key and its blocks.
- * @return list of block deletion results.
- * @throws IOException if there is any failure.
- */
- List<DeleteBlockGroupResult>
- deleteKeyBlocks(List<BlockGroup> keyBlocksInfoList) throws IOException;
-
- /**
- * Gets the Clusterid and SCM Id from SCM.
- */
- ScmInfo getScmInfo() throws IOException;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/ScmLocatedBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/ScmLocatedBlock.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/ScmLocatedBlock.java
deleted file mode 100644
index 4e4b3d6..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/ScmLocatedBlock.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.scm.protocol;
-
-import java.util.List;
-import java.util.stream.Collectors;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
-
-/**
- * Holds the nodes that currently host the block for a block key.
- */
-@InterfaceAudience.Private
-public final class ScmLocatedBlock {
- private final String key;
- private final List<DatanodeInfo> locations;
- private final DatanodeInfo leader;
-
- /**
- * Creates a ScmLocatedBlock.
- *
- * @param key object key
- * @param locations nodes that currently host the block
- * @param leader node that currently acts as pipeline leader
- */
- public ScmLocatedBlock(final String key, final List<DatanodeInfo> locations,
- final DatanodeInfo leader) {
- this.key = key;
- this.locations = locations;
- this.leader = leader;
- }
-
- /**
- * Returns the object key.
- *
- * @return object key
- */
- public String getKey() {
- return this.key;
- }
-
- /**
- * Returns the node that currently acts as pipeline leader.
- *
- * @return node that currently acts as pipeline leader
- */
- public DatanodeInfo getLeader() {
- return this.leader;
- }
-
- /**
- * Returns the nodes that currently host the block.
- *
- * @return List<DatanodeInfo> nodes that currently host the block
- */
- public List<DatanodeInfo> getLocations() {
- return this.locations;
- }
-
- @Override
- public boolean equals(Object otherObj) {
- if (otherObj == null) {
- return false;
- }
- if (!(otherObj instanceof ScmLocatedBlock)) {
- return false;
- }
- ScmLocatedBlock other = (ScmLocatedBlock)otherObj;
- return this.key == null ? other.key == null : this.key.equals(other.key);
- }
-
- @Override
- public int hashCode() {
- return key.hashCode();
- }
-
- @Override
- public String toString() {
- return getClass().getSimpleName() + "{key=" + key + "; locations="
- + locations.stream().map(loc -> loc.toString()).collect(Collectors
- .joining(",")) + "; leader=" + leader + "}";
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/StorageContainerLocationProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/StorageContainerLocationProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/StorageContainerLocationProtocol.java
deleted file mode 100644
index ad78bd8..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/StorageContainerLocationProtocol.java
+++ /dev/null
@@ -1,123 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.scm.protocol;
-
-import java.io.IOException;
-import java.util.EnumSet;
-import java.util.List;
-
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.ObjectStageChangeRequestProto;
-import org.apache.hadoop.scm.ScmInfo;
-import org.apache.hadoop.scm.container.common.helpers.ContainerInfo;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-
-/**
- * ContainerLocationProtocol is used by an HDFS node to find the set of nodes
- * that currently host a container.
- */
-public interface StorageContainerLocationProtocol {
- /**
- * Asks SCM where a container should be allocated. SCM responds with the
- * set of datanodes that should be used creating this container.
- *
- */
- Pipeline allocateContainer(OzoneProtos.ReplicationType replicationType,
- OzoneProtos.ReplicationFactor factor, String containerName, String owner)
- throws IOException;
-
- /**
- * Ask SCM the location of the container. SCM responds with a group of
- * nodes where this container and its replicas are located.
- *
- * @param containerName - Name of the container.
- * @return Pipeline - the pipeline where container locates.
- * @throws IOException
- */
- Pipeline getContainer(String containerName) throws IOException;
-
- /**
- * Ask SCM a list of containers with a range of container names
- * and the limit of count.
- * Search container names between start name(exclusive), and
- * use prefix name to filter the result. the max size of the
- * searching range cannot exceed the value of count.
- *
- * @param startName start name, if null, start searching at the head.
- * @param prefixName prefix name, if null, then filter is disabled.
- * @param count count, if count < 0, the max size is unlimited.(
- * Usually the count will be replace with a very big
- * value instead of being unlimited in case the db is very big)
- *
- * @return a list of container.
- * @throws IOException
- */
- List<ContainerInfo> listContainer(String startName, String prefixName,
- int count) throws IOException;
-
- /**
- * Deletes a container in SCM.
- *
- * @param containerName
- * @throws IOException
- * if failed to delete the container mapping from db store
- * or container doesn't exist.
- */
- void deleteContainer(String containerName) throws IOException;
-
- /**
- * Queries a list of Node Statuses.
- * @param nodeStatuses
- * @return List of Datanodes.
- */
- OzoneProtos.NodePool queryNode(EnumSet<OzoneProtos.NodeState> nodeStatuses,
- OzoneProtos.QueryScope queryScope, String poolName) throws IOException;
-
- /**
- * Notify from client when begin or finish creating objects like pipeline
- * or containers on datanodes.
- * Container will be in Operational state after that.
- * @param type object type
- * @param name object name
- * @param op operation type (e.g., create, close, delete)
- * @param stage creation stage
- */
- void notifyObjectStageChange(
- ObjectStageChangeRequestProto.Type type, String name,
- ObjectStageChangeRequestProto.Op op,
- ObjectStageChangeRequestProto.Stage stage) throws IOException;
-
- /**
- * Creates a replication pipeline of a specified type.
- * @param type - replication type
- * @param factor - factor 1 or 3
- * @param nodePool - optional machine list to build a pipeline.
- * @throws IOException
- */
- Pipeline createReplicationPipeline(OzoneProtos.ReplicationType type,
- OzoneProtos.ReplicationFactor factor, OzoneProtos.NodePool nodePool)
- throws IOException;
-
- /**
- * Returns information about SCM.
- *
- * @return {@link ScmInfo}
- * @throws IOException
- */
- ScmInfo getScmInfo() throws IOException;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/package-info.java
deleted file mode 100644
index 274f859..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/package-info.java
+++ /dev/null
@@ -1,19 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.scm.protocol;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocolPB/ScmBlockLocationProtocolClientSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocolPB/ScmBlockLocationProtocolClientSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocolPB/ScmBlockLocationProtocolClientSideTranslatorPB.java
deleted file mode 100644
index 02d973a..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocolPB/ScmBlockLocationProtocolClientSideTranslatorPB.java
+++ /dev/null
@@ -1,207 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.scm.protocolPB;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Sets;
-import com.google.protobuf.RpcController;
-import com.google.protobuf.ServiceException;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.ipc.ProtobufHelper;
-import org.apache.hadoop.ipc.ProtocolTranslator;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.ozone.common.DeleteBlockGroupResult;
-import org.apache.hadoop.ozone.common.BlockGroup;
-import org.apache.hadoop.ozone.protocol.proto.ScmBlockLocationProtocolProtos.DeleteScmKeyBlocksRequestProto;
-import org.apache.hadoop.ozone.protocol.proto.ScmBlockLocationProtocolProtos.AllocateScmBlockRequestProto;
-import org.apache.hadoop.ozone.protocol.proto.ScmBlockLocationProtocolProtos.AllocateScmBlockResponseProto;
-import org.apache.hadoop.ozone.protocol.proto.ScmBlockLocationProtocolProtos.DeleteScmKeyBlocksResponseProto;
-import org.apache.hadoop.ozone.protocol.proto.ScmBlockLocationProtocolProtos.GetScmBlockLocationsRequestProto;
-import org.apache.hadoop.ozone.protocol.proto.ScmBlockLocationProtocolProtos.GetScmBlockLocationsResponseProto;
-import org.apache.hadoop.ozone.protocol.proto.ScmBlockLocationProtocolProtos.ScmLocatedBlockProto;
-import org.apache.hadoop.ozone.protocol.proto.ScmBlockLocationProtocolProtos.KeyBlocks;
-import org.apache.hadoop.scm.container.common.helpers.AllocatedBlock;
-import org.apache.hadoop.scm.ScmInfo;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.scm.protocol.ScmBlockLocationProtocol;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-/**
- * This class is the client-side translator to translate the requests made on
- * the {@link ScmBlockLocationProtocol} interface to the RPC server
- * implementing {@link ScmBlockLocationProtocolPB}.
- */
-@InterfaceAudience.Private
-public final class ScmBlockLocationProtocolClientSideTranslatorPB
- implements ScmBlockLocationProtocol, ProtocolTranslator, Closeable {
-
- /**
- * RpcController is not used and hence is set to null.
- */
- private static final RpcController NULL_RPC_CONTROLLER = null;
-
- private final ScmBlockLocationProtocolPB rpcProxy;
-
- /**
- * Creates a new StorageContainerLocationProtocolClientSideTranslatorPB.
- *
- * @param rpcProxy {@link StorageContainerLocationProtocolPB} RPC proxy
- */
- public ScmBlockLocationProtocolClientSideTranslatorPB(
- ScmBlockLocationProtocolPB rpcProxy) {
- this.rpcProxy = rpcProxy;
- }
-
- /**
- * Find the set of nodes to read/write a block, as
- * identified by the block key. This method supports batch lookup by
- * passing multiple keys.
- *
- * @param keys batch of block keys to find
- * @return allocated blocks for each block key
- * @throws IOException if there is any failure
- */
- @Override
- public Set<AllocatedBlock> getBlockLocations(Set<String> keys)
- throws IOException {
- GetScmBlockLocationsRequestProto.Builder req =
- GetScmBlockLocationsRequestProto.newBuilder();
- for (String key : keys) {
- req.addKeys(key);
- }
- final GetScmBlockLocationsResponseProto resp;
- try {
- resp = rpcProxy.getScmBlockLocations(NULL_RPC_CONTROLLER,
- req.build());
- } catch (ServiceException e) {
- throw ProtobufHelper.getRemoteException(e);
- }
- Set<AllocatedBlock> locatedBlocks =
- Sets.newLinkedHashSetWithExpectedSize(resp.getLocatedBlocksCount());
- for (ScmLocatedBlockProto locatedBlock : resp.getLocatedBlocksList()) {
- locatedBlocks.add(new AllocatedBlock.Builder()
- .setKey(locatedBlock.getKey())
- .setPipeline(Pipeline.getFromProtoBuf(locatedBlock.getPipeline()))
- .build());
- }
- return locatedBlocks;
- }
-
- /**
- * Asks SCM where a block should be allocated. SCM responds with the
- * set of datanodes that should be used creating this block.
- * @param size - size of the block.
- * @return allocated block accessing info (key, pipeline).
- * @throws IOException
- */
- @Override
- public AllocatedBlock allocateBlock(long size,
- OzoneProtos.ReplicationType type, OzoneProtos.ReplicationFactor factor,
- String owner) throws IOException {
- Preconditions.checkArgument(size > 0, "block size must be greater than 0");
-
- AllocateScmBlockRequestProto request =
- AllocateScmBlockRequestProto.newBuilder().setSize(size).setType(type)
- .setFactor(factor).setOwner(owner).build();
- final AllocateScmBlockResponseProto response;
- try {
- response = rpcProxy.allocateScmBlock(NULL_RPC_CONTROLLER, request);
- } catch (ServiceException e) {
- throw ProtobufHelper.getRemoteException(e);
- }
- if (response.getErrorCode() !=
- AllocateScmBlockResponseProto.Error.success) {
- throw new IOException(response.hasErrorMessage() ?
- response.getErrorMessage() : "Allocate block failed.");
- }
- AllocatedBlock.Builder builder = new AllocatedBlock.Builder()
- .setKey(response.getKey())
- .setPipeline(Pipeline.getFromProtoBuf(response.getPipeline()))
- .setShouldCreateContainer(response.getCreateContainer());
- return builder.build();
- }
-
- /**
- * Delete the set of keys specified.
- *
- * @param keyBlocksInfoList batch of block keys to delete.
- * @return list of block deletion results.
- * @throws IOException if there is any failure.
- *
- */
- @Override
- public List<DeleteBlockGroupResult> deleteKeyBlocks(
- List<BlockGroup> keyBlocksInfoList) throws IOException {
- List<KeyBlocks> keyBlocksProto = keyBlocksInfoList.stream()
- .map(BlockGroup::getProto).collect(Collectors.toList());
- DeleteScmKeyBlocksRequestProto request = DeleteScmKeyBlocksRequestProto
- .newBuilder().addAllKeyBlocks(keyBlocksProto).build();
-
- final DeleteScmKeyBlocksResponseProto resp;
- try {
- resp = rpcProxy.deleteScmKeyBlocks(NULL_RPC_CONTROLLER, request);
- } catch (ServiceException e) {
- throw ProtobufHelper.getRemoteException(e);
- }
- List<DeleteBlockGroupResult> results =
- new ArrayList<>(resp.getResultsCount());
- results.addAll(resp.getResultsList().stream().map(
- result -> new DeleteBlockGroupResult(result.getObjectKey(),
- DeleteBlockGroupResult
- .convertBlockResultProto(result.getBlockResultsList())))
- .collect(Collectors.toList()));
- return results;
- }
-
- /**
- * Gets the cluster Id and Scm Id from SCM.
- * @return ScmInfo
- * @throws IOException
- */
- @Override
- public ScmInfo getScmInfo() throws IOException {
- OzoneProtos.GetScmInfoRequestProto request =
- OzoneProtos.GetScmInfoRequestProto.getDefaultInstance();
- OzoneProtos.GetScmInfoRespsonseProto resp;
- try {
- resp = rpcProxy.getScmInfo(NULL_RPC_CONTROLLER, request);
- } catch (ServiceException e) {
- throw ProtobufHelper.getRemoteException(e);
- }
- ScmInfo.Builder builder = new ScmInfo.Builder()
- .setClusterId(resp.getClusterId())
- .setScmId(resp.getScmId());
- return builder.build();
- }
-
- @Override
- public Object getUnderlyingProxyObject() {
- return rpcProxy;
- }
-
- @Override
- public void close() {
- RPC.stopProxy(rpcProxy);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocolPB/ScmBlockLocationProtocolPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocolPB/ScmBlockLocationProtocolPB.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocolPB/ScmBlockLocationProtocolPB.java
deleted file mode 100644
index 256e735..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocolPB/ScmBlockLocationProtocolPB.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.scm.protocolPB;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.ipc.ProtocolInfo;
-import org.apache.hadoop.ozone.protocol.proto.ScmBlockLocationProtocolProtos
- .ScmBlockLocationProtocolService;
-
-/**
- * Protocol used from an HDFS node to StorageContainerManager. This extends the
- * Protocol Buffers service interface to add Hadoop-specific annotations.
- */
-@ProtocolInfo(protocolName =
- "org.apache.hadoop.ozone.protocol.ScmBlockLocationProtocol",
- protocolVersion = 1)
-@InterfaceAudience.Private
-public interface ScmBlockLocationProtocolPB
- extends ScmBlockLocationProtocolService.BlockingInterface {
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org