You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ae...@apache.org on 2018/03/15 16:32:36 UTC
[21/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure
Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton,
Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain,
Nanda Kumar.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/ListContainerHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/ListContainerHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/ListContainerHandler.java
deleted file mode 100644
index d0ac974..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/ListContainerHandler.java
+++ /dev/null
@@ -1,121 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.scm.cli.container;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.HelpFormatter;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.Options;
-import org.apache.hadoop.ozone.scm.cli.OzoneCommandHandler;
-import org.apache.hadoop.ozone.web.utils.JsonUtils;
-import org.apache.hadoop.scm.client.ScmClient;
-import org.apache.hadoop.scm.container.common.helpers.ContainerInfo;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-
-import java.io.IOException;
-import java.util.List;
-
-import static org.apache.hadoop.ozone.scm.cli.SCMCLI.CMD_WIDTH;
-import static org.apache.hadoop.ozone.scm.cli.SCMCLI.HELP_OP;
-
-/**
- * This is the handler that process container list command.
- */
-public class ListContainerHandler extends OzoneCommandHandler {
-
- public static final String CONTAINER_LIST = "list";
- public static final String OPT_START_CONTAINER = "start";
- public static final String OPT_PREFIX_CONTAINER = "prefix";
- public static final String OPT_COUNT = "count";
-
- /**
- * Constructs a handler object.
- *
- * @param scmClient scm client
- */
- public ListContainerHandler(ScmClient scmClient) {
- super(scmClient);
- }
-
- @Override
- public void execute(CommandLine cmd) throws IOException {
- if (!cmd.hasOption(CONTAINER_LIST)) {
- throw new IOException("Expecting container list");
- }
- if (cmd.hasOption(HELP_OP)) {
- displayHelp();
- return;
- }
-
- if (!cmd.hasOption(OPT_COUNT)) {
- displayHelp();
- if (!cmd.hasOption(HELP_OP)) {
- throw new IOException("Expecting container count");
- } else {
- return;
- }
- }
-
- String startName = cmd.getOptionValue(OPT_START_CONTAINER);
- String prefixName = cmd.getOptionValue(OPT_PREFIX_CONTAINER);
- int count = 0;
-
- if (cmd.hasOption(OPT_COUNT)) {
- count = Integer.parseInt(cmd.getOptionValue(OPT_COUNT));
- if (count < 0) {
- displayHelp();
- throw new IOException("-count should not be negative");
- }
- }
-
- List<ContainerInfo> containerList =
- getScmClient().listContainer(startName, prefixName, count);
-
- // Output data list
- for (ContainerInfo container : containerList) {
- outputContainerPipeline(container.getPipeline());
- }
- }
-
- private void outputContainerPipeline(Pipeline pipeline) throws IOException {
- // Print container report info.
- logOut("%s", JsonUtils.toJsonStringWithDefaultPrettyPrinter(
- pipeline.toJsonString()));
- }
-
- @Override
- public void displayHelp() {
- Options options = new Options();
- addOptions(options);
- HelpFormatter helpFormatter = new HelpFormatter();
- helpFormatter.printHelp(CMD_WIDTH, "hdfs scm -container -list <option>",
- "where <option> can be the following", options, "");
- }
-
- public static void addOptions(Options options) {
- Option startContainerOpt = new Option(OPT_START_CONTAINER,
- true, "Specify start container name");
- Option endContainerOpt = new Option(OPT_PREFIX_CONTAINER,
- true, "Specify prefix container name");
- Option countOpt = new Option(OPT_COUNT, true,
- "Specify count number, required");
- options.addOption(countOpt);
- options.addOption(startContainerOpt);
- options.addOption(endContainerOpt);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/package-info.java
deleted file mode 100644
index 1c9f40d..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/package-info.java
+++ /dev/null
@@ -1,19 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.scm.cli.container;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/package-info.java
deleted file mode 100644
index d1f9775..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/package-info.java
+++ /dev/null
@@ -1,19 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.scm.cli;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerMapping.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerMapping.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerMapping.java
deleted file mode 100644
index 5950d9a..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerMapping.java
+++ /dev/null
@@ -1,606 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * <p>http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * <p>Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.scm.container;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.lease.Lease;
-import org.apache.hadoop.ozone.lease.LeaseException;
-import org.apache.hadoop.ozone.lease.LeaseManager;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.ReplicationFactor;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.ReplicationType;
-import org.apache.hadoop.ozone.protocol.proto
- .StorageContainerDatanodeProtocolProtos;
-import org.apache.hadoop.ozone.protocol.proto
- .StorageContainerDatanodeProtocolProtos.ContainerReportsRequestProto;
-import org.apache.hadoop.ozone.scm.container.closer.ContainerCloser;
-import org.apache.hadoop.ozone.scm.container.replication.ContainerSupervisor;
-import org.apache.hadoop.ozone.scm.exceptions.SCMException;
-import org.apache.hadoop.ozone.scm.node.NodeManager;
-import org.apache.hadoop.ozone.scm.pipelines.PipelineSelector;
-import org.apache.hadoop.ozone.web.utils.OzoneUtils;
-import org.apache.hadoop.scm.ScmConfigKeys;
-import org.apache.hadoop.scm.container.common.helpers.ContainerInfo;
-import org.apache.hadoop.utils.MetadataKeyFilters.KeyPrefixFilter;
-import org.apache.hadoop.utils.MetadataKeyFilters.MetadataKeyFilter;
-import org.apache.hadoop.utils.MetadataStore;
-import org.apache.hadoop.utils.MetadataStoreBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.charset.Charset;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantLock;
-
-import static org.apache.hadoop.ozone.OzoneConsts.SCM_CONTAINER_DB;
-import static org.apache.hadoop.ozone.scm.exceptions.SCMException.ResultCodes
- .FAILED_TO_CHANGE_CONTAINER_STATE;
-import static org.apache.hadoop.scm.ScmConfigKeys
- .OZONE_SCM_CONTAINER_SIZE_DEFAULT;
-import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_GB;
-
-/**
- * Mapping class contains the mapping from a name to a pipeline mapping. This
- * is used by SCM when
- * allocating new locations and when looking up a key.
- */
-public class ContainerMapping implements Mapping {
- private static final Logger LOG = LoggerFactory.getLogger(ContainerMapping
- .class);
-
- private final NodeManager nodeManager;
- private final long cacheSize;
- private final Lock lock;
- private final Charset encoding = Charset.forName("UTF-8");
- private final MetadataStore containerStore;
- private final PipelineSelector pipelineSelector;
- private final ContainerStateManager containerStateManager;
- private final LeaseManager<ContainerInfo> containerLeaseManager;
- private final ContainerSupervisor containerSupervisor;
- private final float containerCloseThreshold;
- private final ContainerCloser closer;
- private final long size;
-
- /**
- * Constructs a mapping class that creates mapping between container names
- * and pipelines.
- *
- * @param nodeManager - NodeManager so that we can get the nodes that are
- * healthy to place new
- * containers.
- * @param cacheSizeMB - Amount of memory reserved for the LSM tree to cache
- * its nodes. This is
- * passed to LevelDB and this memory is allocated in Native code space.
- * CacheSize is specified
- * in MB.
- * @throws IOException on Failure.
- */
- @SuppressWarnings("unchecked")
- public ContainerMapping(
- final Configuration conf, final NodeManager nodeManager, final int
- cacheSizeMB) throws IOException {
- this.nodeManager = nodeManager;
- this.cacheSize = cacheSizeMB;
- this.closer = new ContainerCloser(nodeManager, conf);
-
- File metaDir = OzoneUtils.getOzoneMetaDirPath(conf);
-
- // Write the container name to pipeline mapping.
- File containerDBPath = new File(metaDir, SCM_CONTAINER_DB);
- containerStore =
- MetadataStoreBuilder.newBuilder()
- .setConf(conf)
- .setDbFile(containerDBPath)
- .setCacheSize(this.cacheSize * OzoneConsts.MB)
- .build();
-
- this.lock = new ReentrantLock();
-
- this.pipelineSelector = new PipelineSelector(nodeManager, conf);
-
- // To be replaced with code getStorageSize once it is committed.
- size = conf.getLong(OZONE_SCM_CONTAINER_SIZE_GB,
- OZONE_SCM_CONTAINER_SIZE_DEFAULT) * 1024 * 1024 * 1024;
- this.containerStateManager =
- new ContainerStateManager(conf, this);
- this.containerSupervisor =
- new ContainerSupervisor(conf, nodeManager,
- nodeManager.getNodePoolManager());
- this.containerCloseThreshold = conf.getFloat(
- ScmConfigKeys.OZONE_SCM_CONTAINER_CLOSE_THRESHOLD,
- ScmConfigKeys.OZONE_SCM_CONTAINER_CLOSE_THRESHOLD_DEFAULT);
- LOG.trace("Container State Manager created.");
-
- long containerCreationLeaseTimeout = conf.getTimeDuration(
- ScmConfigKeys.OZONE_SCM_CONTAINER_CREATION_LEASE_TIMEOUT,
- ScmConfigKeys.OZONE_SCM_CONTAINER_CREATION_LEASE_TIMEOUT_DEFAULT,
- TimeUnit.MILLISECONDS);
- LOG.trace("Starting Container Lease Manager.");
- containerLeaseManager = new LeaseManager<>(containerCreationLeaseTimeout);
- containerLeaseManager.start();
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public ContainerInfo getContainer(final String containerName) throws
- IOException {
- ContainerInfo containerInfo;
- lock.lock();
- try {
- byte[] containerBytes = containerStore.get(containerName.getBytes(
- encoding));
- if (containerBytes == null) {
- throw new SCMException(
- "Specified key does not exist. key : " + containerName,
- SCMException.ResultCodes.FAILED_TO_FIND_CONTAINER);
- }
-
- OzoneProtos.SCMContainerInfo temp = OzoneProtos.SCMContainerInfo.PARSER
- .parseFrom(containerBytes);
- containerInfo = ContainerInfo.fromProtobuf(temp);
- return containerInfo;
- } finally {
- lock.unlock();
- }
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public List<ContainerInfo> listContainer(String startName,
- String prefixName, int count) throws IOException {
- List<ContainerInfo> containerList = new ArrayList<>();
- lock.lock();
- try {
- if (containerStore.isEmpty()) {
- throw new IOException("No container exists in current db");
- }
- MetadataKeyFilter prefixFilter = new KeyPrefixFilter(prefixName);
- byte[] startKey = startName == null ? null : DFSUtil.string2Bytes(
- startName);
- List<Map.Entry<byte[], byte[]>> range =
- containerStore.getSequentialRangeKVs(startKey, count, prefixFilter);
-
- // Transform the values into the pipelines.
- // TODO: filter by container state
- for (Map.Entry<byte[], byte[]> entry : range) {
- ContainerInfo containerInfo =
- ContainerInfo.fromProtobuf(
- OzoneProtos.SCMContainerInfo.PARSER.parseFrom(
- entry.getValue()));
- Preconditions.checkNotNull(containerInfo);
- containerList.add(containerInfo);
- }
- } finally {
- lock.unlock();
- }
- return containerList;
- }
-
- /**
- * Allocates a new container.
- *
- * @param replicationFactor - replication factor of the container.
- * @param containerName - Name of the container.
- * @param owner - The string name of the Service that owns this container.
- * @return - Pipeline that makes up this container.
- * @throws IOException - Exception
- */
- @Override
- public ContainerInfo allocateContainer(
- ReplicationType type,
- ReplicationFactor replicationFactor,
- final String containerName,
- String owner)
- throws IOException {
- Preconditions.checkNotNull(containerName);
- Preconditions.checkState(!containerName.isEmpty());
-
- ContainerInfo containerInfo;
- if (!nodeManager.isOutOfChillMode()) {
- throw new SCMException(
- "Unable to create container while in chill mode",
- SCMException.ResultCodes.CHILL_MODE_EXCEPTION);
- }
-
- lock.lock();
- try {
- byte[] containerBytes = containerStore.get(containerName.getBytes(
- encoding));
- if (containerBytes != null) {
- throw new SCMException(
- "Specified container already exists. key : " + containerName,
- SCMException.ResultCodes.CONTAINER_EXISTS);
- }
- containerInfo =
- containerStateManager.allocateContainer(
- pipelineSelector, type, replicationFactor, containerName,
- owner);
- containerStore.put(
- containerName.getBytes(encoding), containerInfo.getProtobuf()
- .toByteArray());
- } finally {
- lock.unlock();
- }
- return containerInfo;
- }
-
- /**
- * Deletes a container from SCM.
- *
- * @param containerName - Container name
- * @throws IOException if container doesn't exist or container store failed
- * to delete the
- * specified key.
- */
- @Override
- public void deleteContainer(String containerName) throws IOException {
- lock.lock();
- try {
- byte[] dbKey = containerName.getBytes(encoding);
- byte[] containerBytes = containerStore.get(dbKey);
- if (containerBytes == null) {
- throw new SCMException(
- "Failed to delete container " + containerName + ", reason : " +
- "container doesn't exist.",
- SCMException.ResultCodes.FAILED_TO_FIND_CONTAINER);
- }
- containerStore.delete(dbKey);
- } finally {
- lock.unlock();
- }
- }
-
- /**
- * {@inheritDoc} Used by client to update container state on SCM.
- */
- @Override
- public OzoneProtos.LifeCycleState updateContainerState(
- String containerName, OzoneProtos.LifeCycleEvent event) throws
- IOException {
- ContainerInfo containerInfo;
- lock.lock();
- try {
- byte[] dbKey = containerName.getBytes(encoding);
- byte[] containerBytes = containerStore.get(dbKey);
- if (containerBytes == null) {
- throw new SCMException(
- "Failed to update container state"
- + containerName
- + ", reason : container doesn't exist.",
- SCMException.ResultCodes.FAILED_TO_FIND_CONTAINER);
- }
- containerInfo =
- ContainerInfo.fromProtobuf(OzoneProtos.SCMContainerInfo.PARSER
- .parseFrom(containerBytes));
-
- Preconditions.checkNotNull(containerInfo);
- switch (event) {
- case CREATE:
- // Acquire lease on container
- Lease<ContainerInfo> containerLease =
- containerLeaseManager.acquire(containerInfo);
- // Register callback to be executed in case of timeout
- containerLease.registerCallBack(() -> {
- updateContainerState(containerName,
- OzoneProtos.LifeCycleEvent.TIMEOUT);
- return null;
- });
- break;
- case CREATED:
- // Release the lease on container
- containerLeaseManager.release(containerInfo);
- break;
- case FINALIZE:
- // TODO: we don't need a lease manager here for closing as the
- // container report will include the container state after HDFS-13008
- // If a client failed to update the container close state, DN container
- // report from 3 DNs will be used to close the container eventually.
- break;
- case CLOSE:
- break;
- case UPDATE:
- break;
- case DELETE:
- break;
- case TIMEOUT:
- break;
- case CLEANUP:
- break;
- default:
- throw new SCMException("Unsupported container LifeCycleEvent.",
- FAILED_TO_CHANGE_CONTAINER_STATE);
- }
- // If the below updateContainerState call fails, we should revert the
- // changes made in switch case.
- // Like releasing the lease in case of BEGIN_CREATE.
- ContainerInfo updatedContainer = containerStateManager
- .updateContainerState(containerInfo, event);
- containerStore.put(dbKey, updatedContainer.getProtobuf().toByteArray());
- return updatedContainer.getState();
- } catch (LeaseException e) {
- throw new IOException("Lease Exception.", e);
- } finally {
- lock.unlock();
- }
- }
-
- /**
- * Returns the container State Manager.
- *
- * @return ContainerStateManager
- */
- @Override
- public ContainerStateManager getStateManager() {
- return containerStateManager;
- }
-
- /**
- * Process container report from Datanode.
- * <p>
- * Processing follows a very simple logic for time being.
- * <p>
- * 1. Datanodes report the current State -- denoted by the datanodeState
- * <p>
- * 2. We are the older SCM state from the Database -- denoted by
- * the knownState.
- * <p>
- * 3. We copy the usage etc. from currentState to newState and log that
- * newState to the DB. This allows us SCM to bootup again and read the
- * state of the world from the DB, and then reconcile the state from
- * container reports, when they arrive.
- *
- * @param reports Container report
- */
- @Override
- public void processContainerReports(ContainerReportsRequestProto reports)
- throws IOException {
- List<StorageContainerDatanodeProtocolProtos.ContainerInfo>
- containerInfos = reports.getReportsList();
- containerSupervisor.handleContainerReport(reports);
- for (StorageContainerDatanodeProtocolProtos.ContainerInfo datanodeState :
- containerInfos) {
- byte[] dbKey = datanodeState.getContainerNameBytes().toByteArray();
- lock.lock();
- try {
- byte[] containerBytes = containerStore.get(dbKey);
- if (containerBytes != null) {
- OzoneProtos.SCMContainerInfo knownState =
- OzoneProtos.SCMContainerInfo.PARSER.parseFrom(containerBytes);
-
- OzoneProtos.SCMContainerInfo newState =
- reconcileState(datanodeState, knownState);
-
- // FIX ME: This can be optimized, we write twice to memory, where a
- // single write would work well.
- //
- // We need to write this to DB again since the closed only write
- // the updated State.
- containerStore.put(dbKey, newState.toByteArray());
-
- // If the container is closed, then state is already written to SCM
- // DB.TODO: So can we can write only once to DB.
- if (closeContainerIfNeeded(newState)) {
- LOG.info("Closing the Container: {}", newState.getContainerName());
- }
- } else {
- // Container not found in our container db.
- LOG.error("Error while processing container report from datanode :" +
- " {}, for container: {}, reason: container doesn't exist in" +
- "container database.", reports.getDatanodeID(),
- datanodeState.getContainerName());
- }
- } finally {
- lock.unlock();
- }
- }
- }
-
- /**
- * Reconciles the state from Datanode with the state in SCM.
- *
- * @param datanodeState - State from the Datanode.
- * @param knownState - State inside SCM.
- * @return new SCM State for this container.
- */
- private OzoneProtos.SCMContainerInfo reconcileState(
- StorageContainerDatanodeProtocolProtos.ContainerInfo datanodeState,
- OzoneProtos.SCMContainerInfo knownState) {
- OzoneProtos.SCMContainerInfo.Builder builder =
- OzoneProtos.SCMContainerInfo.newBuilder();
- builder.setContainerName(knownState.getContainerName());
- builder.setPipeline(knownState.getPipeline());
- // If used size is greater than allocated size, we will be updating
- // allocated size with used size. This update is done as a fallback
- // mechanism in case SCM crashes without properly updating allocated
- // size. Correct allocated value will be updated by
- // ContainerStateManager during SCM shutdown.
- long usedSize = datanodeState.getUsed();
- long allocated = knownState.getAllocatedBytes() > usedSize ?
- knownState.getAllocatedBytes() : usedSize;
- builder.setAllocatedBytes(allocated);
- builder.setUsedBytes(usedSize);
- builder.setNumberOfKeys(datanodeState.getKeyCount());
- builder.setState(knownState.getState());
- builder.setStateEnterTime(knownState.getStateEnterTime());
- builder.setContainerID(knownState.getContainerID());
- if (knownState.getOwner() != null) {
- builder.setOwner(knownState.getOwner());
- }
- return builder.build();
- }
-
- /**
- * Queues the close container command, to datanode and writes the new state
- * to container DB.
- * <p>
- * TODO : Remove this 2 ContainerInfo definitions. It is brain dead to have
- * one protobuf in one file and another definition in another file.
- *
- * @param newState - This is the state we maintain in SCM.
- * @throws IOException
- */
- private boolean closeContainerIfNeeded(OzoneProtos.SCMContainerInfo newState)
- throws IOException {
- float containerUsedPercentage = 1.0f *
- newState.getUsedBytes() / this.size;
-
- ContainerInfo scmInfo = getContainer(newState.getContainerName());
- if (containerUsedPercentage >= containerCloseThreshold
- && !isClosed(scmInfo)) {
- // We will call closer till get to the closed state.
- // That is SCM will make this call repeatedly until we reach the closed
- // state.
- closer.close(newState);
-
- if (shouldClose(scmInfo)) {
- // This event moves the Container from Open to Closing State, this is
- // a state inside SCM. This is the desired state that SCM wants this
- // container to reach. We will know that a container has reached the
- // closed state from container reports. This state change should be
- // invoked once and only once.
- OzoneProtos.LifeCycleState state = updateContainerState(
- scmInfo.getContainerName(),
- OzoneProtos.LifeCycleEvent.FINALIZE);
- if (state != OzoneProtos.LifeCycleState.CLOSING) {
- LOG.error("Failed to close container {}, reason : Not able " +
- "to " +
- "update container state, current container state: {}.",
- newState.getContainerName(), state);
- return false;
- }
- return true;
- }
- }
- return false;
- }
-
- /**
- * In Container is in closed state, if it is in closed, Deleting or Deleted
- * State.
- *
- * @param info - ContainerInfo.
- * @return true if is in open state, false otherwise
- */
- private boolean shouldClose(ContainerInfo info) {
- return info.getState() == OzoneProtos.LifeCycleState.OPEN;
- }
-
- private boolean isClosed(ContainerInfo info) {
- return info.getState() == OzoneProtos.LifeCycleState.CLOSED;
- }
-
- @VisibleForTesting
- public ContainerCloser getCloser() {
- return closer;
- }
-
- /**
- * Closes this stream and releases any system resources associated with it.
- * If the stream is
- * already closed then invoking this method has no effect.
- * <p>
- * <p>As noted in {@link AutoCloseable#close()}, cases where the close may
- * fail require careful
- * attention. It is strongly advised to relinquish the underlying resources
- * and to internally
- * <em>mark</em> the {@code Closeable} as closed, prior to throwing the
- * {@code IOException}.
- *
- * @throws IOException if an I/O error occurs
- */
- @Override
- public void close() throws IOException {
- if (containerLeaseManager != null) {
- containerLeaseManager.shutdown();
- }
- if (containerStateManager != null) {
- flushContainerInfo();
- containerStateManager.close();
- }
- if (containerStore != null) {
- containerStore.close();
- }
- }
-
- /**
- * Since allocatedBytes of a container is only in memory, stored in
- * containerStateManager, when closing ContainerMapping, we need to update
- * this in the container store.
- *
- * @throws IOException on failure.
- */
- @VisibleForTesting
- public void flushContainerInfo() throws IOException {
- List<ContainerInfo> containers = containerStateManager.getAllContainers();
- List<String> failedContainers = new ArrayList<>();
- for (ContainerInfo info : containers) {
- // even if some container updated failed, others can still proceed
- try {
- byte[] dbKey = info.getContainerName().getBytes(encoding);
- byte[] containerBytes = containerStore.get(dbKey);
- // TODO : looks like when a container is deleted, the container is
- // removed from containerStore but not containerStateManager, so it can
- // return info of a deleted container. may revisit this in the future,
- // for now, just skip a not-found container
- if (containerBytes != null) {
- OzoneProtos.SCMContainerInfo oldInfoProto =
- OzoneProtos.SCMContainerInfo.PARSER.parseFrom(containerBytes);
- ContainerInfo oldInfo = ContainerInfo.fromProtobuf(oldInfoProto);
- ContainerInfo newInfo = new ContainerInfo.Builder()
- .setAllocatedBytes(info.getAllocatedBytes())
- .setContainerName(oldInfo.getContainerName())
- .setNumberOfKeys(oldInfo.getNumberOfKeys())
- .setOwner(oldInfo.getOwner())
- .setPipeline(oldInfo.getPipeline())
- .setState(oldInfo.getState())
- .setUsedBytes(oldInfo.getUsedBytes())
- .build();
- containerStore.put(dbKey, newInfo.getProtobuf().toByteArray());
- } else {
- LOG.debug("Container state manager has container {} but not found " +
- "in container store, a deleted container?",
- info.getContainerName());
- }
- } catch (IOException ioe) {
- failedContainers.add(info.getContainerName());
- }
- }
- if (!failedContainers.isEmpty()) {
- throw new IOException("Error in flushing container info from container " +
- "state manager: " + failedContainers);
- }
- }
-
- @VisibleForTesting
- public MetadataStore getContainerStore() {
- return containerStore;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStateManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStateManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStateManager.java
deleted file mode 100644
index c586107..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStateManager.java
+++ /dev/null
@@ -1,451 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.scm.container;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.common.statemachine.InvalidStateTransitionException;
-import org.apache.hadoop.ozone.common.statemachine.StateMachine;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.LifeCycleEvent;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.LifeCycleState;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.ReplicationFactor;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.ReplicationType;
-import org.apache.hadoop.ozone.scm.container.ContainerStates.ContainerID;
-import org.apache.hadoop.ozone.scm.container.ContainerStates.ContainerState;
-import org.apache.hadoop.ozone.scm.container.ContainerStates.ContainerStateMap;
-import org.apache.hadoop.ozone.scm.exceptions.SCMException;
-import org.apache.hadoop.ozone.scm.pipelines.PipelineSelector;
-import org.apache.hadoop.scm.ScmConfigKeys;
-import org.apache.hadoop.scm.container.common.helpers.ContainerInfo;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.util.Time;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.NavigableSet;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.atomic.AtomicLong;
-
-import static org.apache.hadoop.ozone.scm.exceptions.SCMException.ResultCodes
- .FAILED_TO_CHANGE_CONTAINER_STATE;
-
-/**
- * A container state manager keeps track of container states and returns
- * containers that match various queries.
- * <p>
- * This state machine is driven by a combination of server and client actions.
- * <p>
- * This is how a create container happens: 1. When a container is created, the
- * Server(or SCM) marks that Container as ALLOCATED state. In this state, SCM
- * has chosen a pipeline for container to live on. However, the container is not
- * created yet. This container along with the pipeline is returned to the
- * client.
- * <p>
- * 2. The client when it sees the Container state as ALLOCATED understands that
- * container needs to be created on the specified pipeline. The client lets the
- * SCM know that saw this flag and is initiating the on the data nodes.
- * <p>
- * This is done by calling into notifyObjectCreation(ContainerName,
- * BEGIN_CREATE) flag. When SCM gets this call, SCM puts the container state
- * into CREATING. All this state means is that SCM told Client to create a
- * container and client saw that request.
- * <p>
- * 3. Then client makes calls to datanodes directly, asking the datanodes to
- * create the container. This is done with the help of pipeline that supports
- * this container.
- * <p>
- * 4. Once the creation of the container is complete, the client will make
- * another call to the SCM, this time specifying the containerName and the
- * COMPLETE_CREATE as the Event.
- * <p>
- * 5. With COMPLETE_CREATE event, the container moves to an Open State. This is
- * the state when clients can write to a container.
- * <p>
- * 6. If the client does not respond with the COMPLETE_CREATE event with a
- * certain time, the state machine times out and triggers a delete operation of
- * the container.
- * <p>
- * Please see the function initializeStateMachine below to see how this looks in
- * code.
- * <p>
- * Reusing existing container :
- * <p>
- * The create container call is not made all the time, the system tries to use
- * open containers as much as possible. So in those cases, it looks thru the
- * list of open containers and will return containers that match the specific
- * signature.
- * <p>
- * Please note : Logically there are 3 separate state machines in the case of
- * containers.
- * <p>
- * The Create State Machine -- Commented extensively above.
- * <p>
- * Open/Close State Machine - Once the container is in the Open State,
- * eventually it will be closed, once sufficient data has been written to it.
- * <p>
- * TimeOut Delete Container State Machine - if the container creating times out,
- * then Container State manager decides to delete the container.
- */
-public class ContainerStateManager implements Closeable {
- private static final Logger LOG =
- LoggerFactory.getLogger(ContainerStateManager.class);
-
- private final StateMachine<OzoneProtos.LifeCycleState,
- OzoneProtos.LifeCycleEvent> stateMachine;
-
- private final long containerSize;
- private final ConcurrentHashMap<ContainerState, ContainerID> lastUsedMap;
- private final ContainerStateMap containers;
- private final AtomicLong containerCount;
-
- /**
- * Constructs a Container State Manager that tracks all containers owned by
- * SCM for the purpose of allocation of blocks.
- * <p>
- * TODO : Add Container Tags so we know which containers are owned by SCM.
- */
- @SuppressWarnings("unchecked")
- public ContainerStateManager(Configuration configuration,
- Mapping containerMapping) {
-
- // Initialize the container state machine.
- Set<OzoneProtos.LifeCycleState> finalStates = new HashSet();
-
- // These are the steady states of a container.
- finalStates.add(LifeCycleState.OPEN);
- finalStates.add(LifeCycleState.CLOSED);
- finalStates.add(LifeCycleState.DELETED);
-
- this.stateMachine = new StateMachine<>(LifeCycleState.ALLOCATED,
- finalStates);
- initializeStateMachine();
-
- this.containerSize = OzoneConsts.GB * configuration.getInt(
- ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_GB,
- ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT);
-
- lastUsedMap = new ConcurrentHashMap<>();
- containerCount = new AtomicLong(0);
- containers = new ContainerStateMap();
- loadExistingContainers(containerMapping);
- }
-
- private void loadExistingContainers(Mapping containerMapping) {
-
- List<ContainerInfo> containerList;
- try {
- containerList = containerMapping.listContainer(null,
- null, Integer.MAX_VALUE);
-
- // if there are no container to load, let us return.
- if (containerList == null || containerList.size() == 0) {
- LOG.info("No containers to load for this cluster.");
- return;
- }
- } catch (IOException e) {
- if (!e.getMessage().equals("No container exists in current db")) {
- LOG.error("Could not list the containers", e);
- }
- return;
- }
-
- try {
- long maxID = 0;
- for (ContainerInfo container : containerList) {
- containers.addContainer(container);
-
- if (maxID < container.getContainerID()) {
- maxID = container.getContainerID();
- }
-
- containerCount.set(maxID);
- }
- } catch (SCMException ex) {
- LOG.error("Unable to create a container information. ", ex);
- // Fix me, what is the proper shutdown procedure for SCM ??
- // System.exit(1) // Should we exit here?
- }
- }
-
- /**
- * Return the info of all the containers kept by the in-memory mapping.
- *
- * @return the list of all container info.
- */
- public List<ContainerInfo> getAllContainers() {
- List<ContainerInfo> list = new ArrayList<>();
-
- //No Locking needed since the return value is an immutable map.
- containers.getContainerMap().forEach((key, value) -> list.add(value));
- return list;
- }
-
- /*
- *
- * Event and State Transition Mapping:
- *
- * State: ALLOCATED ---------------> CREATING
- * Event: CREATE
- *
- * State: CREATING ---------------> OPEN
- * Event: CREATED
- *
- * State: OPEN ---------------> CLOSING
- * Event: FINALIZE
- *
- * State: CLOSING ---------------> CLOSED
- * Event: CLOSE
- *
- * State: CLOSED ----------------> DELETING
- * Event: DELETE
- *
- * State: DELETING ----------------> DELETED
- * Event: CLEANUP
- *
- * State: CREATING ---------------> DELETING
- * Event: TIMEOUT
- *
- *
- * Container State Flow:
- *
- * [ALLOCATED]------->[CREATING]--------->[OPEN]---------->[CLOSING]------->[CLOSED]
- * (CREATE) | (CREATED) (FINALIZE) (CLOSE) |
- * | |
- * | |
- * |(TIMEOUT) (DELETE)|
- * | |
- * +------------------> [DELETING] <-------------------+
- * |
- * |
- * (CLEANUP)|
- * |
- * [DELETED]
- */
- private void initializeStateMachine() {
- stateMachine.addTransition(LifeCycleState.ALLOCATED,
- LifeCycleState.CREATING,
- LifeCycleEvent.CREATE);
-
- stateMachine.addTransition(LifeCycleState.CREATING,
- LifeCycleState.OPEN,
- LifeCycleEvent.CREATED);
-
- stateMachine.addTransition(LifeCycleState.OPEN,
- LifeCycleState.CLOSING,
- LifeCycleEvent.FINALIZE);
-
- stateMachine.addTransition(LifeCycleState.CLOSING,
- LifeCycleState.CLOSED,
- LifeCycleEvent.CLOSE);
-
- stateMachine.addTransition(LifeCycleState.CLOSED,
- LifeCycleState.DELETING,
- LifeCycleEvent.DELETE);
-
- stateMachine.addTransition(LifeCycleState.CREATING,
- LifeCycleState.DELETING,
- LifeCycleEvent.TIMEOUT);
-
- stateMachine.addTransition(LifeCycleState.DELETING,
- LifeCycleState.DELETED,
- LifeCycleEvent.CLEANUP);
- }
-
- /**
- * allocates a new container based on the type, replication etc.
- *
- * @param selector -- Pipeline selector class.
- * @param type -- Replication type.
- * @param replicationFactor - Replication replicationFactor.
- * @param containerName - Container Name.
- * @return Container Info.
- * @throws IOException on Failure.
- */
- public ContainerInfo allocateContainer(PipelineSelector selector, OzoneProtos
- .ReplicationType type, OzoneProtos.ReplicationFactor replicationFactor,
- final String containerName, String owner) throws
- IOException {
-
- Pipeline pipeline = selector.getReplicationPipeline(type,
- replicationFactor, containerName);
- ContainerInfo containerInfo = new ContainerInfo.Builder()
- .setContainerName(containerName)
- .setState(OzoneProtos.LifeCycleState.ALLOCATED)
- .setPipeline(pipeline)
- // This is bytes allocated for blocks inside container, not the
- // container size
- .setAllocatedBytes(0)
- .setUsedBytes(0)
- .setNumberOfKeys(0)
- .setStateEnterTime(Time.monotonicNow())
- .setOwner(owner)
- .setContainerID(containerCount.incrementAndGet())
- .build();
- Preconditions.checkNotNull(containerInfo);
- containers.addContainer(containerInfo);
- LOG.trace("New container allocated: {}", containerInfo);
- return containerInfo;
- }
-
- /**
- * Update the Container State to the next state.
- *
- * @param info - ContainerInfo
- * @param event - LifeCycle Event
- * @return Updated ContainerInfo.
- * @throws SCMException on Failure.
- */
- public ContainerInfo updateContainerState(ContainerInfo
- info, OzoneProtos.LifeCycleEvent event) throws SCMException {
- LifeCycleState newState;
- try {
- newState = this.stateMachine.getNextState(info.getState(), event);
- } catch (InvalidStateTransitionException ex) {
- String error = String.format("Failed to update container state %s, " +
- "reason: invalid state transition from state: %s upon " +
- "event: %s.",
- info.getPipeline().getContainerName(), info.getState(), event);
- LOG.error(error);
- throw new SCMException(error, FAILED_TO_CHANGE_CONTAINER_STATE);
- }
-
- // This is a post condition after executing getNextState.
- Preconditions.checkNotNull(newState);
- containers.updateState(info, info.getState(), newState);
- return containers.getContainerInfo(info);
- }
-
- /**
- * Update the container State.
- * @param info - Container Info
- * @return ContainerInfo
- * @throws SCMException - on Error.
- */
- public ContainerInfo updateContainerInfo(ContainerInfo info)
- throws SCMException {
- containers.updateContainerInfo(info);
- return containers.getContainerInfo(info);
- }
-
-
- /**
- * Return a container matching the attributes specified.
- *
- * @param size - Space needed in the Container.
- * @param owner - Owner of the container - A specific nameservice.
- * @param type - Replication Type {StandAlone, Ratis}
- * @param factor - Replication Factor {ONE, THREE}
- * @param state - State of the Container-- {Open, Allocated etc.}
- * @return ContainerInfo, null if there is no match found.
- */
- public ContainerInfo getMatchingContainer(final long size,
- String owner, ReplicationType type, ReplicationFactor factor,
- LifeCycleState state) {
-
- // Find containers that match the query spec, if no match return null.
- NavigableSet<ContainerID> matchingSet =
- containers.getMatchingContainerIDs(state, owner, factor, type);
- if (matchingSet == null || matchingSet.size() == 0) {
- return null;
- }
-
- // Get the last used container and find container above the last used
- // container ID.
- ContainerState key = new ContainerState(owner, type, factor);
- ContainerID lastID = lastUsedMap.get(key);
- if(lastID == null) {
- lastID = matchingSet.first();
- }
-
- // There is a small issue here. The first time, we will skip the first
- // container. But in most cases it will not matter.
- NavigableSet<ContainerID> resultSet = matchingSet.tailSet(lastID, false);
- if (resultSet.size() == 0) {
- resultSet = matchingSet;
- }
-
- ContainerInfo selectedContainer =
- findContainerWithSpace(size, resultSet, owner);
- if (selectedContainer == null) {
-
- // If we did not find any space in the tailSet, we need to look for
- // space in the headset, we need to pass true to deal with the
- // situation that we have a lone container that has space. That is we
- // ignored the last used container under the assumption we can find
- // other containers with space, but if have a single container that is
- // not true. Hence we need to include the last used container as the
- // last element in the sorted set.
-
- resultSet = matchingSet.headSet(lastID, true);
- selectedContainer = findContainerWithSpace(size, resultSet, owner);
- }
- // Update the allocated Bytes on this container.
- if(selectedContainer != null) {
- selectedContainer.updateAllocatedBytes(size);
- }
- return selectedContainer;
-
- }
-
- private ContainerInfo findContainerWithSpace(long size,
- NavigableSet<ContainerID> searchSet, String owner) {
- // Get the container with space to meet our request.
- for (ContainerID id : searchSet) {
- ContainerInfo containerInfo = containers.getContainerInfo(id.getId());
- if (containerInfo.getAllocatedBytes() + size <= this.containerSize) {
- containerInfo.updateLastUsedTime();
-
- ContainerState key = new ContainerState(owner,
- containerInfo.getPipeline().getType(),
- containerInfo.getPipeline().getFactor());
- lastUsedMap.put(key, containerInfo.containerID());
- return containerInfo;
- }
- }
- return null;
- }
-
- /**
- * Returns a set of ContainerIDs that match the Container.
- *
- * @param owner Owner of the Containers.
- * @param type - Replication Type of the containers
- * @param factor - Replication factor of the containers.
- * @param state - Current State, like Open, Close etc.
- * @return Set of containers that match the specific query parameters.
- */
- public NavigableSet<ContainerID> getMatchingContainerIDs(
- String owner, ReplicationType type, ReplicationFactor factor,
- LifeCycleState state) {
- return containers.getMatchingContainerIDs(state, owner,
- factor, type);
- }
-
- @Override
- public void close() throws IOException {
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStates/ContainerAttribute.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStates/ContainerAttribute.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStates/ContainerAttribute.java
deleted file mode 100644
index 1372e7f..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStates/ContainerAttribute.java
+++ /dev/null
@@ -1,244 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- *
- */
-package org.apache.hadoop.ozone.scm.container.ContainerStates;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.ozone.scm.exceptions.SCMException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.NavigableSet;
-import java.util.TreeSet;
-
-import static org.apache.hadoop.ozone.scm.exceptions.SCMException.ResultCodes
- .FAILED_TO_CHANGE_CONTAINER_STATE;
-
-/**
- * Each Attribute that we manage for a container is maintained as a map.
- * <p>
- * Currently we manage the following attributes for a container.
- * <p>
- * 1. StateMap - LifeCycleState -> Set of ContainerIDs
- * 2. TypeMap - ReplicationType -> Set of ContainerIDs
- * 3. OwnerMap - OwnerNames -> Set of ContainerIDs
- * 4. FactorMap - ReplicationFactor -> Set of ContainerIDs
- * <p>
- * This means that for a cluster size of 750 PB -- we will have around 150
- * Million containers, if we assume 5GB average container size.
- * <p>
- * That implies that these maps will take around 2/3 GB of RAM which will be
- * pinned down in the SCM. This is deemed acceptable since we can tune the
- * container size --say we make it 10GB average size, then we can deal with a
- * cluster size of 1.5 exa bytes with the same metadata in SCMs memory.
- * <p>
- * Please note: **This class is not thread safe**. This used to be thread safe,
- * while bench marking we found that ContainerStateMap would be taking 5
- * locks for a single container insert. If we remove locks in this class,
- * then we are able to perform about 540K operations per second, with the
- * locks in this class it goes down to 246K operations per second. Hence we
- * are going to rely on ContainerStateMap locks to maintain consistency of
- * data in these classes too, since ContainerAttribute is only used by
- * ContainerStateMap class.
- */
-public class ContainerAttribute<T> {
- private static final Logger LOG =
- LoggerFactory.getLogger(ContainerAttribute.class);
-
- private final Map<T, NavigableSet<ContainerID>> attributeMap;
- private static final NavigableSet<ContainerID> EMPTY_SET = Collections
- .unmodifiableNavigableSet(new TreeSet<>());
-
- /**
- * Creates a Container Attribute map from an existing Map.
- *
- * @param attributeMap - AttributeMap
- */
- public ContainerAttribute(Map<T, NavigableSet<ContainerID>> attributeMap) {
- this.attributeMap = attributeMap;
- }
-
- /**
- * Create an empty Container Attribute map.
- */
- public ContainerAttribute() {
- this.attributeMap = new HashMap<>();
- }
-
- /**
- * Insert or update the value in the Attribute map.
- *
- * @param key - The key to the set where the ContainerID should exist.
- * @param value - Actual Container ID.
- * @throws SCMException - on Error
- */
- public boolean insert(T key, ContainerID value) throws SCMException {
- Preconditions.checkNotNull(key);
- Preconditions.checkNotNull(value);
-
- if (attributeMap.containsKey(key)) {
- if (attributeMap.get(key).add(value)) {
- return true; //we inserted the value as it doesn’t exist in the set.
- } else { // Failure indicates that this ContainerID exists in the Set
- if (!attributeMap.get(key).remove(value)) {
- LOG.error("Failure to remove the object from the Map.Key:{}, " +
- "ContainerID: {}", key, value);
- throw new SCMException("Failure to remove the object from the Map",
- FAILED_TO_CHANGE_CONTAINER_STATE);
- }
- attributeMap.get(key).add(value);
- return true;
- }
- } else {
- // This key does not exist, we need to allocate this key in the map.
- // TODO: Replace TreeSet with FoldedTreeSet from HDFS Utils.
- // Skipping for now, since FoldedTreeSet does not have implementations
- // for headSet and TailSet. We need those calls.
- this.attributeMap.put(key, new TreeSet<>());
- // This should not fail, we just allocated this object.
- attributeMap.get(key).add(value);
- return true;
- }
- }
-
- /**
- * Returns true if have this bucket in the attribute map.
- *
- * @param key - Key to lookup
- * @return true if we have the key
- */
- public boolean hasKey(T key) {
- Preconditions.checkNotNull(key);
- return this.attributeMap.containsKey(key);
- }
-
- /**
- * Returns true if we have the key and the containerID in the bucket.
- *
- * @param key - Key to the bucket
- * @param id - container ID that we want to lookup
- * @return true or false
- */
- public boolean hasContainerID(T key, ContainerID id) {
- Preconditions.checkNotNull(key);
- Preconditions.checkNotNull(id);
-
- return this.attributeMap.containsKey(key) &&
- this.attributeMap.get(key).contains(id);
- }
-
- /**
- * Returns true if we have the key and the containerID in the bucket.
- *
- * @param key - Key to the bucket
- * @param id - container ID that we want to lookup
- * @return true or false
- */
- public boolean hasContainerID(T key, int id) {
- return hasContainerID(key, ContainerID.valueof(id));
- }
-
- /**
- * Clears all entries for this key type.
- *
- * @param key - Key that identifies the Set.
- */
- public void clearSet(T key) {
- Preconditions.checkNotNull(key);
-
- if (attributeMap.containsKey(key)) {
- attributeMap.get(key).clear();
- } else {
- LOG.debug("key: {} does not exist in the attributeMap", key);
- }
- }
-
- /**
- * Removes a container ID from the set pointed by the key.
- *
- * @param key - key to identify the set.
- * @param value - Container ID
- */
- public boolean remove(T key, ContainerID value) {
- Preconditions.checkNotNull(key);
- Preconditions.checkNotNull(value);
-
- if (attributeMap.containsKey(key)) {
- if (!attributeMap.get(key).remove(value)) {
- LOG.debug("ContainerID: {} does not exist in the set pointed by " +
- "key:{}", value, key);
- return false;
- }
- return true;
- } else {
- LOG.debug("key: {} does not exist in the attributeMap", key);
- return false;
- }
- }
-
- /**
- * Returns the collection that maps to the given key.
- *
- * @param key - Key to the bucket.
- * @return Underlying Set in immutable form.
- */
- public NavigableSet<ContainerID> getCollection(T key) {
- Preconditions.checkNotNull(key);
-
- if (this.attributeMap.containsKey(key)) {
- return Collections.unmodifiableNavigableSet(this.attributeMap.get(key));
- }
- LOG.debug("No such Key. Key {}", key);
- return EMPTY_SET;
- }
-
- /**
- * Moves a ContainerID from one bucket to another.
- *
- * @param currentKey - Current Key
- * @param newKey - newKey
- * @param value - ContainerID
- * @throws SCMException on Error
- */
- public void update(T currentKey, T newKey, ContainerID value)
- throws SCMException {
- Preconditions.checkNotNull(currentKey);
- Preconditions.checkNotNull(newKey);
-
- boolean removed = false;
- try {
- removed = remove(currentKey, value);
- if (!removed) {
- throw new SCMException("Unable to find key in the current key bucket",
- FAILED_TO_CHANGE_CONTAINER_STATE);
- }
- insert(newKey, value);
- } catch (SCMException ex) {
- // if we removed the key, insert it back to original bucket, since the
- // next insert failed.
- LOG.error("error in update.", ex);
- if (removed) {
- insert(currentKey, value);
- LOG.trace("reinserted the removed key. {}", currentKey);
- }
- throw ex;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStates/ContainerState.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStates/ContainerState.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStates/ContainerState.java
deleted file mode 100644
index 6c492ff..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStates/ContainerState.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- *
- */
-
-package org.apache.hadoop.ozone.scm.container.ContainerStates;
-
-import org.apache.commons.lang3.builder.EqualsBuilder;
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-
-/**
- * Class that acts as the container state.
- */
-public class ContainerState {
- private final OzoneProtos.ReplicationType type;
- private final String owner;
- private final OzoneProtos.ReplicationFactor replicationFactor;
-
- /**
- * Constructs a Container Key.
- *
- * @param owner - Container Owners
- * @param type - Replication Type.
- * @param factor - Replication Factors
- */
- public ContainerState(String owner, OzoneProtos.ReplicationType type,
- OzoneProtos.ReplicationFactor factor) {
- this.type = type;
- this.owner = owner;
- this.replicationFactor = factor;
- }
-
-
- public OzoneProtos.ReplicationType getType() {
- return type;
- }
-
- public String getOwner() {
- return owner;
- }
-
- public OzoneProtos.ReplicationFactor getFactor() {
- return replicationFactor;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
-
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
-
- ContainerState that = (ContainerState) o;
-
- return new EqualsBuilder()
- .append(type, that.type)
- .append(owner, that.owner)
- .append(replicationFactor, that.replicationFactor)
- .isEquals();
- }
-
- @Override
- public int hashCode() {
- return new HashCodeBuilder(137, 757)
- .append(type)
- .append(owner)
- .append(replicationFactor)
- .toHashCode();
- }
-
- @Override
- public String toString() {
- return "ContainerKey{" +
- ", type=" + type +
- ", owner=" + owner +
- ", replicationFactor=" + replicationFactor +
- '}';
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStates/ContainerStateMap.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStates/ContainerStateMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStates/ContainerStateMap.java
deleted file mode 100644
index eebc6be..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStates/ContainerStateMap.java
+++ /dev/null
@@ -1,402 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- *
- */
-
-package org.apache.hadoop.ozone.scm.container.ContainerStates;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.LifeCycleState;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.ReplicationFactor;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.ReplicationType;
-import org.apache.hadoop.ozone.scm.exceptions.SCMException;
-import org.apache.hadoop.scm.container.common.helpers.ContainerInfo;
-import org.apache.hadoop.util.AutoCloseableLock;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.NavigableSet;
-import java.util.TreeSet;
-
-import static org.apache.hadoop.ozone.scm.exceptions.SCMException.ResultCodes
- .CONTAINER_EXISTS;
-import static org.apache.hadoop.ozone.scm.exceptions.SCMException.ResultCodes
- .FAILED_TO_CHANGE_CONTAINER_STATE;
-import static org.apache.hadoop.ozone.scm.exceptions.SCMException.ResultCodes
- .FAILED_TO_FIND_CONTAINER;
-
-/**
- * Container State Map acts like a unified map for various attributes that are
- * used to select containers when we need allocated blocks.
- * <p>
- * This class provides the ability to query 4 classes of attributes. They are
- * <p>
- * 1. LifeCycleStates - LifeCycle States of container describe in which state
- * a container is. For example, a container needs to be in Open State for a
- * client to able to write to it.
- * <p>
- * 2. Owners - Each instance of Name service, for example, Namenode of HDFS or
- * Key Space Manager (KSM) of Ozone or CBlockServer -- is an owner. It is
- * possible to have many KSMs for a Ozone cluster and only one SCM. But SCM
- * keeps the data from each KSM in separate bucket, never mixing them. To
- * write data, often we have to find all open containers for a specific owner.
- * <p>
- * 3. ReplicationType - The clients are allowed to specify what kind of
- * replication pipeline they want to use. Each Container exists on top of a
- * pipeline, so we need to get ReplicationType that is specified by the user.
- * <p>
- * 4. ReplicationFactor - The replication factor represents how many copies
- * of data should be made, right now we support 2 different types, ONE
- * Replica and THREE Replica. User can specify how many copies should be made
- * for a ozone key.
- * <p>
- * The most common access pattern of this class is to select a container based
- * on all these parameters, for example, when allocating a block we will
- * select a container that belongs to user1, with Ratis replication which can
- * make 3 copies of data. The fact that we will look for open containers by
- * default and if we cannot find them we will add new containers.
- */
-public class ContainerStateMap {
- private static final Logger LOG =
- LoggerFactory.getLogger(ContainerStateMap.class);
-
- private final ContainerAttribute<LifeCycleState> lifeCycleStateMap;
- private final ContainerAttribute<String> ownerMap;
- private final ContainerAttribute<ReplicationFactor> factorMap;
- private final ContainerAttribute<ReplicationType> typeMap;
-
- private final Map<ContainerID, ContainerInfo> containerMap;
- private final static NavigableSet<ContainerID> EMPTY_SET =
- Collections.unmodifiableNavigableSet(new TreeSet<>());
-
- // Container State Map lock should be held before calling into
- // Update ContainerAttributes. The consistency of ContainerAttributes is
- // protected by this lock.
- private final AutoCloseableLock autoLock;
-
- /**
- * Create a ContainerStateMap.
- */
- public ContainerStateMap() {
- lifeCycleStateMap = new ContainerAttribute<>();
- ownerMap = new ContainerAttribute<>();
- factorMap = new ContainerAttribute<>();
- typeMap = new ContainerAttribute<>();
- containerMap = new HashMap<>();
- autoLock = new AutoCloseableLock();
-// new InstrumentedLock(getClass().getName(), LOG,
-// new ReentrantLock(),
-// 1000,
-// 300));
- }
-
- /**
- * Adds a ContainerInfo Entry in the ContainerStateMap.
- *
- * @param info - container info
- * @throws SCMException - throws if create failed.
- */
- public void addContainer(ContainerInfo info)
- throws SCMException {
-
- try (AutoCloseableLock lock = autoLock.acquire()) {
- ContainerID id = ContainerID.valueof(info.getContainerID());
- if (containerMap.putIfAbsent(id, info) != null) {
- LOG.debug("Duplicate container ID detected. {}", id);
- throw new
- SCMException("Duplicate container ID detected.",
- CONTAINER_EXISTS);
- }
-
- lifeCycleStateMap.insert(info.getState(), id);
- ownerMap.insert(info.getOwner(), id);
- factorMap.insert(info.getPipeline().getFactor(), id);
- typeMap.insert(info.getPipeline().getType(), id);
- LOG.trace("Created container with {} successfully.", id);
- }
- }
-
- /**
- * Returns the latest state of Container from SCM's Container State Map.
- *
- * @param info - ContainerInfo
- * @return ContainerInfo
- */
- public ContainerInfo getContainerInfo(ContainerInfo info) {
- return getContainerInfo(info.getContainerID());
- }
-
- /**
- * Returns the latest state of Container from SCM's Container State Map.
- *
- * @param containerID - int
- * @return container info, if found.
- */
- public ContainerInfo getContainerInfo(long containerID) {
- ContainerID id = new ContainerID(containerID);
- return containerMap.get(id);
- }
-
- /**
- * Returns the full container Map.
- *
- * @return - Map
- */
- public Map<ContainerID, ContainerInfo> getContainerMap() {
- try (AutoCloseableLock lock = autoLock.acquire()) {
- return Collections.unmodifiableMap(containerMap);
- }
- }
-
- /**
- * Just update the container State.
- * @param info ContainerInfo.
- */
- public void updateContainerInfo(ContainerInfo info) throws SCMException {
- Preconditions.checkNotNull(info);
- ContainerInfo currentInfo = null;
- try (AutoCloseableLock lock = autoLock.acquire()) {
- currentInfo = containerMap.get(
- ContainerID.valueof(info.getContainerID()));
-
- if (currentInfo == null) {
- throw new SCMException("No such container.", FAILED_TO_FIND_CONTAINER);
- }
- containerMap.put(info.containerID(), info);
- }
- }
-
- /**
- * Update the State of a container.
- *
- * @param info - ContainerInfo
- * @param currentState - CurrentState
- * @param newState - NewState.
- * @throws SCMException - in case of failure.
- */
- public void updateState(ContainerInfo info, LifeCycleState currentState,
- LifeCycleState newState) throws SCMException {
- Preconditions.checkNotNull(currentState);
- Preconditions.checkNotNull(newState);
-
- ContainerID id = new ContainerID(info.getContainerID());
- ContainerInfo currentInfo = null;
-
- try (AutoCloseableLock lock = autoLock.acquire()) {
- currentInfo = containerMap.get(id);
-
- if (currentInfo == null) {
- throw new
- SCMException("No such container.", FAILED_TO_FIND_CONTAINER);
- }
- // We are updating two places before this update is done, these can
- // fail independently, since the code needs to handle it.
-
- // We update the attribute map, if that fails it will throw an exception,
- // so no issues, if we are successful, we keep track of the fact that we
- // have updated the lifecycle state in the map, and update the container
- // state. If this second update fails, we will attempt to roll back the
- // earlier change we did. If the rollback fails, we can be in an
- // inconsistent state,
-
- info.setState(newState);
- containerMap.put(id, info);
- lifeCycleStateMap.update(currentState, newState, id);
- LOG.trace("Updated the container {} to new state. Old = {}, new = " +
- "{}", id, currentState, newState);
- } catch (SCMException ex) {
- LOG.error("Unable to update the container state. {}", ex);
- // we need to revert the change in this attribute since we are not
- // able to update the hash table.
- LOG.info("Reverting the update to lifecycle state. Moving back to " +
- "old state. Old = {}, Attempted state = {}", currentState,
- newState);
-
- containerMap.put(id, currentInfo);
-
- // if this line throws, the state map can be in an inconsistent
- // state, since we will have modified the attribute by the
- // container state will not in sync since we were not able to put
- // that into the hash table.
- lifeCycleStateMap.update(newState, currentState, id);
-
- throw new SCMException("Updating the container map failed.", ex,
- FAILED_TO_CHANGE_CONTAINER_STATE);
- }
- }
-
- /**
- * Returns A list of containers owned by a name service.
- *
- * @param ownerName - Name of the NameService.
- * @return - NavigableSet of ContainerIDs.
- */
- NavigableSet<ContainerID> getContainerIDsByOwner(String ownerName) {
- Preconditions.checkNotNull(ownerName);
-
- try (AutoCloseableLock lock = autoLock.acquire()) {
- return ownerMap.getCollection(ownerName);
- }
- }
-
- /**
- * Returns Containers in the System by the Type.
- *
- * @param type - Replication type -- StandAlone, Ratis etc.
- * @return NavigableSet
- */
- NavigableSet<ContainerID> getContainerIDsByType(ReplicationType type) {
- Preconditions.checkNotNull(type);
-
- try (AutoCloseableLock lock = autoLock.acquire()) {
- return typeMap.getCollection(type);
- }
- }
-
- /**
- * Returns Containers by replication factor.
- *
- * @param factor - Replication Factor.
- * @return NavigableSet.
- */
- NavigableSet<ContainerID> getContainerIDsByFactor(ReplicationFactor factor) {
- Preconditions.checkNotNull(factor);
-
- try (AutoCloseableLock lock = autoLock.acquire()) {
- return factorMap.getCollection(factor);
- }
- }
-
- /**
- * Returns Containers by State.
- *
- * @param state - State - Open, Closed etc.
- * @return List of containers by state.
- */
- NavigableSet<ContainerID> getContainerIDsByState(LifeCycleState state) {
- Preconditions.checkNotNull(state);
-
- try (AutoCloseableLock lock = autoLock.acquire()) {
- return lifeCycleStateMap.getCollection(state);
- }
- }
-
- /**
- * Gets the containers that matches the following filters.
- *
- * @param state - LifeCycleState
- * @param owner - Owner
- * @param factor - Replication Factor
- * @param type - Replication Type
- * @return ContainerInfo or Null if not container satisfies the criteria.
- */
- public NavigableSet<ContainerID> getMatchingContainerIDs(
- LifeCycleState state, String owner,
- ReplicationFactor factor, ReplicationType type) {
-
- Preconditions.checkNotNull(state, "State cannot be null");
- Preconditions.checkNotNull(owner, "Owner cannot be null");
- Preconditions.checkNotNull(factor, "Factor cannot be null");
- Preconditions.checkNotNull(type, "Type cannot be null");
-
- try (AutoCloseableLock lock = autoLock.acquire()) {
-
- // If we cannot meet any one condition we return EMPTY_SET immediately.
- // Since when we intersect these sets, the result will be empty if any
- // one is empty.
- NavigableSet<ContainerID> stateSet =
- lifeCycleStateMap.getCollection(state);
- if (stateSet.size() == 0) {
- return EMPTY_SET;
- }
-
- NavigableSet<ContainerID> ownerSet = ownerMap.getCollection(owner);
- if (ownerSet.size() == 0) {
- return EMPTY_SET;
- }
-
- NavigableSet<ContainerID> factorSet = factorMap.getCollection(factor);
- if (factorSet.size() == 0) {
- return EMPTY_SET;
- }
-
- NavigableSet<ContainerID> typeSet = typeMap.getCollection(type);
- if (typeSet.size() == 0) {
- return EMPTY_SET;
- }
-
-
- // if we add more constraints we will just add those sets here..
- NavigableSet<ContainerID>[] sets = sortBySize(stateSet,
- ownerSet, factorSet, typeSet);
-
- NavigableSet<ContainerID> currentSet = sets[0];
- // We take the smallest set and intersect against the larger sets. This
- // allows us to reduce the lookups to the least possible number.
- for (int x = 1; x < sets.length; x++) {
- currentSet = intersectSets(currentSet, sets[x]);
- }
- return currentSet;
- }
- }
-
- /**
- * Calculates the intersection between sets and returns a new set.
- *
- * @param smaller - First Set
- * @param bigger - Second Set
- * @return resultSet which is the intersection of these two sets.
- */
- private NavigableSet<ContainerID> intersectSets(
- NavigableSet<ContainerID> smaller,
- NavigableSet<ContainerID> bigger) {
- Preconditions.checkState(smaller.size() <= bigger.size(),
- "This function assumes the first set is lesser or equal to second " +
- "set");
- NavigableSet<ContainerID> resultSet = new TreeSet<>();
- for (ContainerID id : smaller) {
- if (bigger.contains(id)) {
- resultSet.add(id);
- }
- }
- return resultSet;
- }
-
- /**
- * Sorts a list of Sets based on Size. This is useful when we are
- * intersecting the sets.
- *
- * @param sets - varagrs of sets
- * @return Returns a sorted array of sets based on the size of the set.
- */
- @SuppressWarnings("unchecked")
- private NavigableSet<ContainerID>[] sortBySize(
- NavigableSet<ContainerID>... sets) {
- for (int x = 0; x < sets.length - 1; x++) {
- for (int y = 0; y < sets.length - x - 1; y++) {
- if (sets[y].size() > sets[y + 1].size()) {
- NavigableSet temp = sets[y];
- sets[y] = sets[y + 1];
- sets[y + 1] = temp;
- }
- }
- }
- return sets;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStates/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStates/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStates/package-info.java
deleted file mode 100644
index 6a7e663..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStates/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- *
- */
-
-/**
- * Container States management package.
- */
-package org.apache.hadoop.ozone.scm.container.ContainerStates;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/Mapping.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/Mapping.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/Mapping.java
deleted file mode 100644
index 0d442d1..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/Mapping.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.scm.container;
-
-
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-import org.apache.hadoop.ozone.protocol.proto
- .StorageContainerDatanodeProtocolProtos.ContainerReportsRequestProto;
-import org.apache.hadoop.scm.container.common.helpers.ContainerInfo;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.List;
-
-/**
- * Mapping class contains the mapping from a name to a pipeline mapping. This is
- * used by SCM when allocating new locations and when looking up a key.
- */
-public interface Mapping extends Closeable {
- /**
- * Returns the ContainerInfo from the container name.
- *
- * @param containerName - Name
- * @return - ContainerInfo such as creation state and the pipeline.
- * @throws IOException
- */
- ContainerInfo getContainer(String containerName) throws IOException;
-
- /**
- * Returns containers under certain conditions.
- * Search container names from start name(exclusive),
- * and use prefix name to filter the result. The max
- * size of the searching range cannot exceed the
- * value of count.
- *
- * @param startName start name, if null, start searching at the head.
- * @param prefixName prefix name, if null, then filter is disabled.
- * @param count count, if count < 0, the max size is unlimited.(
- * Usually the count will be replace with a very big
- * value instead of being unlimited in case the db is very big)
- *
- * @return a list of container.
- * @throws IOException
- */
- List<ContainerInfo> listContainer(String startName, String prefixName,
- int count) throws IOException;
-
- /**
- * Allocates a new container for a given keyName and replication factor.
- *
- * @param replicationFactor - replication factor of the container.
- * @param containerName - Name.
- * @param owner
- * @return - Container Info.
- * @throws IOException
- */
- ContainerInfo allocateContainer(OzoneProtos.ReplicationType type,
- OzoneProtos.ReplicationFactor replicationFactor,
- String containerName, String owner) throws IOException;
-
- /**
- * Deletes a container from SCM.
- *
- * @param containerName - Container Name
- * @throws IOException
- */
- void deleteContainer(String containerName) throws IOException;
-
- /**
- * Update container state.
- * @param containerName - Container Name
- * @param event - container life cycle event
- * @return - new container state
- * @throws IOException
- */
- OzoneProtos.LifeCycleState updateContainerState(String containerName,
- OzoneProtos.LifeCycleEvent event) throws IOException;
-
- /**
- * Returns the container State Manager.
- * @return ContainerStateManager
- */
- ContainerStateManager getStateManager();
-
- /**
- * Process container report from Datanode.
- *
- * @param reports Container report
- */
- void processContainerReports(ContainerReportsRequestProto reports)
- throws IOException;
-
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org