You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by om...@apache.org on 2018/04/26 21:00:42 UTC
[13/68] [abbrv] [partial] hadoop git commit: HDFS-13405. Ozone:
Rename HDSL to HDDS. Contributed by Ajay Kumar, Elek Marton, Mukul Kumar Singh,
Shashikant Banerjee and Anu Engineer.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b832f3c/hadoop-hdsl/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ChunkManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdsl/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ChunkManagerImpl.java b/hadoop-hdsl/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ChunkManagerImpl.java
deleted file mode 100644
index 7c950dc..0000000
--- a/hadoop-hdsl/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ChunkManagerImpl.java
+++ /dev/null
@@ -1,232 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.container.common.impl;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.hdsl.protocol.proto.ContainerProtos;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.container.common.helpers.ContainerData;
-import org.apache.hadoop.scm.container.common.helpers.StorageContainerException;
-import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
-import org.apache.hadoop.ozone.container.common.helpers.ChunkUtils;
-import org.apache.hadoop.ozone.container.common.interfaces.ChunkManager;
-import org.apache.hadoop.ozone.container.common.interfaces.ContainerManager;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.file.Files;
-import java.nio.file.StandardCopyOption;
-import java.security.NoSuchAlgorithmException;
-import java.util.concurrent.ExecutionException;
-
-import static org.apache.hadoop.hdsl.protocol.proto.ContainerProtos
- .Result.CONTAINER_INTERNAL_ERROR;
-import static org.apache.hadoop.hdsl.protocol.proto.ContainerProtos
- .Result.UNSUPPORTED_REQUEST;
-
-/**
- * An implementation of ChunkManager that is used by default in ozone.
- */
-public class ChunkManagerImpl implements ChunkManager {
- static final Logger LOG =
- LoggerFactory.getLogger(ChunkManagerImpl.class);
-
- private final ContainerManager containerManager;
-
- /**
- * Constructs a ChunkManager.
- *
- * @param manager - ContainerManager.
- */
- public ChunkManagerImpl(ContainerManager manager) {
- this.containerManager = manager;
- }
-
- /**
- * writes a given chunk.
- *
- * @param pipeline - Name and the set of machines that make this container.
- * @param keyName - Name of the Key.
- * @param info - ChunkInfo.
- * @throws StorageContainerException
- */
- @Override
- public void writeChunk(Pipeline pipeline, String keyName, ChunkInfo info,
- byte[] data, ContainerProtos.Stage stage)
- throws StorageContainerException {
- // we don't want container manager to go away while we are writing chunks.
- containerManager.readLock();
-
- // TODO : Take keyManager Write lock here.
- try {
- Preconditions.checkNotNull(pipeline, "Pipeline cannot be null");
- String containerName = pipeline.getContainerName();
- Preconditions.checkNotNull(containerName,
- "Container name cannot be null");
- ContainerData container =
- containerManager.readContainer(containerName);
- File chunkFile = ChunkUtils.validateChunk(pipeline, container, info);
- File tmpChunkFile = getTmpChunkFile(chunkFile, info);
-
- LOG.debug("writing chunk:{} chunk stage:{} chunk file:{} tmp chunk file",
- info.getChunkName(), stage, chunkFile, tmpChunkFile);
- switch (stage) {
- case WRITE_DATA:
- ChunkUtils.writeData(tmpChunkFile, info, data);
- break;
- case COMMIT_DATA:
- commitChunk(tmpChunkFile, chunkFile, containerName, info.getLen());
- break;
- case COMBINED:
- // directly write to the chunk file
- long oldSize = chunkFile.length();
- ChunkUtils.writeData(chunkFile, info, data);
- long newSize = chunkFile.length();
- containerManager.incrBytesUsed(containerName, newSize - oldSize);
- containerManager.incrWriteCount(containerName);
- containerManager.incrWriteBytes(containerName, info.getLen());
- break;
- }
- } catch (ExecutionException | NoSuchAlgorithmException | IOException e) {
- LOG.error("write data failed. error: {}", e);
- throw new StorageContainerException("Internal error: ", e,
- CONTAINER_INTERNAL_ERROR);
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- LOG.error("write data failed. error: {}", e);
- throw new StorageContainerException("Internal error: ", e,
- CONTAINER_INTERNAL_ERROR);
- } finally {
- containerManager.readUnlock();
- }
- }
-
- // Create a temporary file in the same container directory
- // in the format "<chunkname>.tmp"
- private static File getTmpChunkFile(File chunkFile, ChunkInfo info)
- throws StorageContainerException {
- return new File(chunkFile.getParent(),
- chunkFile.getName() +
- OzoneConsts.CONTAINER_CHUNK_NAME_DELIMITER +
- OzoneConsts.CONTAINER_TEMPORARY_CHUNK_PREFIX);
- }
-
- // Commit the chunk by renaming the temporary chunk file to chunk file
- private void commitChunk(File tmpChunkFile, File chunkFile,
- String containerName, long chunkLen) throws IOException {
- long sizeDiff = tmpChunkFile.length() - chunkFile.length();
- // It is safe to replace here as the earlier chunk if existing should be
- // caught as part of validateChunk
- Files.move(tmpChunkFile.toPath(), chunkFile.toPath(),
- StandardCopyOption.REPLACE_EXISTING);
- containerManager.incrBytesUsed(containerName, sizeDiff);
- containerManager.incrWriteCount(containerName);
- containerManager.incrWriteBytes(containerName, chunkLen);
- }
-
- /**
- * reads the data defined by a chunk.
- *
- * @param pipeline - container pipeline.
- * @param keyName - Name of the Key
- * @param info - ChunkInfo.
- * @return byte array
- * @throws StorageContainerException
- * TODO: Right now we do not support partial reads and writes of chunks.
- * TODO: Explore if we need to do that for ozone.
- */
- @Override
- public byte[] readChunk(Pipeline pipeline, String keyName, ChunkInfo info)
- throws StorageContainerException {
- containerManager.readLock();
- try {
- Preconditions.checkNotNull(pipeline, "Pipeline cannot be null");
- String containerName = pipeline.getContainerName();
- Preconditions.checkNotNull(containerName,
- "Container name cannot be null");
- ContainerData container =
- containerManager.readContainer(containerName);
- File chunkFile = ChunkUtils.getChunkFile(pipeline, container, info);
- ByteBuffer data = ChunkUtils.readData(chunkFile, info);
- containerManager.incrReadCount(containerName);
- containerManager.incrReadBytes(containerName, chunkFile.length());
- return data.array();
- } catch (ExecutionException | NoSuchAlgorithmException e) {
- LOG.error("read data failed. error: {}", e);
- throw new StorageContainerException("Internal error: ",
- e, CONTAINER_INTERNAL_ERROR);
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- LOG.error("read data failed. error: {}", e);
- throw new StorageContainerException("Internal error: ",
- e, CONTAINER_INTERNAL_ERROR);
- } finally {
- containerManager.readUnlock();
- }
- }
-
- /**
- * Deletes a given chunk.
- *
- * @param pipeline - Pipeline.
- * @param keyName - Key Name
- * @param info - Chunk Info
- * @throws StorageContainerException
- */
- @Override
- public void deleteChunk(Pipeline pipeline, String keyName, ChunkInfo info)
- throws StorageContainerException {
- containerManager.readLock();
- try {
- Preconditions.checkNotNull(pipeline, "Pipeline cannot be null");
- String containerName = pipeline.getContainerName();
- Preconditions.checkNotNull(containerName,
- "Container name cannot be null");
- File chunkFile = ChunkUtils.getChunkFile(pipeline, containerManager
- .readContainer(containerName), info);
- if ((info.getOffset() == 0) && (info.getLen() == chunkFile.length())) {
- FileUtil.fullyDelete(chunkFile);
- containerManager.decrBytesUsed(containerName, chunkFile.length());
- } else {
- LOG.error("Not Supported Operation. Trying to delete a " +
- "chunk that is in shared file. chunk info : " + info.toString());
- throw new StorageContainerException("Not Supported Operation. " +
- "Trying to delete a chunk that is in shared file. chunk info : "
- + info.toString(), UNSUPPORTED_REQUEST);
- }
- } finally {
- containerManager.readUnlock();
- }
- }
-
- /**
- * Shutdown the chunkManager.
- *
- * In the chunkManager we haven't acquired any resources, so nothing to do
- * here. This call is made with containerManager Writelock held.
- */
- @Override
- public void shutdown() {
- Preconditions.checkState(this.containerManager.hasWriteLock());
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b832f3c/hadoop-hdsl/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerLocationManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdsl/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerLocationManagerImpl.java b/hadoop-hdsl/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerLocationManagerImpl.java
deleted file mode 100644
index e4c1989..0000000
--- a/hadoop-hdsl/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerLocationManagerImpl.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.container.common.impl;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.hdfs.server.datanode.StorageLocation;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.container.common.interfaces
- .ContainerLocationManager;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.nio.file.Path;
-import java.util.List;
-
-/**
- * A class that tells the ContainerManager where to place the containers.
- * Please note : There is *no* one-to-one correlation between metadata
- * metadataLocations and data metadataLocations.
- *
- * For example : A user could map all container files to a
- * SSD but leave data/metadata on bunch of other disks.
- */
-public class ContainerLocationManagerImpl implements ContainerLocationManager {
- private static final Logger LOG =
- LoggerFactory.getLogger(ContainerLocationManagerImpl.class);
-
- private final List<StorageLocation> dataLocations;
- private int currentIndex;
- private final List<StorageLocation> metadataLocations;
-
- /**
- * Constructs a Location Manager.
- * @param metadataLocations - Refers to the metadataLocations
- * where we store the container metadata.
- * @param dataDirs - metadataLocations where we store the actual
- * data or chunk files.
- * @throws IOException
- */
- public ContainerLocationManagerImpl(List<StorageLocation> metadataLocations,
- List<StorageLocation> dataDirs)
- throws IOException {
- dataLocations = dataDirs;
- this.metadataLocations = metadataLocations;
- }
- /**
- * Returns the path where the container should be placed from a set of
- * metadataLocations.
- *
- * @return A path where we should place this container and metadata.
- * @throws IOException
- */
- @Override
- public Path getContainerPath()
- throws IOException {
- Preconditions.checkState(metadataLocations.size() > 0);
- int index = currentIndex % metadataLocations.size();
- Path path = metadataLocations.get(index).getFile().toPath();
- return path.resolve(OzoneConsts.CONTAINER_ROOT_PREFIX);
- }
-
- /**
- * Returns the path where the container Data file are stored.
- *
- * @return a path where we place the LevelDB and data files of a container.
- * @throws IOException
- */
- @Override
- public Path getDataPath(String containerName) throws IOException {
- Path currentPath = dataLocations.get(currentIndex++ % dataLocations.size())
- .getFile().toPath();
- currentPath = currentPath.resolve(OzoneConsts.CONTAINER_PREFIX);
- return currentPath.resolve(containerName);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b832f3c/hadoop-hdsl/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerReportManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdsl/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerReportManagerImpl.java b/hadoop-hdsl/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerReportManagerImpl.java
deleted file mode 100644
index a300767..0000000
--- a/hadoop-hdsl/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerReportManagerImpl.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.container.common.impl;
-
-import org.apache.commons.lang3.RandomUtils;
-import static org.apache.hadoop.ozone.scm.HdslServerUtil
- .getScmHeartbeatInterval;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.ozone.container.common.interfaces.ContainerReportManager;
-import org.apache.hadoop.hdsl.protocol.proto.StorageContainerDatanodeProtocolProtos.ReportState;
-import org.apache.hadoop.util.Time;
-
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLong;
-
-/**
- * Class wraps the container report operations on datanode.
- * // TODO: support incremental/delta container report
- */
-public class ContainerReportManagerImpl implements ContainerReportManager {
- // Last non-empty container report time
- private long lastContainerReportTime;
- private final long containerReportInterval;
- private final long heartbeatInterval;
- private AtomicLong reportCount;
- private static final ReportState NO_CONTAINER_REPORTSTATE =
- ReportState.newBuilder()
- .setState(ReportState.states.noContainerReports)
- .setCount(0).build();
-
- public ContainerReportManagerImpl(Configuration config) {
- this.lastContainerReportTime = -1;
- this.reportCount = new AtomicLong(0L);
- this.containerReportInterval = config.getTimeDuration(
- OzoneConfigKeys.OZONE_CONTAINER_REPORT_INTERVAL,
- OzoneConfigKeys.OZONE_CONTAINER_REPORT_INTERVAL_DEFAULT,
- TimeUnit.MILLISECONDS);
- this.heartbeatInterval = getScmHeartbeatInterval(config);
- }
-
- public ReportState getContainerReportState() {
- if (lastContainerReportTime < 0) {
- return getFullContainerReportState();
- } else {
- // Add a random delay (0~30s) on top of the container report
- // interval (60s) so tha the SCM is overwhelmed by the container reports
- // sent in sync.
- if (Time.monotonicNow() - lastContainerReportTime >
- (containerReportInterval + getRandomReportDelay())) {
- return getFullContainerReportState();
- } else {
- return getNoContainerReportState();
- }
- }
- }
-
- private ReportState getFullContainerReportState() {
- ReportState.Builder rsBuilder = ReportState.newBuilder();
- rsBuilder.setState(ReportState.states.completeContinerReport);
- rsBuilder.setCount(reportCount.incrementAndGet());
- this.lastContainerReportTime = Time.monotonicNow();
- return rsBuilder.build();
- }
-
- private ReportState getNoContainerReportState() {
- return NO_CONTAINER_REPORTSTATE;
- }
-
- private long getRandomReportDelay() {
- return RandomUtils.nextLong(0, heartbeatInterval);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b832f3c/hadoop-hdsl/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-hdsl/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerStatus.java b/hadoop-hdsl/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerStatus.java
deleted file mode 100644
index 5577323..0000000
--- a/hadoop-hdsl/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerStatus.java
+++ /dev/null
@@ -1,217 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.container.common.impl;
-
-import org.apache.hadoop.ozone.container.common.helpers.ContainerData;
-
-import java.util.concurrent.atomic.AtomicLong;
-
-/**
- * This class represents the state of a container. if the
- * container reading encountered an error when we boot up we will post that
- * info to a recovery queue and keep the info in the containerMap.
- * <p/>
- * if and when the issue is fixed, the expectation is that this entry will be
- * deleted by the recovery thread from the containerMap and will insert entry
- * instead of modifying this class.
- */
-public class ContainerStatus {
- private final ContainerData containerData;
-
- /**
- * Number of pending deletion blocks in container.
- */
- private int numPendingDeletionBlocks;
-
- private AtomicLong readBytes;
-
- private AtomicLong writeBytes;
-
- private AtomicLong readCount;
-
- private AtomicLong writeCount;
-
- /**
- * Creates a Container Status class.
- *
- * @param containerData - ContainerData.
- */
- ContainerStatus(ContainerData containerData) {
- this.numPendingDeletionBlocks = 0;
- this.containerData = containerData;
- this.readCount = new AtomicLong(0L);
- this.readBytes = new AtomicLong(0L);
- this.writeCount = new AtomicLong(0L);
- this.writeBytes = new AtomicLong(0L);
- }
-
- /**
- * Returns container if it is active. It is not active if we have had an
- * error and we are waiting for the background threads to fix the issue.
- *
- * @return ContainerData.
- */
- public ContainerData getContainer() {
- return containerData;
- }
-
- /**
- * Increase the count of pending deletion blocks.
- *
- * @param numBlocks increment number
- */
- public void incrPendingDeletionBlocks(int numBlocks) {
- this.numPendingDeletionBlocks += numBlocks;
- }
-
- /**
- * Decrease the count of pending deletion blocks.
- *
- * @param numBlocks decrement number
- */
- public void decrPendingDeletionBlocks(int numBlocks) {
- this.numPendingDeletionBlocks -= numBlocks;
- }
-
- /**
- * Get the number of pending deletion blocks.
- */
- public int getNumPendingDeletionBlocks() {
- return this.numPendingDeletionBlocks;
- }
-
- /**
- * Get the number of bytes read from the container.
- * @return the number of bytes read from the container.
- */
- public long getReadBytes() {
- return readBytes.get();
- }
-
- /**
- * Increase the number of bytes read from the container.
- * @param bytes number of bytes read.
- */
- public void incrReadBytes(long bytes) {
- this.readBytes.addAndGet(bytes);
- }
-
- /**
- * Get the number of times the container is read.
- * @return the number of times the container is read.
- */
- public long getReadCount() {
- return readCount.get();
- }
-
- /**
- * Increase the number of container read count by 1.
- */
- public void incrReadCount() {
- this.readCount.incrementAndGet();
- }
-
- /**
- * Get the number of bytes write into the container.
- * @return the number of bytes write into the container.
- */
- public long getWriteBytes() {
- return writeBytes.get();
- }
-
- /**
- * Increase the number of bytes write into the container.
- * @param bytes the number of bytes write into the container.
- */
- public void incrWriteBytes(long bytes) {
- this.writeBytes.addAndGet(bytes);
- }
-
- /**
- * Get the number of writes into the container.
- * @return the number of writes into the container.
- */
- public long getWriteCount() {
- return writeCount.get();
- }
-
- /**
- * Increase the number of writes into the container by 1.
- */
- public void incrWriteCount() {
- this.writeCount.incrementAndGet();
- }
-
- /**
- * Get the number of bytes used by the container.
- * @return the number of bytes used by the container.
- */
- public long getBytesUsed() {
- return containerData.getBytesUsed();
- }
-
- /**
- * Increase the number of bytes used by the container.
- * @param used number of bytes used by the container.
- * @return the current number of bytes used by the container afert increase.
- */
- public long incrBytesUsed(long used) {
- return containerData.addBytesUsed(used);
- }
-
- /**
- * Set the number of bytes used by the container.
- * @param used the number of bytes used by the container.
- */
- public void setBytesUsed(long used) {
- containerData.setBytesUsed(used);
- }
-
- /**
- * Decrease the number of bytes used by the container.
- * @param reclaimed the number of bytes reclaimed from the container.
- * @return the current number of bytes used by the container after decrease.
- */
- public long decrBytesUsed(long reclaimed) {
- return this.containerData.addBytesUsed(-1L * reclaimed);
- }
-
- /**
- * Get the maximum container size.
- * @return the maximum container size.
- */
- public long getMaxSize() {
- return containerData.getMaxSize();
- }
-
- /**
- * Set the maximum container size.
- * @param size the maximum container size.
- */
- public void setMaxSize(long size) {
- this.containerData.setMaxSize(size);
- }
-
- /**
- * Get the number of keys in the container.
- * @return the number of keys in the container.
- */
- public long getNumKeys() {
- return containerData.getKeyCount();
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b832f3c/hadoop-hdsl/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerStorageLocation.java
----------------------------------------------------------------------
diff --git a/hadoop-hdsl/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerStorageLocation.java b/hadoop-hdsl/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerStorageLocation.java
deleted file mode 100644
index 07a3a53..0000000
--- a/hadoop-hdsl/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerStorageLocation.java
+++ /dev/null
@@ -1,203 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.container.common.impl;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CachingGetSpaceUsed;
-import org.apache.hadoop.fs.DF;
-import org.apache.hadoop.fs.GetSpaceUsed;
-import org.apache.hadoop.hdfs.server.datanode.StorageLocation;
-import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.util.ShutdownHookManager;
-import org.apache.hadoop.util.Time;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.FileNotFoundException;
-import java.io.OutputStreamWriter;
-import java.io.FileOutputStream;
-import java.net.URI;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Paths;
-import java.util.Scanner;
-
-import static org.apache.hadoop.util.RunJar.SHUTDOWN_HOOK_PRIORITY;
-
-/**
- * Class that wraps the space usage of the Datanode Container Storage Location
- * by SCM containers.
- */
-public class ContainerStorageLocation {
- private static final Logger LOG =
- LoggerFactory.getLogger(ContainerStorageLocation.class);
-
- private static final String DU_CACHE_FILE = "scmUsed";
- private volatile boolean scmUsedSaved = false;
-
- private final StorageLocation dataLocation;
- private final String storageUuId;
- private final DF usage;
- private final GetSpaceUsed scmUsage;
- private final File scmUsedFile;
-
- public ContainerStorageLocation(StorageLocation dataLoc, Configuration conf)
- throws IOException {
- this.dataLocation = dataLoc;
- this.storageUuId = DatanodeStorage.generateUuid();
- File dataDir = Paths.get(dataLoc.getNormalizedUri()).resolve(
- OzoneConsts.CONTAINER_PREFIX).toFile();
- // Initialize container data root if it does not exist as required by DF/DU
- if (!dataDir.exists()) {
- if (!dataDir.mkdirs()) {
- LOG.error("Unable to create the container storage location at : {}",
- dataDir);
- throw new IllegalArgumentException("Unable to create the container" +
- " storage location at : " + dataDir);
- }
- }
- scmUsedFile = new File(dataDir, DU_CACHE_FILE);
- // get overall disk usage
- this.usage = new DF(dataDir, conf);
- // get SCM specific usage
- this.scmUsage = new CachingGetSpaceUsed.Builder().setPath(dataDir)
- .setConf(conf)
- .setInitialUsed(loadScmUsed())
- .build();
-
- // Ensure scm usage is saved during shutdown.
- ShutdownHookManager.get().addShutdownHook(
- new Runnable() {
- @Override
- public void run() {
- if (!scmUsedSaved) {
- saveScmUsed();
- }
- }
- }, SHUTDOWN_HOOK_PRIORITY);
- }
-
- public URI getNormalizedUri() {
- return dataLocation.getNormalizedUri();
- }
-
- public String getStorageUuId() {
- return storageUuId;
- }
- public long getCapacity() {
- long capacity = usage.getCapacity();
- return (capacity > 0) ? capacity : 0;
- }
-
- public long getAvailable() throws IOException {
- long remaining = getCapacity() - getScmUsed();
- long available = usage.getAvailable();
- if (remaining > available) {
- remaining = available;
- }
- return (remaining > 0) ? remaining : 0;
- }
-
- public long getScmUsed() throws IOException{
- return scmUsage.getUsed();
- }
-
- public void shutdown() {
- saveScmUsed();
- scmUsedSaved = true;
-
- if (scmUsage instanceof CachingGetSpaceUsed) {
- IOUtils.cleanupWithLogger(null, ((CachingGetSpaceUsed) scmUsage));
- }
- }
-
- /**
- * Read in the cached DU value and return it if it is less than 600 seconds
- * old (DU update interval). Slight imprecision of scmUsed is not critical
- * and skipping DU can significantly shorten the startup time.
- * If the cached value is not available or too old, -1 is returned.
- */
- long loadScmUsed() {
- long cachedScmUsed;
- long mtime;
- Scanner sc;
-
- try {
- sc = new Scanner(scmUsedFile, "UTF-8");
- } catch (FileNotFoundException fnfe) {
- return -1;
- }
-
- try {
- // Get the recorded scmUsed from the file.
- if (sc.hasNextLong()) {
- cachedScmUsed = sc.nextLong();
- } else {
- return -1;
- }
- // Get the recorded mtime from the file.
- if (sc.hasNextLong()) {
- mtime = sc.nextLong();
- } else {
- return -1;
- }
-
- // Return the cached value if mtime is okay.
- if (mtime > 0 && (Time.now() - mtime < 600000L)) {
- LOG.info("Cached ScmUsed found for {} : {} ", dataLocation,
- cachedScmUsed);
- return cachedScmUsed;
- }
- return -1;
- } finally {
- sc.close();
- }
- }
-
- /**
- * Write the current scmUsed to the cache file.
- */
- void saveScmUsed() {
- if (scmUsedFile.exists() && !scmUsedFile.delete()) {
- LOG.warn("Failed to delete old scmUsed file in {}.", dataLocation);
- }
- OutputStreamWriter out = null;
- try {
- long used = getScmUsed();
- if (used > 0) {
- out = new OutputStreamWriter(new FileOutputStream(scmUsedFile),
- StandardCharsets.UTF_8);
- // mtime is written last, so that truncated writes won't be valid.
- out.write(Long.toString(used) + " " + Long.toString(Time.now()));
- out.flush();
- out.close();
- out = null;
- }
- } catch (IOException ioe) {
- // If write failed, the volume might be bad. Since the cache file is
- // not critical, log the error and continue.
- LOG.warn("Failed to write scmUsed to " + scmUsedFile, ioe);
- } finally {
- IOUtils.cleanupWithLogger(null, out);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b832f3c/hadoop-hdsl/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/Dispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdsl/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/Dispatcher.java b/hadoop-hdsl/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/Dispatcher.java
deleted file mode 100644
index a1690b5..0000000
--- a/hadoop-hdsl/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/Dispatcher.java
+++ /dev/null
@@ -1,708 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.container.common.impl;
-
-import com.google.common.base.Preconditions;
-import com.google.protobuf.ByteString;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdsl.protocol.proto.ContainerProtos;
-import org.apache.hadoop.hdsl.protocol.proto.ContainerProtos
- .ContainerCommandRequestProto;
-import org.apache.hadoop.hdsl.protocol.proto.ContainerProtos
- .ContainerCommandResponseProto;
-import org.apache.hadoop.hdsl.protocol.proto.ContainerProtos.Type;
-import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
-import org.apache.hadoop.ozone.container.common.helpers.ChunkUtils;
-import org.apache.hadoop.ozone.container.common.helpers.ContainerData;
-import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics;
-import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
-import org.apache.hadoop.ozone.container.common.helpers.FileUtils;
-import org.apache.hadoop.ozone.container.common.helpers.KeyData;
-import org.apache.hadoop.ozone.container.common.helpers.KeyUtils;
-import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
-import org.apache.hadoop.ozone.container.common.interfaces.ContainerManager;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.scm.container.common.helpers.StorageContainerException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.security.NoSuchAlgorithmException;
-import java.util.LinkedList;
-import java.util.List;
-
-import static org.apache.hadoop.hdsl.protocol.proto.ContainerProtos.Result.CLOSED_CONTAINER_IO;
-import static org.apache.hadoop.hdsl.protocol.proto.ContainerProtos.Result.GET_SMALL_FILE_ERROR;
-import static org.apache.hadoop.hdsl.protocol.proto.ContainerProtos.Result.NO_SUCH_ALGORITHM;
-import static org.apache.hadoop.hdsl.protocol.proto.ContainerProtos.Result.PUT_SMALL_FILE_ERROR;
-
-/**
- * Ozone Container dispatcher takes a call from the netty server and routes it
- * to the right handler function.
- */
-public class Dispatcher implements ContainerDispatcher {
- static final Logger LOG = LoggerFactory.getLogger(Dispatcher.class);
-
- private final ContainerManager containerManager;
- private ContainerMetrics metrics;
- private Configuration conf;
-
- /**
- * Constructs an OzoneContainer that receives calls from
- * XceiverServerHandler.
- *
- * @param containerManager - A class that manages containers.
- */
- public Dispatcher(ContainerManager containerManager, Configuration config) {
- Preconditions.checkNotNull(containerManager);
- this.containerManager = containerManager;
- this.metrics = null;
- this.conf = config;
- }
-
- @Override
- public void init() {
- this.metrics = ContainerMetrics.create(conf);
- }
-
- @Override
- public void shutdown() {
- }
-
- @Override
- public ContainerCommandResponseProto dispatch(
- ContainerCommandRequestProto msg) {
- LOG.trace("Command {}, trace ID: {} ", msg.getCmdType().toString(),
- msg.getTraceID());
- long startNanos = System.nanoTime();
- ContainerCommandResponseProto resp = null;
- try {
- Preconditions.checkNotNull(msg);
- Type cmdType = msg.getCmdType();
- metrics.incContainerOpcMetrics(cmdType);
- if ((cmdType == Type.CreateContainer) ||
- (cmdType == Type.DeleteContainer) ||
- (cmdType == Type.ReadContainer) ||
- (cmdType == Type.ListContainer) ||
- (cmdType == Type.UpdateContainer) ||
- (cmdType == Type.CloseContainer)) {
- resp = containerProcessHandler(msg);
- }
-
- if ((cmdType == Type.PutKey) ||
- (cmdType == Type.GetKey) ||
- (cmdType == Type.DeleteKey) ||
- (cmdType == Type.ListKey)) {
- resp = keyProcessHandler(msg);
- }
-
- if ((cmdType == Type.WriteChunk) ||
- (cmdType == Type.ReadChunk) ||
- (cmdType == Type.DeleteChunk)) {
- resp = chunkProcessHandler(msg);
- }
-
- if ((cmdType == Type.PutSmallFile) ||
- (cmdType == Type.GetSmallFile)) {
- resp = smallFileHandler(msg);
- }
-
- if (resp != null) {
- metrics.incContainerOpsLatencies(cmdType,
- System.nanoTime() - startNanos);
- return resp;
- }
-
- return ContainerUtils.unsupportedRequest(msg);
- } catch (StorageContainerException e) {
- // This useful since the trace ID will allow us to correlate failures.
- return ContainerUtils.logAndReturnError(LOG, e, msg);
- } catch (IllegalStateException | NullPointerException e) {
- return ContainerUtils.logAndReturnError(LOG, e, msg);
- }
- }
-
- public ContainerMetrics getContainerMetrics() {
- return metrics;
- }
-
- /**
- * Handles the all Container related functionality.
- *
- * @param msg - command
- * @return - response
- * @throws StorageContainerException
- */
- private ContainerCommandResponseProto containerProcessHandler(
- ContainerCommandRequestProto msg) throws StorageContainerException {
- try {
-
- switch (msg.getCmdType()) {
- case CreateContainer:
- return handleCreateContainer(msg);
-
- case DeleteContainer:
- return handleDeleteContainer(msg);
-
- case ListContainer:
- // TODO : Support List Container.
- return ContainerUtils.unsupportedRequest(msg);
-
- case UpdateContainer:
- return handleUpdateContainer(msg);
-
- case ReadContainer:
- return handleReadContainer(msg);
-
- case CloseContainer:
- return handleCloseContainer(msg);
-
- default:
- return ContainerUtils.unsupportedRequest(msg);
- }
- } catch (StorageContainerException e) {
- return ContainerUtils.logAndReturnError(LOG, e, msg);
- } catch (IOException ex) {
- LOG.warn("Container operation failed. " +
- "Container: {} Operation: {} trace ID: {} Error: {}",
- msg.getCreateContainer().getContainerData().getName(),
- msg.getCmdType().name(),
- msg.getTraceID(),
- ex.toString(), ex);
-
- // TODO : Replace with finer error codes.
- return ContainerUtils.getContainerResponse(msg,
- ContainerProtos.Result.CONTAINER_INTERNAL_ERROR,
- ex.toString()).build();
- }
- }
-
- /**
- * Handles the all key related functionality.
- *
- * @param msg - command
- * @return - response
- * @throws StorageContainerException
- */
- private ContainerCommandResponseProto keyProcessHandler(
- ContainerCommandRequestProto msg) throws StorageContainerException {
- try {
- switch (msg.getCmdType()) {
- case PutKey:
- return handlePutKey(msg);
-
- case GetKey:
- return handleGetKey(msg);
-
- case DeleteKey:
- return handleDeleteKey(msg);
-
- case ListKey:
- return ContainerUtils.unsupportedRequest(msg);
-
- default:
- return ContainerUtils.unsupportedRequest(msg);
-
- }
- } catch (StorageContainerException e) {
- return ContainerUtils.logAndReturnError(LOG, e, msg);
- } catch (IOException ex) {
- LOG.warn("Container operation failed. " +
- "Container: {} Operation: {} trace ID: {} Error: {}",
- msg.getCreateContainer().getContainerData().getName(),
- msg.getCmdType().name(),
- msg.getTraceID(),
- ex.toString(), ex);
-
- // TODO : Replace with finer error codes.
- return ContainerUtils.getContainerResponse(msg,
- ContainerProtos.Result.CONTAINER_INTERNAL_ERROR,
- ex.toString()).build();
- }
- }
-
- /**
- * Handles the all chunk related functionality.
- *
- * @param msg - command
- * @return - response
- * @throws StorageContainerException
- */
- private ContainerCommandResponseProto chunkProcessHandler(
- ContainerCommandRequestProto msg) throws StorageContainerException {
- try {
- switch (msg.getCmdType()) {
- case WriteChunk:
- return handleWriteChunk(msg);
-
- case ReadChunk:
- return handleReadChunk(msg);
-
- case DeleteChunk:
- return handleDeleteChunk(msg);
-
- case ListChunk:
- return ContainerUtils.unsupportedRequest(msg);
-
- default:
- return ContainerUtils.unsupportedRequest(msg);
- }
- } catch (StorageContainerException e) {
- return ContainerUtils.logAndReturnError(LOG, e, msg);
- } catch (IOException ex) {
- LOG.warn("Container operation failed. " +
- "Container: {} Operation: {} trace ID: {} Error: {}",
- msg.getCreateContainer().getContainerData().getName(),
- msg.getCmdType().name(),
- msg.getTraceID(),
- ex.toString(), ex);
-
- // TODO : Replace with finer error codes.
- return ContainerUtils.getContainerResponse(msg,
- ContainerProtos.Result.CONTAINER_INTERNAL_ERROR,
- ex.toString()).build();
- }
- }
-
- /**
- * Dispatch calls to small file hanlder.
- * @param msg - request
- * @return response
- * @throws StorageContainerException
- */
- private ContainerCommandResponseProto smallFileHandler(
- ContainerCommandRequestProto msg) throws StorageContainerException {
- switch (msg.getCmdType()) {
- case PutSmallFile:
- return handlePutSmallFile(msg);
- case GetSmallFile:
- return handleGetSmallFile(msg);
- default:
- return ContainerUtils.unsupportedRequest(msg);
- }
- }
-
- /**
- * Update an existing container with the new container data.
- *
- * @param msg Request
- * @return ContainerCommandResponseProto
- * @throws IOException
- */
- private ContainerCommandResponseProto handleUpdateContainer(
- ContainerCommandRequestProto msg)
- throws IOException {
- if (!msg.hasUpdateContainer()) {
- LOG.debug("Malformed read container request. trace ID: {}",
- msg.getTraceID());
- return ContainerUtils.malformedRequest(msg);
- }
-
- Pipeline pipeline = Pipeline.getFromProtoBuf(
- msg.getUpdateContainer().getPipeline());
- String containerName = msg.getUpdateContainer()
- .getContainerData().getName();
-
- ContainerData data = ContainerData.getFromProtBuf(
- msg.getUpdateContainer().getContainerData(), conf);
- boolean forceUpdate = msg.getUpdateContainer().getForceUpdate();
- this.containerManager.updateContainer(
- pipeline, containerName, data, forceUpdate);
- return ContainerUtils.getContainerResponse(msg);
- }
-
- /**
- * Calls into container logic and returns appropriate response.
- *
- * @param msg - Request
- * @return ContainerCommandResponseProto
- * @throws IOException
- */
- private ContainerCommandResponseProto handleReadContainer(
- ContainerCommandRequestProto msg)
- throws IOException {
-
- if (!msg.hasReadContainer()) {
- LOG.debug("Malformed read container request. trace ID: {}",
- msg.getTraceID());
- return ContainerUtils.malformedRequest(msg);
- }
-
- String name = msg.getReadContainer().getName();
- ContainerData container = this.containerManager.readContainer(name);
- return ContainerUtils.getReadContainerResponse(msg, container);
- }
-
- /**
- * Calls into container logic and returns appropriate response.
- *
- * @param msg - Request
- * @return Response.
- * @throws IOException
- */
- private ContainerCommandResponseProto handleDeleteContainer(
- ContainerCommandRequestProto msg) throws IOException {
-
- if (!msg.hasDeleteContainer()) {
- LOG.debug("Malformed delete container request. trace ID: {}",
- msg.getTraceID());
- return ContainerUtils.malformedRequest(msg);
- }
-
- Pipeline pipeline = Pipeline.getFromProtoBuf(
- msg.getDeleteContainer().getPipeline());
- Preconditions.checkNotNull(pipeline);
- String name = msg.getDeleteContainer().getName();
- boolean forceDelete = msg.getDeleteContainer().getForceDelete();
- this.containerManager.deleteContainer(pipeline, name, forceDelete);
- return ContainerUtils.getContainerResponse(msg);
- }
-
- /**
- * Calls into container logic and returns appropriate response.
- *
- * @param msg - Request
- * @return Response.
- * @throws IOException
- */
- private ContainerCommandResponseProto handleCreateContainer(
- ContainerCommandRequestProto msg) throws IOException {
- if (!msg.hasCreateContainer()) {
- LOG.debug("Malformed create container request. trace ID: {}",
- msg.getTraceID());
- return ContainerUtils.malformedRequest(msg);
- }
- ContainerData cData = ContainerData.getFromProtBuf(
- msg.getCreateContainer().getContainerData(), conf);
- Preconditions.checkNotNull(cData, "Container data is null");
-
- Pipeline pipeline = Pipeline.getFromProtoBuf(
- msg.getCreateContainer().getPipeline());
- Preconditions.checkNotNull(pipeline, "Pipeline cannot be null");
-
- this.containerManager.createContainer(pipeline, cData);
- return ContainerUtils.getContainerResponse(msg);
- }
-
- /**
- * closes an open container.
- *
- * @param msg -
- * @return
- * @throws IOException
- */
- private ContainerCommandResponseProto handleCloseContainer(
- ContainerCommandRequestProto msg) throws IOException {
- try {
- if (!msg.hasCloseContainer()) {
- LOG.debug("Malformed close Container request. trace ID: {}",
- msg.getTraceID());
- return ContainerUtils.malformedRequest(msg);
- }
- Pipeline pipeline = Pipeline.getFromProtoBuf(msg.getCloseContainer()
- .getPipeline());
- Preconditions.checkNotNull(pipeline, "Pipeline cannot be null");
- if (!this.containerManager.isOpen(pipeline.getContainerName())) {
- throw new StorageContainerException("Attempting to close a closed " +
- "container.", CLOSED_CONTAINER_IO);
- }
- this.containerManager.closeContainer(pipeline.getContainerName());
- return ContainerUtils.getContainerResponse(msg);
- } catch (NoSuchAlgorithmException e) {
- throw new StorageContainerException("No such Algorithm", e,
- NO_SUCH_ALGORITHM);
- }
- }
-
- /**
- * Calls into chunk manager to write a chunk.
- *
- * @param msg - Request.
- * @return Response.
- * @throws IOException
- */
- private ContainerCommandResponseProto handleWriteChunk(
- ContainerCommandRequestProto msg) throws IOException {
- if (!msg.hasWriteChunk()) {
- LOG.debug("Malformed write chunk request. trace ID: {}",
- msg.getTraceID());
- return ContainerUtils.malformedRequest(msg);
- }
- String keyName = msg.getWriteChunk().getKeyName();
- Pipeline pipeline = Pipeline.getFromProtoBuf(
- msg.getWriteChunk().getPipeline());
- Preconditions.checkNotNull(pipeline);
- if (!this.containerManager.isOpen(pipeline.getContainerName())) {
- throw new StorageContainerException("Write to closed container.",
- CLOSED_CONTAINER_IO);
- }
-
- ChunkInfo chunkInfo = ChunkInfo.getFromProtoBuf(msg.getWriteChunk()
- .getChunkData());
- Preconditions.checkNotNull(chunkInfo);
- byte[] data = null;
- if (msg.getWriteChunk().getStage() == ContainerProtos.Stage.WRITE_DATA
- || msg.getWriteChunk().getStage() == ContainerProtos.Stage.COMBINED) {
- data = msg.getWriteChunk().getData().toByteArray();
- metrics.incContainerBytesStats(Type.WriteChunk, data.length);
-
- }
- this.containerManager.getChunkManager()
- .writeChunk(pipeline, keyName, chunkInfo,
- data, msg.getWriteChunk().getStage());
-
- return ChunkUtils.getChunkResponse(msg);
- }
-
- /**
- * Calls into chunk manager to read a chunk.
- *
- * @param msg - Request.
- * @return - Response.
- * @throws IOException
- */
- private ContainerCommandResponseProto handleReadChunk(
- ContainerCommandRequestProto msg) throws IOException {
- if (!msg.hasReadChunk()) {
- LOG.debug("Malformed read chunk request. trace ID: {}",
- msg.getTraceID());
- return ContainerUtils.malformedRequest(msg);
- }
-
- String keyName = msg.getReadChunk().getKeyName();
- Pipeline pipeline = Pipeline.getFromProtoBuf(
- msg.getReadChunk().getPipeline());
- Preconditions.checkNotNull(pipeline);
-
- ChunkInfo chunkInfo = ChunkInfo.getFromProtoBuf(msg.getReadChunk()
- .getChunkData());
- Preconditions.checkNotNull(chunkInfo);
- byte[] data = this.containerManager.getChunkManager().readChunk(pipeline,
- keyName, chunkInfo);
- metrics.incContainerBytesStats(Type.ReadChunk, data.length);
- return ChunkUtils.getReadChunkResponse(msg, data, chunkInfo);
- }
-
- /**
- * Calls into chunk manager to write a chunk.
- *
- * @param msg - Request.
- * @return Response.
- * @throws IOException
- */
- private ContainerCommandResponseProto handleDeleteChunk(
- ContainerCommandRequestProto msg) throws IOException {
- if (!msg.hasDeleteChunk()) {
- LOG.debug("Malformed delete chunk request. trace ID: {}",
- msg.getTraceID());
- return ContainerUtils.malformedRequest(msg);
- }
-
- String keyName = msg.getDeleteChunk().getKeyName();
- Pipeline pipeline = Pipeline.getFromProtoBuf(
- msg.getDeleteChunk().getPipeline());
- Preconditions.checkNotNull(pipeline);
- if (!this.containerManager.isOpen(pipeline.getContainerName())) {
- throw new StorageContainerException("Write to closed container.",
- CLOSED_CONTAINER_IO);
- }
- ChunkInfo chunkInfo = ChunkInfo.getFromProtoBuf(msg.getDeleteChunk()
- .getChunkData());
- Preconditions.checkNotNull(chunkInfo);
-
- this.containerManager.getChunkManager().deleteChunk(pipeline, keyName,
- chunkInfo);
- return ChunkUtils.getChunkResponse(msg);
- }
-
- /**
- * Put Key handler.
- *
- * @param msg - Request.
- * @return - Response.
- * @throws IOException
- */
- private ContainerCommandResponseProto handlePutKey(
- ContainerCommandRequestProto msg) throws IOException {
- if (!msg.hasPutKey()) {
- LOG.debug("Malformed put key request. trace ID: {}",
- msg.getTraceID());
- return ContainerUtils.malformedRequest(msg);
- }
- Pipeline pipeline = Pipeline.getFromProtoBuf(msg.getPutKey().getPipeline());
- Preconditions.checkNotNull(pipeline);
- if (!this.containerManager.isOpen(pipeline.getContainerName())) {
- throw new StorageContainerException("Write to closed container.",
- CLOSED_CONTAINER_IO);
- }
- KeyData keyData = KeyData.getFromProtoBuf(msg.getPutKey().getKeyData());
- Preconditions.checkNotNull(keyData);
- this.containerManager.getKeyManager().putKey(pipeline, keyData);
- long numBytes = keyData.getProtoBufMessage().toByteArray().length;
- metrics.incContainerBytesStats(Type.PutKey, numBytes);
- return KeyUtils.getKeyResponse(msg);
- }
-
- /**
- * Handle Get Key.
- *
- * @param msg - Request.
- * @return - Response.
- * @throws IOException
- */
- private ContainerCommandResponseProto handleGetKey(
- ContainerCommandRequestProto msg) throws IOException {
- if (!msg.hasGetKey()) {
- LOG.debug("Malformed get key request. trace ID: {}",
- msg.getTraceID());
- return ContainerUtils.malformedRequest(msg);
- }
- KeyData keyData = KeyData.getFromProtoBuf(msg.getGetKey().getKeyData());
- Preconditions.checkNotNull(keyData);
- KeyData responseData =
- this.containerManager.getKeyManager().getKey(keyData);
- long numBytes = responseData.getProtoBufMessage().toByteArray().length;
- metrics.incContainerBytesStats(Type.GetKey, numBytes);
- return KeyUtils.getKeyDataResponse(msg, responseData);
- }
-
- /**
- * Handle Delete Key.
- *
- * @param msg - Request.
- * @return - Response.
- * @throws IOException
- */
- private ContainerCommandResponseProto handleDeleteKey(
- ContainerCommandRequestProto msg) throws IOException {
- if (!msg.hasDeleteKey()) {
- LOG.debug("Malformed delete key request. trace ID: {}",
- msg.getTraceID());
- return ContainerUtils.malformedRequest(msg);
- }
- Pipeline pipeline =
- Pipeline.getFromProtoBuf(msg.getDeleteKey().getPipeline());
- Preconditions.checkNotNull(pipeline);
- if (!this.containerManager.isOpen(pipeline.getContainerName())) {
- throw new StorageContainerException("Write to closed container.",
- CLOSED_CONTAINER_IO);
- }
- String keyName = msg.getDeleteKey().getName();
- Preconditions.checkNotNull(keyName);
- Preconditions.checkState(!keyName.isEmpty());
- this.containerManager.getKeyManager().deleteKey(pipeline, keyName);
- return KeyUtils.getKeyResponse(msg);
- }
-
- /**
- * Handles writing a chunk and associated key using single RPC.
- *
- * @param msg - Message.
- * @return ContainerCommandResponseProto
- * @throws StorageContainerException
- */
- private ContainerCommandResponseProto handlePutSmallFile(
- ContainerCommandRequestProto msg) throws StorageContainerException {
-
- if (!msg.hasPutSmallFile()) {
- LOG.debug("Malformed put small file request. trace ID: {}",
- msg.getTraceID());
- return ContainerUtils.malformedRequest(msg);
- }
- try {
-
- Pipeline pipeline =
- Pipeline.getFromProtoBuf(msg.getPutSmallFile()
- .getKey().getPipeline());
-
- Preconditions.checkNotNull(pipeline);
- if (!this.containerManager.isOpen(pipeline.getContainerName())) {
- throw new StorageContainerException("Write to closed container.",
- CLOSED_CONTAINER_IO);
- }
- KeyData keyData = KeyData.getFromProtoBuf(msg.getPutSmallFile().getKey()
- .getKeyData());
- ChunkInfo chunkInfo = ChunkInfo.getFromProtoBuf(msg.getPutSmallFile()
- .getChunkInfo());
- byte[] data = msg.getPutSmallFile().getData().toByteArray();
-
- metrics.incContainerBytesStats(Type.PutSmallFile, data.length);
- this.containerManager.getChunkManager().writeChunk(pipeline, keyData
- .getKeyName(), chunkInfo, data, ContainerProtos.Stage.COMBINED);
- List<ContainerProtos.ChunkInfo> chunks = new LinkedList<>();
- chunks.add(chunkInfo.getProtoBufMessage());
- keyData.setChunks(chunks);
- this.containerManager.getKeyManager().putKey(pipeline, keyData);
- return FileUtils.getPutFileResponse(msg);
- } catch (StorageContainerException e) {
- return ContainerUtils.logAndReturnError(LOG, e, msg);
- } catch (IOException e) {
- throw new StorageContainerException("Put Small File Failed.", e,
- PUT_SMALL_FILE_ERROR);
- }
- }
-
- /**
- * Handles getting a data stream using a key. This helps in reducing the RPC
- * overhead for small files.
- *
- * @param msg - ContainerCommandRequestProto
- * @return ContainerCommandResponseProto
- * @throws StorageContainerException
- */
- private ContainerCommandResponseProto handleGetSmallFile(
- ContainerCommandRequestProto msg) throws StorageContainerException {
- ByteString dataBuf = ByteString.EMPTY;
- if (!msg.hasGetSmallFile()) {
- LOG.debug("Malformed get small file request. trace ID: {}",
- msg.getTraceID());
- return ContainerUtils.malformedRequest(msg);
- }
- try {
- Pipeline pipeline =
- Pipeline.getFromProtoBuf(msg.getGetSmallFile()
- .getKey().getPipeline());
-
- long bytes = 0;
- Preconditions.checkNotNull(pipeline);
- KeyData keyData = KeyData.getFromProtoBuf(msg.getGetSmallFile()
- .getKey().getKeyData());
- KeyData data = this.containerManager.getKeyManager().getKey(keyData);
- ContainerProtos.ChunkInfo c = null;
- for (ContainerProtos.ChunkInfo chunk : data.getChunks()) {
- bytes += chunk.getSerializedSize();
- ByteString current =
- ByteString.copyFrom(this.containerManager.getChunkManager()
- .readChunk(
- pipeline, keyData.getKeyName(), ChunkInfo.getFromProtoBuf(
- chunk)));
- dataBuf = dataBuf.concat(current);
- c = chunk;
- }
- metrics.incContainerBytesStats(Type.GetSmallFile, bytes);
- return FileUtils.getGetSmallFileResponse(msg, dataBuf.toByteArray(),
- ChunkInfo.getFromProtoBuf(c));
- } catch (StorageContainerException e) {
- return ContainerUtils.logAndReturnError(LOG, e, msg);
- } catch (IOException e) {
- throw new StorageContainerException("Get Small File Failed", e,
- GET_SMALL_FILE_ERROR);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b832f3c/hadoop-hdsl/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/KeyManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdsl/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/KeyManagerImpl.java b/hadoop-hdsl/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/KeyManagerImpl.java
deleted file mode 100644
index c195ad2..0000000
--- a/hadoop-hdsl/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/KeyManagerImpl.java
+++ /dev/null
@@ -1,201 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.container.common.impl;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.hdsl.protocol.proto.ContainerProtos;
-import org.apache.hadoop.ozone.container.common.helpers.ContainerData;
-import org.apache.hadoop.ozone.container.common.helpers.KeyData;
-import org.apache.hadoop.ozone.container.common.helpers.KeyUtils;
-import org.apache.hadoop.ozone.container.common.interfaces.ContainerManager;
-import org.apache.hadoop.ozone.container.common.interfaces.KeyManager;
-import org.apache.hadoop.ozone.container.common.utils.ContainerCache;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.scm.container.common.helpers.StorageContainerException;
-import org.apache.hadoop.utils.MetadataKeyFilters.KeyPrefixFilter;
-import org.apache.hadoop.utils.MetadataKeyFilters.MetadataKeyFilter;
-import org.apache.hadoop.utils.MetadataStore;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-import static org.apache.hadoop.hdsl.protocol.proto.ContainerProtos
- .Result.NO_SUCH_KEY;
-
-/**
- * Key Manager impl.
- */
-public class KeyManagerImpl implements KeyManager {
- static final Logger LOG =
- LoggerFactory.getLogger(KeyManagerImpl.class);
-
- private static final float LOAD_FACTOR = 0.75f;
- private final ContainerManager containerManager;
- private final Configuration conf;
-
- /**
- * Constructs a key Manager.
- *
- * @param containerManager - Container Manager.
- */
- public KeyManagerImpl(ContainerManager containerManager, Configuration conf) {
- Preconditions.checkNotNull(containerManager, "Container manager cannot be" +
- " null");
- Preconditions.checkNotNull(conf, "Config cannot be null");
- this.containerManager = containerManager;
- this.conf = conf;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public void putKey(Pipeline pipeline, KeyData data) throws IOException {
- containerManager.readLock();
- try {
- // We are not locking the key manager since LevelDb serializes all actions
- // against a single DB. We rely on DB level locking to avoid conflicts.
- Preconditions.checkNotNull(pipeline, "Pipeline cannot be null");
- String containerName = pipeline.getContainerName();
- Preconditions.checkNotNull(containerName,
- "Container name cannot be null");
- ContainerData cData = containerManager.readContainer(containerName);
- MetadataStore db = KeyUtils.getDB(cData, conf);
-
- // This is a post condition that acts as a hint to the user.
- // Should never fail.
- Preconditions.checkNotNull(db, "DB cannot be null here");
- db.put(data.getKeyName().getBytes(KeyUtils.ENCODING), data
- .getProtoBufMessage().toByteArray());
- } finally {
- containerManager.readUnlock();
- }
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public KeyData getKey(KeyData data) throws IOException {
- containerManager.readLock();
- try {
- Preconditions.checkNotNull(data, "Key data cannot be null");
- Preconditions.checkNotNull(data.getContainerName(),
- "Container name cannot be null");
- ContainerData cData = containerManager.readContainer(data
- .getContainerName());
- MetadataStore db = KeyUtils.getDB(cData, conf);
-
- // This is a post condition that acts as a hint to the user.
- // Should never fail.
- Preconditions.checkNotNull(db, "DB cannot be null here");
-
- byte[] kData = db.get(data.getKeyName().getBytes(KeyUtils.ENCODING));
- if (kData == null) {
- throw new StorageContainerException("Unable to find the key.",
- NO_SUCH_KEY);
- }
- ContainerProtos.KeyData keyData =
- ContainerProtos.KeyData.parseFrom(kData);
- return KeyData.getFromProtoBuf(keyData);
- } finally {
- containerManager.readUnlock();
- }
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public void deleteKey(Pipeline pipeline, String keyName)
- throws IOException {
- containerManager.readLock();
- try {
- Preconditions.checkNotNull(pipeline, "Pipeline cannot be null");
- String containerName = pipeline.getContainerName();
- Preconditions.checkNotNull(containerName,
- "Container name cannot be null");
- ContainerData cData = containerManager.readContainer(containerName);
- MetadataStore db = KeyUtils.getDB(cData, conf);
-
- // This is a post condition that acts as a hint to the user.
- // Should never fail.
- Preconditions.checkNotNull(db, "DB cannot be null here");
- // Note : There is a race condition here, since get and delete
- // are not atomic. Leaving it here since the impact is refusing
- // to delete a key which might have just gotten inserted after
- // the get check.
-
- byte[] kData = db.get(keyName.getBytes(KeyUtils.ENCODING));
- if (kData == null) {
- throw new StorageContainerException("Unable to find the key.",
- NO_SUCH_KEY);
- }
- db.delete(keyName.getBytes(KeyUtils.ENCODING));
- } finally {
- containerManager.readUnlock();
- }
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public List<KeyData> listKey(
- Pipeline pipeline, String prefix, String startKey, int count)
- throws IOException {
- Preconditions.checkNotNull(pipeline,
- "Pipeline cannot be null.");
- Preconditions.checkArgument(count > 0,
- "Count must be a positive number.");
- ContainerData cData = containerManager.readContainer(pipeline
- .getContainerName());
- MetadataStore db = KeyUtils.getDB(cData, conf);
-
- List<KeyData> result = new ArrayList<KeyData>();
- byte[] startKeyInBytes = startKey == null ? null :
- DFSUtil.string2Bytes(startKey);
- MetadataKeyFilter prefixFilter = new KeyPrefixFilter(prefix);
- List<Map.Entry<byte[], byte[]>> range =
- db.getSequentialRangeKVs(startKeyInBytes, count, prefixFilter);
- for (Map.Entry<byte[], byte[]> entry : range) {
- String keyName = KeyUtils.getKeyName(entry.getKey());
- KeyData value = KeyUtils.getKeyData(entry.getValue());
- KeyData data = new KeyData(value.getContainerName(), keyName);
- result.add(data);
- }
- return result;
- }
-
- /**
- * Shutdown keyManager.
- */
- @Override
- public void shutdown() {
- Preconditions.checkState(this.containerManager.hasWriteLock(), "asserts " +
- "that we are holding the container manager lock when shutting down.");
- KeyUtils.shutdownCache(ContainerCache.getInstance(conf));
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b832f3c/hadoop-hdsl/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/RandomContainerDeletionChoosingPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdsl/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/RandomContainerDeletionChoosingPolicy.java b/hadoop-hdsl/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/RandomContainerDeletionChoosingPolicy.java
deleted file mode 100644
index 0123ab1..0000000
--- a/hadoop-hdsl/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/RandomContainerDeletionChoosingPolicy.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.container.common.impl;
-
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.ozone.container.common.helpers.ContainerData;
-import org.apache.hadoop.ozone.container.common.interfaces.ContainerDeletionChoosingPolicy;
-import org.apache.hadoop.scm.container.common.helpers.StorageContainerException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.base.Preconditions;
-
-/**
- * Randomly choosing containers for block deletion.
- */
-public class RandomContainerDeletionChoosingPolicy
- implements ContainerDeletionChoosingPolicy {
- private static final Logger LOG =
- LoggerFactory.getLogger(RandomContainerDeletionChoosingPolicy.class);
-
- @Override
- public List<ContainerData> chooseContainerForBlockDeletion(int count,
- Map<String, ContainerStatus> candidateContainers)
- throws StorageContainerException {
- Preconditions.checkNotNull(candidateContainers,
- "Internal assertion: candidate containers cannot be null");
-
- int currentCount = 0;
- List<ContainerData> result = new LinkedList<>();
- ContainerStatus[] values = new ContainerStatus[candidateContainers.size()];
- // to get a shuffle list
- for (ContainerStatus entry : DFSUtil.shuffle(
- candidateContainers.values().toArray(values))) {
- if (currentCount < count) {
- result.add(entry.getContainer());
- currentCount++;
-
- LOG.debug("Select container {} for block deletion, "
- + "pending deletion blocks num: {}.",
- entry.getContainer().getContainerName(),
- entry.getNumPendingDeletionBlocks());
- } else {
- break;
- }
- }
-
- return result;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b832f3c/hadoop-hdsl/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/StorageLocationReport.java
----------------------------------------------------------------------
diff --git a/hadoop-hdsl/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/StorageLocationReport.java b/hadoop-hdsl/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/StorageLocationReport.java
deleted file mode 100644
index 7ef91a9..0000000
--- a/hadoop-hdsl/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/StorageLocationReport.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.container.common.impl;
-
-/**
- * Storage location stats of datanodes that provide back store for containers.
- *
- */
-public class StorageLocationReport {
- public static final StorageLocationReport[] EMPTY_ARRAY = {};
-
- private final String id;
- private final boolean failed;
- private final long capacity;
- private final long scmUsed;
- private final long remaining;
-
- public StorageLocationReport(String id, boolean failed,
- long capacity, long scmUsed, long remaining) {
- this.id = id;
- this.failed = failed;
- this.capacity = capacity;
- this.scmUsed = scmUsed;
- this.remaining = remaining;
- }
-
- public String getId() {
- return id;
- }
-
- public boolean isFailed() {
- return failed;
- }
-
- public long getCapacity() {
- return capacity;
- }
-
- public long getScmUsed() {
- return scmUsed;
- }
-
- public long getRemaining() {
- return remaining;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b832f3c/hadoop-hdsl/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/TopNOrderedContainerDeletionChoosingPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdsl/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/TopNOrderedContainerDeletionChoosingPolicy.java b/hadoop-hdsl/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/TopNOrderedContainerDeletionChoosingPolicy.java
deleted file mode 100644
index 3f4cdaa..0000000
--- a/hadoop-hdsl/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/TopNOrderedContainerDeletionChoosingPolicy.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.container.common.impl;
-
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.hadoop.ozone.container.common.helpers.ContainerData;
-import org.apache.hadoop.ozone.container.common.interfaces.ContainerDeletionChoosingPolicy;
-import org.apache.hadoop.scm.container.common.helpers.StorageContainerException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.base.Preconditions;
-
-/**
- * TopN Ordered choosing policy that choosing containers based on pending
- * deletion blocks' number.
- */
-public class TopNOrderedContainerDeletionChoosingPolicy
- implements ContainerDeletionChoosingPolicy {
- private static final Logger LOG =
- LoggerFactory.getLogger(TopNOrderedContainerDeletionChoosingPolicy.class);
-
- /** customized comparator used to compare differentiate container status. **/
- private static final Comparator<ContainerStatus> CONTAINER_STATUS_COMPARATOR
- = new Comparator<ContainerStatus>() {
- @Override
- public int compare(ContainerStatus c1, ContainerStatus c2) {
- return Integer.compare(c2.getNumPendingDeletionBlocks(),
- c1.getNumPendingDeletionBlocks());
- }
- };
-
- @Override
- public List<ContainerData> chooseContainerForBlockDeletion(int count,
- Map<String, ContainerStatus> candidateContainers)
- throws StorageContainerException {
- Preconditions.checkNotNull(candidateContainers,
- "Internal assertion: candidate containers cannot be null");
-
- List<ContainerData> result = new LinkedList<>();
- List<ContainerStatus> orderedList = new LinkedList<>();
- orderedList.addAll(candidateContainers.values());
- Collections.sort(orderedList, CONTAINER_STATUS_COMPARATOR);
-
- // get top N list ordered by pending deletion blocks' number
- int currentCount = 0;
- for (ContainerStatus entry : orderedList) {
- if (currentCount < count) {
- if (entry.getNumPendingDeletionBlocks() > 0) {
- result.add(entry.getContainer());
- currentCount++;
-
- LOG.debug(
- "Select container {} for block deletion, "
- + "pending deletion blocks num: {}.",
- entry.getContainer().getContainerName(),
- entry.getNumPendingDeletionBlocks());
- } else {
- LOG.debug("Stop looking for next container, there is no"
- + " pending deletion block contained in remaining containers.");
- break;
- }
- } else {
- break;
- }
- }
-
- return result;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b832f3c/hadoop-hdsl/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdsl/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/package-info.java b/hadoop-hdsl/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/package-info.java
deleted file mode 100644
index 16da5d9..0000000
--- a/hadoop-hdsl/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.container.common.impl;
-
-/**
- This package is contains Ozone container implementation.
-**/
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b832f3c/hadoop-hdsl/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ChunkManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdsl/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ChunkManager.java b/hadoop-hdsl/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ChunkManager.java
deleted file mode 100644
index a7d3fe5..0000000
--- a/hadoop-hdsl/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ChunkManager.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.container.common.interfaces;
-
-import org.apache.hadoop.hdsl.protocol.proto.ContainerProtos;
-import org.apache.hadoop.scm.container.common.helpers.StorageContainerException;
-import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-
-/**
- * Chunk Manager allows read, write, delete and listing of chunks in
- * a container.
- */
-public interface ChunkManager {
-
- /**
- * writes a given chunk.
- * @param pipeline - Name and the set of machines that make this container.
- * @param keyName - Name of the Key.
- * @param info - ChunkInfo.
- * @param stage - Chunk Stage write.
- * @throws StorageContainerException
- */
- void writeChunk(Pipeline pipeline, String keyName,
- ChunkInfo info, byte[] data, ContainerProtos.Stage stage)
- throws StorageContainerException;
-
- /**
- * reads the data defined by a chunk.
- * @param pipeline - container pipeline.
- * @param keyName - Name of the Key
- * @param info - ChunkInfo.
- * @return byte array
- * @throws StorageContainerException
- *
- * TODO: Right now we do not support partial reads and writes of chunks.
- * TODO: Explore if we need to do that for ozone.
- */
- byte[] readChunk(Pipeline pipeline, String keyName, ChunkInfo info) throws
- StorageContainerException;
-
- /**
- * Deletes a given chunk.
- * @param pipeline - Pipeline.
- * @param keyName - Key Name
- * @param info - Chunk Info
- * @throws StorageContainerException
- */
- void deleteChunk(Pipeline pipeline, String keyName, ChunkInfo info) throws
- StorageContainerException;
-
- // TODO : Support list operations.
-
- /**
- * Shutdown the chunkManager.
- */
- void shutdown();
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b832f3c/hadoop-hdsl/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerDeletionChoosingPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdsl/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerDeletionChoosingPolicy.java b/hadoop-hdsl/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerDeletionChoosingPolicy.java
deleted file mode 100644
index 3e0a283..0000000
--- a/hadoop-hdsl/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerDeletionChoosingPolicy.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.container.common.interfaces;
-
-import java.util.List;
-import java.util.Map;
-
-import org.apache.hadoop.ozone.container.common.helpers.ContainerData;
-import org.apache.hadoop.ozone.container.common.impl.ContainerStatus;
-import org.apache.hadoop.scm.container.common.helpers.StorageContainerException;
-
-/**
- * This interface is used for choosing desired containers for
- * block deletion.
- */
-public interface ContainerDeletionChoosingPolicy {
-
- /**
- * Chooses desired containers for block deletion.
- * @param count
- * how many to return
- * @param candidateContainers
- * candidate containers collection
- * @return container data list
- * @throws StorageContainerException
- */
- List<ContainerData> chooseContainerForBlockDeletion(int count,
- Map<String, ContainerStatus> candidateContainers)
- throws StorageContainerException;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b832f3c/hadoop-hdsl/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerDispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdsl/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerDispatcher.java b/hadoop-hdsl/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerDispatcher.java
deleted file mode 100644
index cb3fdea..0000000
--- a/hadoop-hdsl/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerDispatcher.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.container.common.interfaces;
-
-import org.apache.hadoop.hdsl.protocol.proto.ContainerProtos.ContainerCommandResponseProto;
-import org.apache.hadoop.hdsl.protocol.proto.ContainerProtos.ContainerCommandRequestProto;
-
-/**
- * Dispatcher acts as the bridge between the transport layer and
- * the actual container layer. This layer is capable of transforming
- * protobuf objects into corresponding class and issue the function call
- * into the lower layers.
- *
- * The reply from the request is dispatched to the client.
- */
-public interface ContainerDispatcher {
- /**
- * Dispatches commands to container layer.
- * @param msg - Command Request
- * @return Command Response
- */
- ContainerCommandResponseProto dispatch(ContainerCommandRequestProto msg);
-
- /**
- * Initialize the Dispatcher.
- */
- void init();
-
- /**
- * Shutdown Dispatcher services.
- */
- void shutdown();
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org