You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by bh...@apache.org on 2018/07/09 20:33:25 UTC
[16/37] hadoop git commit: HDDS-173. Refactor Dispatcher and
implement Handler for new ContainerIO design.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13579f92/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerImpl.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerImpl.java
new file mode 100644
index 0000000..258d4da
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerImpl.java
@@ -0,0 +1,240 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.container.keyvalue.impl;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.hdds.client.BlockID;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
+import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
+import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
+import org.apache.hadoop.ozone.container.keyvalue.helpers.ChunkUtils;
+import org.apache.hadoop.ozone.container.common.impl.ChunkLayOutVersion;
+import org.apache.hadoop.ozone.container.keyvalue.interfaces.ChunkManager;
+import org.apache.hadoop.ozone.container.common.interfaces.Container;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.file.Files;
+import java.nio.file.StandardCopyOption;
+import java.security.NoSuchAlgorithmException;
+import java.util.concurrent.ExecutionException;
+
+import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
+ .Result.CONTAINER_INTERNAL_ERROR;
+import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
+ .Result.NO_SUCH_ALGORITHM;
+import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.UNSUPPORTED_REQUEST;
+
+/**
+ * This class is for performing chunk related operations.
+ */
+public class ChunkManagerImpl implements ChunkManager {
+ static final Logger LOG = LoggerFactory.getLogger(ChunkManagerImpl.class);
+
+ /**
+ * writes a given chunk.
+ *
+ * @param container - Container for the chunk
+ * @param blockID - ID of the block
+ * @param info - ChunkInfo
+ * @param data - data of the chunk
+ * @param stage - Stage of the Chunk operation
+ * @throws StorageContainerException
+ */
+ public void writeChunk(Container container, BlockID blockID, ChunkInfo info,
+ byte[] data, ContainerProtos.Stage stage)
+ throws StorageContainerException {
+
+ try {
+
+ KeyValueContainerData containerData = (KeyValueContainerData) container
+ .getContainerData();
+
+ File chunkFile = ChunkUtils.validateChunk(containerData, info);
+ File tmpChunkFile = getTmpChunkFile(chunkFile, info);
+
+ LOG.debug("writing chunk:{} chunk stage:{} chunk file:{} tmp chunk file",
+ info.getChunkName(), stage, chunkFile, tmpChunkFile);
+
+ switch (stage) {
+ case WRITE_DATA:
+ // Initially writes to temporary chunk file.
+ ChunkUtils.writeData(tmpChunkFile, info, data);
+ break;
+ case COMMIT_DATA:
+ // commit the data, means move chunk data from temporary chunk file
+ // to actual chunk file.
+ long sizeDiff = tmpChunkFile.length() - chunkFile.length();
+ commitChunk(tmpChunkFile, chunkFile);
+ containerData.incrBytesUsed(sizeDiff);
+ containerData.incrWriteCount();
+ containerData.incrWriteBytes(sizeDiff);
+ break;
+ case COMBINED:
+ // directly write to the chunk file
+ ChunkUtils.writeData(chunkFile, info, data);
+ containerData.incrBytesUsed(info.getLen());
+ containerData.incrWriteCount();
+ containerData.incrWriteBytes(info.getLen());
+ break;
+ default:
+ throw new IOException("Can not identify write operation.");
+ }
+ } catch (StorageContainerException ex) {
+ throw ex;
+ } catch (NoSuchAlgorithmException ex) {
+ LOG.error("write data failed. error: {}", ex);
+ throw new StorageContainerException("Internal error: ", ex,
+ NO_SUCH_ALGORITHM);
+ } catch (ExecutionException | IOException ex) {
+ LOG.error("write data failed. error: {}", ex);
+ throw new StorageContainerException("Internal error: ", ex,
+ CONTAINER_INTERNAL_ERROR);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ LOG.error("write data failed. error: {}", e);
+ throw new StorageContainerException("Internal error: ", e,
+ CONTAINER_INTERNAL_ERROR);
+ }
+ }
+
+ /**
+ * reads the data defined by a chunk.
+ *
+ * @param container - Container for the chunk
+ * @param blockID - ID of the block.
+ * @param info - ChunkInfo.
+ * @return byte array
+ * @throws StorageContainerException
+ * TODO: Right now we do not support partial reads and writes of chunks.
+ * TODO: Explore if we need to do that for ozone.
+ */
+ public byte[] readChunk(Container container, BlockID blockID, ChunkInfo info)
+ throws StorageContainerException {
+ try {
+ KeyValueContainerData containerData = (KeyValueContainerData) container
+ .getContainerData();
+ ByteBuffer data;
+
+ // Checking here, which layout version the container is, and reading
+ // the chunk file in that format.
+ // In version1, we verify checksum if it is available and return data
+ // of the chunk file.
+ if (containerData.getLayOutVersion() == ChunkLayOutVersion
+ .getLatestVersion().getVersion()) {
+ File chunkFile = ChunkUtils.getChunkFile(containerData, info);
+ data = ChunkUtils.readData(chunkFile, info);
+ containerData.incrReadCount();
+ containerData.incrReadBytes(chunkFile.length());
+ return data.array();
+ }
+ } catch(NoSuchAlgorithmException ex) {
+ LOG.error("read data failed. error: {}", ex);
+ throw new StorageContainerException("Internal error: ",
+ ex, NO_SUCH_ALGORITHM);
+ } catch (ExecutionException ex) {
+ LOG.error("read data failed. error: {}", ex);
+ throw new StorageContainerException("Internal error: ",
+ ex, CONTAINER_INTERNAL_ERROR);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ LOG.error("read data failed. error: {}", e);
+ throw new StorageContainerException("Internal error: ",
+ e, CONTAINER_INTERNAL_ERROR);
+ }
+ return null;
+ }
+
+ /**
+ * Deletes a given chunk.
+ *
+ * @param container - Container for the chunk
+ * @param blockID - ID of the block
+ * @param info - Chunk Info
+ * @throws StorageContainerException
+ */
+ public void deleteChunk(Container container, BlockID blockID, ChunkInfo info)
+ throws StorageContainerException {
+ Preconditions.checkNotNull(blockID, "Block ID cannot be null.");
+ KeyValueContainerData containerData = (KeyValueContainerData) container
+ .getContainerData();
+ // Checking here, which layout version the container is, and performing
+ // deleting chunk operation.
+ // In version1, we have only chunk file.
+ if (containerData.getLayOutVersion() == ChunkLayOutVersion
+ .getLatestVersion().getVersion()) {
+ File chunkFile = ChunkUtils.getChunkFile(containerData, info);
+ if ((info.getOffset() == 0) && (info.getLen() == chunkFile.length())) {
+ FileUtil.fullyDelete(chunkFile);
+ containerData.decrBytesUsed(chunkFile.length());
+ } else {
+ LOG.error("Not Supported Operation. Trying to delete a " +
+ "chunk that is in shared file. chunk info : " + info.toString());
+ throw new StorageContainerException("Not Supported Operation. " +
+ "Trying to delete a chunk that is in shared file. chunk info : "
+ + info.toString(), UNSUPPORTED_REQUEST);
+ }
+ }
+ }
+
+ /**
+ * Shutdown the chunkManager.
+ *
+ * In the chunkManager we haven't acquired any resources, so nothing to do
+ * here.
+ */
+
+ public void shutdown() {
+ //TODO: need to revisit this during integration of container IO.
+ }
+
+ /**
+ * Returns the temporary chunkFile path.
+ * @param chunkFile
+ * @param info
+ * @return temporary chunkFile path
+ * @throws StorageContainerException
+ */
+ private File getTmpChunkFile(File chunkFile, ChunkInfo info)
+ throws StorageContainerException {
+ return new File(chunkFile.getParent(),
+ chunkFile.getName() +
+ OzoneConsts.CONTAINER_CHUNK_NAME_DELIMITER +
+ OzoneConsts.CONTAINER_TEMPORARY_CHUNK_PREFIX);
+ }
+
+ /**
+ * Commit the chunk by renaming the temporary chunk file to chunk file.
+ * @param tmpChunkFile
+ * @param chunkFile
+ * @throws IOException
+ */
+ private void commitChunk(File tmpChunkFile, File chunkFile) throws
+ IOException {
+ Files.move(tmpChunkFile.toPath(), chunkFile.toPath(),
+ StandardCopyOption.REPLACE_EXISTING);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13579f92/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/KeyManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/KeyManagerImpl.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/KeyManagerImpl.java
new file mode 100644
index 0000000..40736e5
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/KeyManagerImpl.java
@@ -0,0 +1,192 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.container.keyvalue.impl;
+
+import com.google.common.base.Preconditions;
+import com.google.common.primitives.Longs;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.client.BlockID;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
+import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
+
+import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
+import org.apache.hadoop.ozone.container.keyvalue.helpers.KeyUtils;
+import org.apache.hadoop.ozone.container.common.helpers.KeyData;
+import org.apache.hadoop.ozone.container.common.interfaces.Container;
+import org.apache.hadoop.ozone.container.keyvalue.interfaces.KeyManager;
+import org.apache.hadoop.ozone.container.common.utils.ContainerCache;
+import org.apache.hadoop.utils.MetadataStore;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.NO_SUCH_KEY;
+
+/**
+ * This class is for performing key related operations on the KeyValue
+ * Container.
+ */
+public class KeyManagerImpl implements KeyManager {
+
+ static final Logger LOG = LoggerFactory.getLogger(KeyManagerImpl.class);
+
+ private Configuration config;
+
+ /**
+ * Constructs a key Manager.
+ *
+ * @param conf - Ozone configuration
+ */
+ public KeyManagerImpl(Configuration conf) {
+ Preconditions.checkNotNull(conf, "Config cannot be null");
+ this.config = conf;
+ }
+
+ /**
+ * Puts or overwrites a key.
+ *
+ * @param container - Container for which key need to be added.
+ * @param data - Key Data.
+ * @throws IOException
+ */
+ public void putKey(Container container, KeyData data) throws IOException {
+ Preconditions.checkNotNull(data, "KeyData cannot be null for put " +
+ "operation.");
+ Preconditions.checkState(data.getContainerID() >= 0, "Container Id " +
+ "cannot be negative");
+ // We are not locking the key manager since LevelDb serializes all actions
+ // against a single DB. We rely on DB level locking to avoid conflicts.
+ MetadataStore db = KeyUtils.getDB((KeyValueContainerData) container
+ .getContainerData(), config);
+
+ // This is a post condition that acts as a hint to the user.
+ // Should never fail.
+ Preconditions.checkNotNull(db, "DB cannot be null here");
+ db.put(Longs.toByteArray(data.getLocalID()), data.getProtoBufMessage()
+ .toByteArray());
+ }
+
+ /**
+ * Gets an existing key.
+ *
+ * @param container - Container from which key need to be get.
+ * @param blockID - BlockID of the key.
+ * @return Key Data.
+ * @throws IOException
+ */
+ public KeyData getKey(Container container, BlockID blockID)
+ throws IOException {
+ Preconditions.checkNotNull(blockID,
+ "BlockID cannot be null in GetKet request");
+ Preconditions.checkNotNull(blockID.getContainerID(),
+ "Container name cannot be null");
+
+ KeyValueContainerData containerData = (KeyValueContainerData) container
+ .getContainerData();
+ MetadataStore db = KeyUtils.getDB(containerData, config);
+ // This is a post condition that acts as a hint to the user.
+ // Should never fail.
+ Preconditions.checkNotNull(db, "DB cannot be null here");
+ byte[] kData = db.get(Longs.toByteArray(blockID.getLocalID()));
+ if (kData == null) {
+ throw new StorageContainerException("Unable to find the key.",
+ NO_SUCH_KEY);
+ }
+ ContainerProtos.KeyData keyData = ContainerProtos.KeyData.parseFrom(kData);
+ return KeyData.getFromProtoBuf(keyData);
+ }
+
+ /**
+ * Deletes an existing Key.
+ *
+ * @param container - Container from which key need to be deleted.
+ * @param blockID - ID of the block.
+ * @throws StorageContainerException
+ */
+ public void deleteKey(Container container, BlockID blockID) throws
+ IOException {
+ Preconditions.checkNotNull(blockID, "block ID cannot be null.");
+ Preconditions.checkState(blockID.getContainerID() >= 0,
+ "Container ID cannot be negative.");
+ Preconditions.checkState(blockID.getLocalID() >= 0,
+ "Local ID cannot be negative.");
+
+ KeyValueContainerData cData = (KeyValueContainerData) container
+ .getContainerData();
+ MetadataStore db = KeyUtils.getDB(cData, config);
+ // This is a post condition that acts as a hint to the user.
+ // Should never fail.
+ Preconditions.checkNotNull(db, "DB cannot be null here");
+ // Note : There is a race condition here, since get and delete
+ // are not atomic. Leaving it here since the impact is refusing
+ // to delete a key which might have just gotten inserted after
+ // the get check.
+ byte[] kKey = Longs.toByteArray(blockID.getLocalID());
+ byte[] kData = db.get(kKey);
+ if (kData == null) {
+ throw new StorageContainerException("Unable to find the key.",
+ NO_SUCH_KEY);
+ }
+ db.delete(kKey);
+ }
+
+ /**
+ * List keys in a container.
+ *
+ * @param container - Container from which keys need to be listed.
+ * @param startLocalID - Key to start from, 0 to begin.
+ * @param count - Number of keys to return.
+ * @return List of Keys that match the criteria.
+ */
+ public List<KeyData> listKey(Container container, long startLocalID, int
+ count) throws IOException {
+ Preconditions.checkNotNull(container, "container cannot be null");
+ Preconditions.checkState(startLocalID >= 0, "startLocal ID cannot be " +
+ "negative");
+ Preconditions.checkArgument(count > 0,
+ "Count must be a positive number.");
+ container.readLock();
+ List<KeyData> result = null;
+ KeyValueContainerData cData = (KeyValueContainerData) container
+ .getContainerData();
+ MetadataStore db = KeyUtils.getDB(cData, config);
+ result = new ArrayList<>();
+ byte[] startKeyInBytes = Longs.toByteArray(startLocalID);
+ List<Map.Entry<byte[], byte[]>> range = db.getSequentialRangeKVs(
+ startKeyInBytes, count, null);
+ for (Map.Entry<byte[], byte[]> entry : range) {
+ KeyData value = KeyUtils.getKeyData(entry.getValue());
+ KeyData data = new KeyData(value.getBlockID());
+ result.add(data);
+ }
+ return result;
+ }
+
+ /**
+ * Shutdown KeyValueContainerManager.
+ */
+ public void shutdown() {
+ KeyUtils.shutdownCache(ContainerCache.getInstance(config));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13579f92/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/KeyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/KeyManager.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/KeyManager.java
index ebda97e..7a5d48b 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/KeyManager.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/KeyManager.java
@@ -43,11 +43,11 @@ public interface KeyManager {
* Gets an existing key.
*
* @param container - Container from which key need to be get.
- * @param data - Key Data.
+ * @param blockID - BlockID of the Key.
* @return Key Data.
* @throws IOException
*/
- KeyData getKey(Container container, KeyData data) throws IOException;
+ KeyData getKey(Container container, BlockID blockID) throws IOException;
/**
* Deletes an existing Key.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13579f92/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestKeyValueContainerData.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestKeyValueContainerData.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestKeyValueContainerData.java
index 055110c..52f291b 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestKeyValueContainerData.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestKeyValueContainerData.java
@@ -19,12 +19,11 @@
package org.apache.hadoop.ozone.container.common;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
-import org.apache.hadoop.ozone.container.common.impl.KeyValueContainerData;
+import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
import org.junit.Test;
import static org.junit.Assert.assertEquals;
-import java.io.IOException;
import java.util.concurrent.atomic.AtomicLong;
/**
@@ -33,39 +32,6 @@ import java.util.concurrent.atomic.AtomicLong;
public class TestKeyValueContainerData {
@Test
- public void testGetFromProtoBuf() throws IOException {
-
- long containerId = 1L;
- ContainerProtos.ContainerType containerType = ContainerProtos
- .ContainerType.KeyValueContainer;
- int layOutVersion = 1;
- ContainerProtos.ContainerLifeCycleState state = ContainerProtos
- .ContainerLifeCycleState.OPEN;
-
- ContainerProtos.KeyValue.Builder keyValBuilder =
- ContainerProtos.KeyValue.newBuilder();
- ContainerProtos.CreateContainerData containerData = ContainerProtos
- .CreateContainerData.newBuilder()
- .setContainerType(containerType)
- .setContainerId(containerId)
- .addMetadata(0, keyValBuilder.setKey("VOLUME").setValue("ozone")
- .build())
- .addMetadata(1, keyValBuilder.setKey("OWNER").setValue("hdfs")
- .build()).build();
-
- KeyValueContainerData kvData = KeyValueContainerData.getFromProtoBuf(
- containerData);
- assertEquals(containerType, kvData.getContainerType());
- assertEquals(containerId, kvData.getContainerId());
- assertEquals(layOutVersion, kvData.getLayOutVersion());
- assertEquals(state, kvData.getState());
- assertEquals(2, kvData.getMetadata().size());
- assertEquals("ozone", kvData.getMetadata().get("VOLUME"));
- assertEquals("hdfs", kvData.getMetadata().get("OWNER"));
-
- }
-
- @Test
public void testKeyValueData() {
long containerId = 1L;
ContainerProtos.ContainerType containerType = ContainerProtos
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13579f92/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerSet.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerSet.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerSet.java
index 2c9c2c3..5a29e8a 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerSet.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerSet.java
@@ -25,6 +25,8 @@ import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerExcep
import org.apache.hadoop.ozone.container.common.interfaces.Container;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer;
+import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
+import org.apache.hadoop.test.GenericTestUtils;
import org.junit.Test;
import java.io.IOException;
@@ -37,6 +39,7 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
/**
* Class used to test ContainerSet operations.
@@ -59,8 +62,13 @@ public class TestContainerSet {
//addContainer
boolean result = containerSet.addContainer(keyValueContainer);
assertTrue(result);
- result = containerSet.addContainer(keyValueContainer);
- assertFalse(result);
+ try {
+ result = containerSet.addContainer(keyValueContainer);
+ fail("Adding same container ID twice should fail.");
+ } catch (StorageContainerException ex) {
+ GenericTestUtils.assertExceptionContains("Container already exists with" +
+ " container Id " + containerId, ex);
+ }
//getContainer
KeyValueContainer container = (KeyValueContainer) containerSet
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13579f92/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestKeyValueYaml.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestKeyValueYaml.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestKeyValueYaml.java
index 8550c47..75c0139 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestKeyValueYaml.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestKeyValueYaml.java
@@ -21,6 +21,8 @@ package org.apache.hadoop.ozone.container.common.impl;
import org.apache.hadoop.fs.FileSystemTestHelper;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
+import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
+import org.apache.hadoop.ozone.container.keyvalue.KeyValueYaml;
import org.apache.hadoop.test.GenericTestUtils;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13579f92/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/interfaces/TestHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/interfaces/TestHandler.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/interfaces/TestHandler.java
new file mode 100644
index 0000000..50927d1
--- /dev/null
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/interfaces/TestHandler.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.container.common.interfaces;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
+import org.apache.hadoop.ozone.container.common.impl.HddsDispatcher;
+import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
+import org.apache.hadoop.ozone.container.keyvalue.KeyValueHandler;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.Assert;
+import static org.junit.Assert.fail;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestRule;
+import org.junit.rules.Timeout;
+import org.mockito.Mockito;
+
+import java.util.UUID;
+
+/**
+ * Tests Handler interface.
+ */
+public class TestHandler {
+ @Rule
+ public TestRule timeout = new Timeout(300000);
+
+ private Configuration conf;
+ private HddsDispatcher dispatcher;
+ private ContainerSet containerSet;
+ private VolumeSet volumeSet;
+ private Handler handler;
+
+ private final static String SCM_ID = UUID.randomUUID().toString();
+ private final static String DATANODE_UUID = UUID.randomUUID().toString();
+
+ @Before
+ public void setup() throws Exception {
+ this.conf = new Configuration();
+ this.containerSet = Mockito.mock(ContainerSet.class);
+ this.volumeSet = Mockito.mock(VolumeSet.class);
+
+ this.dispatcher = new HddsDispatcher(conf, containerSet, volumeSet, SCM_ID);
+ }
+
+ @Test
+ public void testGetKeyValueHandler() throws Exception {
+ Handler kvHandler = dispatcher.getHandlerForContainerType(
+ ContainerProtos.ContainerType.KeyValueContainer);
+
+ Assert.assertTrue("getHandlerForContainerType returned incorrect handler",
+ (kvHandler instanceof KeyValueHandler));
+ }
+
+ @Test
+ public void testGetHandlerForInvalidContainerType() {
+ // When new ContainerProtos.ContainerType are added, increment the code
+ // for invalid enum.
+ ContainerProtos.ContainerType invalidContainerType =
+ ContainerProtos.ContainerType.forNumber(2);
+
+ Assert.assertEquals("New ContainerType detected. Not an invalid " +
+ "containerType", invalidContainerType, null);
+
+ Handler handler = dispatcher.getHandlerForContainerType(
+ invalidContainerType);
+ Assert.assertEquals("Get Handler for Invalid ContainerType should " +
+ "return null.", handler, null);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13579f92/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java
index ca936c7..4576db6 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java
@@ -18,17 +18,16 @@
package org.apache.hadoop.ozone.container.keyvalue;
-
import org.apache.hadoop.hdds.client.BlockID;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
import org.apache.hadoop.ozone.container.common.helpers.KeyData;
-import org.apache.hadoop.ozone.container.common.impl.KeyValueContainerData;
import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
import org.apache.hadoop.ozone.container.common.volume.RoundRobinVolumeChoosingPolicy;
import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
+import org.apache.hadoop.ozone.container.keyvalue.impl.ChunkManagerImpl;
import org.apache.hadoop.test.GenericTestUtils;
import org.junit.Before;
import org.junit.Rule;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13579f92/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyManagerImpl.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyManagerImpl.java
index a6f50c4..722cece 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyManagerImpl.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyManagerImpl.java
@@ -26,9 +26,9 @@ import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
import org.apache.hadoop.ozone.container.common.helpers.KeyData;
import org.apache.hadoop.ozone.container.common.volume
.RoundRobinVolumeChoosingPolicy;
-import org.apache.hadoop.ozone.container.common.impl.KeyValueContainerData;
import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
+import org.apache.hadoop.ozone.container.keyvalue.impl.KeyManagerImpl;
import org.apache.hadoop.test.GenericTestUtils;
import org.junit.Before;
import org.junit.Rule;
@@ -110,7 +110,7 @@ public class TestKeyManagerImpl {
//Get Key
KeyData fromGetKeyData = keyValueContainerManager.getKey(keyValueContainer,
- keyData);
+ keyData.getBlockID());
assertEquals(keyData.getContainerID(), fromGetKeyData.getContainerID());
assertEquals(keyData.getLocalID(), fromGetKeyData.getLocalID());
@@ -168,8 +168,7 @@ public class TestKeyManagerImpl {
@Test
public void testGetNoSuchKey() throws Exception {
try {
- keyData = new KeyData(new BlockID(1L, 2L));
- keyValueContainerManager.getKey(keyValueContainer, keyData);
+ keyValueContainerManager.getKey(keyValueContainer, new BlockID(1L, 2L));
fail("testGetNoSuchKey failed");
} catch (StorageContainerException ex) {
GenericTestUtils.assertExceptionContains("Unable to find the key.", ex);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13579f92/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java
index b24f601..006b82c 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java
@@ -22,13 +22,15 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
-import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
-import org.apache.hadoop.ozone.container.common.impl.KeyValueContainerData;
-import org.apache.hadoop.ozone.container.common.impl.KeyValueYaml;
+import org.apache.hadoop.hdds.scm.container.common.helpers
+ .StorageContainerException;
import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
-import org.apache.hadoop.ozone.container.common.volume.RoundRobinVolumeChoosingPolicy;
+import org.apache.hadoop.ozone.container.common.volume
+ .RoundRobinVolumeChoosingPolicy;
import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
+import org.apache.hadoop.ozone.container.keyvalue.helpers
+ .KeyValueContainerLocationUtil;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.DiskChecker;
import org.junit.Before;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13579f92/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java
new file mode 100644
index 0000000..f4dd41c
--- /dev/null
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java
@@ -0,0 +1,246 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.container.keyvalue;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
+ .ContainerCommandRequestProto;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
+import org.apache.hadoop.ozone.container.common.impl.HddsDispatcher;
+import org.apache.hadoop.ozone.container.common.interfaces.Container;
+import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestRule;
+import org.junit.rules.Timeout;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyLong;
+import org.mockito.Mockito;
+import static org.mockito.Mockito.times;
+
+import java.util.UUID;
+
+/**
+ * Unit tests for {@link KeyValueHandler}.
+ */
+public class TestKeyValueHandler {
+
+ @Rule
+ public TestRule timeout = new Timeout(300000);
+
+ private Configuration conf;
+ private HddsDispatcher dispatcher;
+ private ContainerSet containerSet;
+ private VolumeSet volumeSet;
+ private KeyValueHandler handler;
+
+ private final static String SCM_ID = UUID.randomUUID().toString();
+ private final static String DATANODE_UUID = UUID.randomUUID().toString();
+ private int containerID;
+
+ private final String baseDir = MiniDFSCluster.getBaseDirectory();
+ private final String volume = baseDir + "disk1";
+
+ private void setup() throws Exception {
+ this.conf = new Configuration();
+ conf.set(ScmConfigKeys.HDDS_DATANODE_DIR_KEY, volume);
+
+ this.containerSet = new ContainerSet();
+ DatanodeDetails datanodeDetails = DatanodeDetails.newBuilder()
+ .setUuid(DATANODE_UUID)
+ .setHostName("localhost")
+ .setIpAddress("127.0.0.1")
+ .build();
+ this.volumeSet = new VolumeSet(datanodeDetails, conf);
+
+ this.dispatcher = new HddsDispatcher(conf, containerSet, volumeSet, SCM_ID);
+ this.handler = (KeyValueHandler) dispatcher.getHandlerForContainerType(
+ ContainerProtos.ContainerType.KeyValueContainer);
+ }
+
+ @Test
+ /**
+ * Test that Handler handles different command types correctly.
+ */
+ public void testHandlerCommandHandling() throws Exception{
+ // Create mock HddsDispatcher and KeyValueHandler.
+ this.handler = Mockito.mock(KeyValueHandler.class);
+ this.dispatcher = Mockito.mock(HddsDispatcher.class);
+ Mockito.when(dispatcher.getHandlerForContainerType(any())).thenReturn
+ (handler);
+ Mockito.when(dispatcher.dispatch(any())).thenCallRealMethod();
+ Mockito.when(dispatcher.getContainer(anyLong())).thenReturn(
+ Mockito.mock(KeyValueContainer.class));
+ Mockito.when(handler.handle(any(), any())).thenCallRealMethod();
+
+ // Test Create Container Request handling
+ ContainerCommandRequestProto createContainerRequest =
+ getDummyCommandRequestProto(ContainerProtos.Type.CreateContainer);
+ dispatcher.dispatch(createContainerRequest);
+ Mockito.verify(handler, times(1)).handleCreateContainer(
+ any(ContainerCommandRequestProto.class), any());
+
+ // Test Read Container Request handling
+ ContainerCommandRequestProto readContainerRequest =
+ getDummyCommandRequestProto(ContainerProtos.Type.ReadContainer);
+ dispatcher.dispatch(readContainerRequest);
+ Mockito.verify(handler, times(1)).handleReadContainer(
+ any(ContainerCommandRequestProto.class), any());
+
+ // Test Update Container Request handling
+ ContainerCommandRequestProto updateContainerRequest =
+ getDummyCommandRequestProto(ContainerProtos.Type.UpdateContainer);
+ dispatcher.dispatch(updateContainerRequest);
+ Mockito.verify(handler, times(1)).handleUpdateContainer(
+ any(ContainerCommandRequestProto.class), any());
+
+ // Test Delete Container Request handling
+ ContainerCommandRequestProto deleteContainerRequest =
+ getDummyCommandRequestProto(ContainerProtos.Type.DeleteContainer);
+ dispatcher.dispatch(deleteContainerRequest);
+ Mockito.verify(handler, times(1)).handleDeleteContainer(
+ any(ContainerCommandRequestProto.class), any());
+
+ // Test List Container Request handling
+ ContainerCommandRequestProto listContainerRequest =
+ getDummyCommandRequestProto(ContainerProtos.Type.ListContainer);
+ dispatcher.dispatch(listContainerRequest);
+ Mockito.verify(handler, times(1)).handleUnsupportedOp(
+ any(ContainerCommandRequestProto.class));
+
+ // Test Close Container Request handling
+ ContainerCommandRequestProto closeContainerRequest =
+ getDummyCommandRequestProto(ContainerProtos.Type.CloseContainer);
+ dispatcher.dispatch(closeContainerRequest);
+ Mockito.verify(handler, times(1)).handleCloseContainer(
+ any(ContainerCommandRequestProto.class), any());
+
+ // Test Put Key Request handling
+ ContainerCommandRequestProto putKeyRequest =
+ getDummyCommandRequestProto(ContainerProtos.Type.PutKey);
+ dispatcher.dispatch(putKeyRequest);
+ Mockito.verify(handler, times(1)).handlePutKey(
+ any(ContainerCommandRequestProto.class), any());
+
+ // Test Get Key Request handling
+ ContainerCommandRequestProto getKeyRequest =
+ getDummyCommandRequestProto(ContainerProtos.Type.GetKey);
+ dispatcher.dispatch(getKeyRequest);
+ Mockito.verify(handler, times(1)).handleGetKey(
+ any(ContainerCommandRequestProto.class), any());
+
+ // Test Delete Key Request handling
+ ContainerCommandRequestProto deleteKeyRequest =
+ getDummyCommandRequestProto(ContainerProtos.Type.DeleteKey);
+ dispatcher.dispatch(deleteKeyRequest);
+ Mockito.verify(handler, times(1)).handleDeleteKey(
+ any(ContainerCommandRequestProto.class), any());
+
+ // Test List Key Request handling
+ ContainerCommandRequestProto listKeyRequest =
+ getDummyCommandRequestProto(ContainerProtos.Type.ListKey);
+ dispatcher.dispatch(listKeyRequest);
+ Mockito.verify(handler, times(2)).handleUnsupportedOp(
+ any(ContainerCommandRequestProto.class));
+
+ // Test Read Chunk Request handling
+ ContainerCommandRequestProto readChunkRequest =
+ getDummyCommandRequestProto(ContainerProtos.Type.ReadChunk);
+ dispatcher.dispatch(readChunkRequest);
+ Mockito.verify(handler, times(1)).handleReadChunk(
+ any(ContainerCommandRequestProto.class), any());
+
+ // Test Delete Chunk Request handling
+ ContainerCommandRequestProto deleteChunkRequest =
+ getDummyCommandRequestProto(ContainerProtos.Type.DeleteChunk);
+ dispatcher.dispatch(deleteChunkRequest);
+ Mockito.verify(handler, times(1)).handleDeleteChunk(
+ any(ContainerCommandRequestProto.class), any());
+
+ // Test Write Chunk Request handling
+ ContainerCommandRequestProto writeChunkRequest =
+ getDummyCommandRequestProto(ContainerProtos.Type.WriteChunk);
+ dispatcher.dispatch(writeChunkRequest);
+ Mockito.verify(handler, times(1)).handleWriteChunk(
+ any(ContainerCommandRequestProto.class), any());
+
+ // Test List Chunk Request handling
+ ContainerCommandRequestProto listChunkRequest =
+ getDummyCommandRequestProto(ContainerProtos.Type.ListChunk);
+ dispatcher.dispatch(listChunkRequest);
+ Mockito.verify(handler, times(3)).handleUnsupportedOp(
+ any(ContainerCommandRequestProto.class));
+
+ // Test Put Small File Request handling
+ ContainerCommandRequestProto putSmallFileRequest =
+ getDummyCommandRequestProto(ContainerProtos.Type.PutSmallFile);
+ dispatcher.dispatch(putSmallFileRequest);
+ Mockito.verify(handler, times(1)).handlePutSmallFile(
+ any(ContainerCommandRequestProto.class), any());
+
+ // Test Get Small File Request handling
+ ContainerCommandRequestProto getSmallFileRequest =
+ getDummyCommandRequestProto(ContainerProtos.Type.GetSmallFile);
+ dispatcher.dispatch(getSmallFileRequest);
+ Mockito.verify(handler, times(1)).handleGetSmallFile(
+ any(ContainerCommandRequestProto.class), any());
+ }
+
+ private ContainerCommandRequestProto getDummyCommandRequestProto
+ (ContainerProtos.Type cmdType) {
+ ContainerCommandRequestProto request =
+ ContainerProtos.ContainerCommandRequestProto.newBuilder()
+ .setCmdType(cmdType)
+ .setDatanodeUuid(DATANODE_UUID)
+ .build();
+
+ return request;
+ }
+
+ @Test
+ public void testCreateContainer() throws Exception {
+ setup();
+
+ long contId = ++containerID;
+ ContainerProtos.CreateContainerRequestProto createReq =
+ ContainerProtos.CreateContainerRequestProto.newBuilder()
+ .setContainerID(contId)
+ .build();
+
+ ContainerCommandRequestProto request =
+ ContainerProtos.ContainerCommandRequestProto.newBuilder()
+ .setCmdType(ContainerProtos.Type.CreateContainer)
+ .setDatanodeUuid(DATANODE_UUID)
+ .setCreateContainer(createReq)
+ .build();
+
+ dispatcher.dispatch(request);
+
+ // Verify that new container is added to containerSet.
+ Container container = containerSet.getContainer(contId);
+ Assert.assertEquals(contId, container.getContainerData().getContainerId());
+ Assert.assertEquals(ContainerProtos.ContainerLifeCycleState.OPEN,
+ container.getContainerState());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13579f92/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupInputStream.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupInputStream.java
index e1a2918..38a4769 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupInputStream.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupInputStream.java
@@ -281,10 +281,10 @@ public class ChunkGroupInputStream extends InputStream implements Seekable {
LOG.debug("get key accessing {} {}",
containerID, containerKey);
groupInputStream.streamOffset[i] = length;
- ContainerProtos.KeyData containerKeyData = OzoneContainerTranslation
- .containerKeyDataForRead(blockID);
+ ContainerProtos.DatanodeBlockID datanodeBlockID = blockID
+ .getDatanodeBlockIDProtobuf();
ContainerProtos.GetKeyResponseProto response = ContainerProtocolCalls
- .getKey(xceiverClient, containerKeyData, requestId);
+ .getKey(xceiverClient, datanodeBlockID, requestId);
List<ContainerProtos.ChunkInfo> chunks =
response.getKeyData().getChunksList();
for (ContainerProtos.ChunkInfo chunk : chunks) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13579f92/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/OzoneContainerTranslation.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/OzoneContainerTranslation.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/OzoneContainerTranslation.java
deleted file mode 100644
index e74fffd..0000000
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/OzoneContainerTranslation.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.client.io;
-
-
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.KeyData;
-import org.apache.hadoop.hdds.client.BlockID;
-
-
-/**
- * This class contains methods that define the translation between the Ozone
- * domain model and the storage container domain model.
- */
-final class OzoneContainerTranslation {
-
- /**
- * Creates key data intended for reading a container key.
- *
- * @param blockID - ID of the block.
- * @return KeyData intended for reading the container key
- */
- public static KeyData containerKeyDataForRead(BlockID blockID) {
- return KeyData
- .newBuilder()
- .setBlockID(blockID.getDatanodeBlockIDProtobuf())
- .build();
- }
-
- /**
- * There is no need to instantiate this class.
- */
- private OzoneContainerTranslation() {
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13579f92/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java
index 7046132..9decdb9 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java
@@ -362,10 +362,7 @@ public final class ContainerTestHelper {
ContainerProtos.CreateContainerRequestProto.Builder createRequest =
ContainerProtos.CreateContainerRequestProto
.newBuilder();
- ContainerProtos.ContainerData.Builder containerData = ContainerProtos
- .ContainerData.newBuilder();
- containerData.setContainerID(containerID);
- createRequest.setContainerData(containerData.build());
+ createRequest.setContainerID(containerID);
ContainerCommandRequestProto.Builder request =
ContainerCommandRequestProto.newBuilder();
@@ -391,19 +388,16 @@ public final class ContainerTestHelper {
long containerID, Map<String, String> metaData) throws IOException {
ContainerProtos.UpdateContainerRequestProto.Builder updateRequestBuilder =
ContainerProtos.UpdateContainerRequestProto.newBuilder();
- ContainerProtos.ContainerData.Builder containerData = ContainerProtos
- .ContainerData.newBuilder();
- containerData.setContainerID(containerID);
+ updateRequestBuilder.setContainerID(containerID);
String[] keys = metaData.keySet().toArray(new String[]{});
for(int i=0; i<keys.length; i++) {
KeyValue.Builder kvBuilder = KeyValue.newBuilder();
kvBuilder.setKey(keys[i]);
kvBuilder.setValue(metaData.get(keys[i]));
- containerData.addMetadata(i, kvBuilder.build());
+ updateRequestBuilder.addMetadata(kvBuilder.build());
}
Pipeline pipeline =
ContainerTestHelper.createSingleNodePipeline();
- updateRequestBuilder.setContainerData(containerData.build());
ContainerCommandRequestProto.Builder request =
ContainerCommandRequestProto.newBuilder();
@@ -478,10 +472,7 @@ public final class ContainerTestHelper {
ContainerProtos.GetKeyRequestProto.Builder getRequest =
ContainerProtos.GetKeyRequestProto.newBuilder();
- ContainerProtos.KeyData.Builder keyData = ContainerProtos.KeyData
- .newBuilder();
- keyData.setBlockID(blockID);
- getRequest.setKeyData(keyData);
+ getRequest.setBlockID(blockID);
ContainerCommandRequestProto.Builder request =
ContainerCommandRequestProto.newBuilder();
@@ -499,13 +490,11 @@ public final class ContainerTestHelper {
* @param response - Response
*/
public static void verifyGetKey(ContainerCommandRequestProto request,
- ContainerCommandResponseProto response) {
+ ContainerCommandResponseProto response, int expectedChunksCount) {
Assert.assertEquals(request.getTraceID(), response.getTraceID());
Assert.assertEquals(ContainerProtos.Result.SUCCESS, response.getResult());
- ContainerProtos.PutKeyRequestProto putKey = request.getPutKey();
- ContainerProtos.GetKeyRequestProto getKey = request.getGetKey();
- Assert.assertEquals(putKey.getKeyData().getChunksCount(),
- getKey.getKeyData().getChunksCount());
+ Assert.assertEquals(expectedChunksCount,
+ response.getGetKey().getKeyData().getChunksCount());
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13579f92/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java
index 67a8160..3f02036 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java
@@ -158,7 +158,8 @@ public class TestOzoneContainer {
// Get Key
request = ContainerTestHelper.getKeyRequest(pipeline, putKeyRequest.getPutKey());
response = client.sendCommand(request);
- ContainerTestHelper.verifyGetKey(request, response);
+ int chunksCount = putKeyRequest.getPutKey().getKeyData().getChunksCount();
+ ContainerTestHelper.verifyGetKey(request, response, chunksCount);
// Delete Key
@@ -331,7 +332,8 @@ public class TestOzoneContainer {
request = ContainerTestHelper.getKeyRequest(client.getPipeline(),
putKeyRequest.getPutKey());
response = client.sendCommand(request);
- ContainerTestHelper.verifyGetKey(request, response);
+ int chunksCount = putKeyRequest.getPutKey().getKeyData().getChunksCount();
+ ContainerTestHelper.verifyGetKey(request, response, chunksCount);
// Delete Key must fail on a closed container.
request =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13579f92/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java
index 13b04c3..c14c1b9 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java
@@ -174,9 +174,7 @@ public class BenchMarkDatanodeDispatcher {
private ContainerCommandRequestProto getCreateContainerCommand(long containerID) {
CreateContainerRequestProto.Builder createRequest =
CreateContainerRequestProto.newBuilder();
- createRequest.setContainerData(
- ContainerData.newBuilder().setContainerID(
- containerID).build());
+ createRequest.setContainerID(containerID).build();
ContainerCommandRequestProto.Builder request =
ContainerCommandRequestProto.newBuilder();
@@ -245,10 +243,9 @@ public class BenchMarkDatanodeDispatcher {
return request.build();
}
- private ContainerCommandRequestProto getGetKeyCommand(
- BlockID blockID, String chunkKey) {
+ private ContainerCommandRequestProto getGetKeyCommand(BlockID blockID) {
GetKeyRequestProto.Builder readKeyRequest = GetKeyRequestProto.newBuilder()
- .setKeyData(getKeyData(blockID, chunkKey));
+ .setBlockID(blockID.getDatanodeBlockIDProtobuf());
ContainerCommandRequestProto.Builder request = ContainerCommandRequestProto
.newBuilder()
.setCmdType(ContainerProtos.Type.GetKey)
@@ -300,8 +297,7 @@ public class BenchMarkDatanodeDispatcher {
@Benchmark
public void getKey(BenchMarkDatanodeDispatcher bmdd) {
BlockID blockID = getRandomBlockID();
- String chunkKey = getNewChunkToWrite();
- bmdd.dispatcher.dispatch(getGetKeyCommand(blockID, chunkKey));
+ bmdd.dispatcher.dispatch(getGetKeyCommand(blockID));
}
// Chunks writes from benchmark only reaches certain containers
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org