You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by sh...@apache.org on 2020/06/18 14:21:02 UTC
[hadoop-ozone] branch master updated: HDDS-3802. Incorrect data
returned by reading a FILE_PER_CHUNK block. (#1079)
This is an automated email from the ASF dual-hosted git repository.
shashikant pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 5ebb065 HDDS-3802. Incorrect data returned by reading a FILE_PER_CHUNK block. (#1079)
5ebb065 is described below
commit 5ebb065848a63b211bcfce646fbe395c2eab042a
Author: Sammi Chen <sa...@apache.org>
AuthorDate: Thu Jun 18 22:20:48 2020 +0800
HDDS-3802. Incorrect data returned by reading a FILE_PER_CHUNK block. (#1079)
---
.../ozone/container/keyvalue/KeyValueHandler.java | 2 +-
.../keyvalue/impl/ChunkManagerDispatcher.java | 5 +-
.../keyvalue/impl/ChunkManagerFactory.java | 13 +++-
.../keyvalue/impl/FilePerChunkStrategy.java | 45 +++++++++---
.../common/impl/TestContainerPersistence.java | 7 +-
.../container/keyvalue/ChunkLayoutTestInfo.java | 12 ++--
.../keyvalue/TestKeyValueContainerCheck.java | 2 +-
.../keyvalue/impl/AbstractTestChunkManager.java | 8 ++-
.../keyvalue/impl/CommonChunkManagerTestCases.java | 7 ++
.../ozone/client/rpc/TestKeyInputStream.java | 82 +++++++++++++++++++---
.../hadoop/ozone/freon/ChunkManagerDiskWrite.java | 3 +-
.../ozone/genesis/BenchmarkChunkManager.java | 2 +-
12 files changed, 154 insertions(+), 34 deletions(-)
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
index 56fd3f4..53797b0 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
@@ -119,7 +119,7 @@ public class KeyValueHandler extends Handler {
super(config, datanodeId, contSet, volSet, metrics, icrSender);
containerType = ContainerType.KeyValueContainer;
blockManager = new BlockManagerImpl(config);
- chunkManager = ChunkManagerFactory.createChunkManager(config);
+ chunkManager = ChunkManagerFactory.createChunkManager(config, blockManager);
try {
volumeChoosingPolicy = conf.getClass(
HDDS_DATANODE_VOLUME_CHOOSING_POLICY, RoundRobinVolumeChoosingPolicy
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerDispatcher.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerDispatcher.java
index 3d2b815..1c4a917 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerDispatcher.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerDispatcher.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
import org.apache.hadoop.ozone.container.common.transport.server.ratis.DispatcherContext;
import org.apache.hadoop.ozone.container.common.impl.ChunkLayOutVersion;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer;
+import org.apache.hadoop.ozone.container.keyvalue.interfaces.BlockManager;
import org.apache.hadoop.ozone.container.keyvalue.interfaces.ChunkManager;
import org.apache.hadoop.ozone.container.common.interfaces.Container;
@@ -55,8 +56,8 @@ public class ChunkManagerDispatcher implements ChunkManager {
private final Map<ChunkLayOutVersion, ChunkManager> handlers
= new EnumMap<>(ChunkLayOutVersion.class);
- ChunkManagerDispatcher(boolean sync) {
- handlers.put(FILE_PER_CHUNK, new FilePerChunkStrategy(sync));
+ ChunkManagerDispatcher(boolean sync, BlockManager manager) {
+ handlers.put(FILE_PER_CHUNK, new FilePerChunkStrategy(sync, manager));
handlers.put(FILE_PER_BLOCK, new FilePerBlockStrategy(sync));
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerFactory.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerFactory.java
index 6b3f58d..68e1864 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerFactory.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerFactory.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.ozone.container.keyvalue.impl;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.container.keyvalue.interfaces.BlockManager;
import org.apache.hadoop.ozone.container.keyvalue.interfaces.ChunkManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -39,7 +40,15 @@ public final class ChunkManagerFactory {
private ChunkManagerFactory() {
}
- public static ChunkManager createChunkManager(ConfigurationSource conf) {
+ /**
+ * Create a chunk manager.
+ * @param conf Configuration
+ * @param manager This parameter will be used only for read data of
+ * FILE_PER_CHUNK layout file. Can be null for other cases.
+ * @return
+ */
+ public static ChunkManager createChunkManager(ConfigurationSource conf,
+ BlockManager manager) {
boolean sync =
conf.getBoolean(OzoneConfigKeys.DFS_CONTAINER_CHUNK_WRITE_SYNC_KEY,
OzoneConfigKeys.DFS_CONTAINER_CHUNK_WRITE_SYNC_DEFAULT);
@@ -67,6 +76,6 @@ public final class ChunkManagerFactory {
return new ChunkManagerDummyImpl();
}
- return new ChunkManagerDispatcher(sync);
+ return new ChunkManagerDispatcher(sync, manager);
}
}
\ No newline at end of file
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/FilePerChunkStrategy.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/FilePerChunkStrategy.java
index 6ecf01d..ad58935 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/FilePerChunkStrategy.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/FilePerChunkStrategy.java
@@ -34,11 +34,13 @@ import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
import org.apache.hadoop.ozone.container.common.volume.VolumeIOStats;
import org.apache.hadoop.ozone.container.keyvalue.helpers.ChunkUtils;
+import org.apache.hadoop.ozone.container.keyvalue.interfaces.BlockManager;
import org.apache.hadoop.ozone.container.keyvalue.interfaces.ChunkManager;
import org.apache.hadoop.ozone.container.common.interfaces.Container;
import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.IO_EXCEPTION;
import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.UNABLE_TO_FIND_CHUNK;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -62,9 +64,11 @@ public class FilePerChunkStrategy implements ChunkManager {
LoggerFactory.getLogger(FilePerChunkStrategy.class);
private final boolean doSyncWrite;
+ private final BlockManager blockManager;
- public FilePerChunkStrategy(boolean sync) {
+ public FilePerChunkStrategy(boolean sync, BlockManager manager) {
doSyncWrite = sync;
+ blockManager = manager;
}
private static void checkLayoutVersion(Container container) {
@@ -218,17 +222,40 @@ public class FilePerChunkStrategy implements ChunkManager {
long len = info.getLen();
ByteBuffer data = ByteBuffer.allocate((int) len);
+ long chunkFileOffset = 0;
+ if (info.getOffset() != 0) {
+ try {
+ BlockData blockData = blockManager.getBlock(kvContainer, blockID);
+ List<ContainerProtos.ChunkInfo> chunks = blockData.getChunks();
+ String chunkName = info.getChunkName();
+ boolean found = false;
+ for (ContainerProtos.ChunkInfo chunk : chunks) {
+ if (chunk.getChunkName().equals(chunkName)) {
+ chunkFileOffset = chunk.getOffset();
+ found = true;
+ break;
+ }
+ }
+ if (!found) {
+ throw new StorageContainerException(
+ "Cannot find chunk " + chunkName + " in block " +
+ blockID.toString(), UNABLE_TO_FIND_CHUNK);
+ }
+ } catch (IOException e) {
+ throw new StorageContainerException(
+ "Cannot find block " + blockID.toString() + " for chunk " +
+ info.getChunkName(), UNABLE_TO_FIND_CHUNK);
+ }
+ }
+
for (File file : possibleFiles) {
try {
- // use offset only if file written by old datanode
- long offset;
- if (file.exists() && file.length() == info.getOffset() + len) {
- offset = info.getOffset();
- } else {
- offset = 0;
+ if (file.exists()) {
+ long offset = info.getOffset() - chunkFileOffset;
+ Preconditions.checkState(offset >= 0);
+ ChunkUtils.readData(file, data, offset, len, volumeIOStats);
+ return ChunkBuffer.wrap(data);
}
- ChunkUtils.readData(file, data, offset, len, volumeIOStats);
- return ChunkBuffer.wrap(data);
} catch (StorageContainerException ex) {
//UNABLE TO FIND chunk is not a problem as we will try with the
//next possible location
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java
index 303252b..785ed8b 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java
@@ -140,7 +140,7 @@ public class TestContainerPersistence {
containerSet = new ContainerSet();
volumeSet = new MutableVolumeSet(DATANODE_UUID, conf);
blockManager = new BlockManagerImpl(conf);
- chunkManager = ChunkManagerFactory.createChunkManager(conf);
+ chunkManager = ChunkManagerFactory.createChunkManager(conf, blockManager);
for (String dir : conf.getStrings(ScmConfigKeys.HDDS_DATANODE_DIR_KEY)) {
StorageLocation location = StorageLocation.parse(dir);
@@ -405,6 +405,7 @@ public class TestContainerPersistence {
BlockID blockID = ContainerTestHelper.getTestBlockID(testContainerID);
List<ChunkInfo> chunks = new ArrayList<>(chunkCount);
+ BlockData blockData = new BlockData(blockID);
for (int x = 0; x < chunkCount; x++) {
ChunkInfo info = getChunk(blockID.getLocalID(), x, x * datalen, datalen);
ChunkBuffer data = getData(datalen);
@@ -412,7 +413,9 @@ public class TestContainerPersistence {
chunkManager.writeChunk(container, blockID, info, data,
getDispatcherContext());
chunks.add(info);
+ blockData.addChunk(info.getProtoBufMessage());
}
+ blockManager.putBlock(container, blockData);
KeyValueContainerData cNewData =
(KeyValueContainerData) container.getContainerData();
@@ -484,7 +487,7 @@ public class TestContainerPersistence {
BlockID blockID = ContainerTestHelper.getTestBlockID(testContainerID);
ChunkInfo info = getChunk(
- blockID.getLocalID(), 0, 0, datalen);
+ blockID.getLocalID(), 1, 0, datalen);
ChunkBuffer data = getData(datalen);
setDataChecksum(info, data);
chunkManager.writeChunk(container, blockID, info, data,
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/ChunkLayoutTestInfo.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/ChunkLayoutTestInfo.java
index b8bafb2..edd3368 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/ChunkLayoutTestInfo.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/ChunkLayoutTestInfo.java
@@ -22,6 +22,7 @@ import org.apache.hadoop.ozone.container.common.impl.ChunkLayOutVersion;
import org.apache.hadoop.ozone.container.keyvalue.impl.ChunkManagerDummyImpl;
import org.apache.hadoop.ozone.container.keyvalue.impl.FilePerBlockStrategy;
import org.apache.hadoop.ozone.container.keyvalue.impl.FilePerChunkStrategy;
+import org.apache.hadoop.ozone.container.keyvalue.interfaces.BlockManager;
import org.apache.hadoop.ozone.container.keyvalue.interfaces.ChunkManager;
import java.io.File;
@@ -40,7 +41,7 @@ public enum ChunkLayoutTestInfo {
DUMMY {
@Override
- public ChunkManager createChunkManager(boolean sync) {
+ public ChunkManager createChunkManager(boolean sync, BlockManager manager) {
return new ChunkManagerDummyImpl();
}
@@ -61,8 +62,8 @@ public enum ChunkLayoutTestInfo {
},
FILE_PER_CHUNK {
- public ChunkManager createChunkManager(boolean sync) {
- return new FilePerChunkStrategy(sync);
+ public ChunkManager createChunkManager(boolean sync, BlockManager manager) {
+ return new FilePerChunkStrategy(sync, manager);
}
@Override
@@ -77,7 +78,7 @@ public enum ChunkLayoutTestInfo {
},
FILE_PER_BLOCK {
- public ChunkManager createChunkManager(boolean sync) {
+ public ChunkManager createChunkManager(boolean sync, BlockManager manager) {
return new FilePerBlockStrategy(sync);
}
@@ -92,7 +93,8 @@ public enum ChunkLayoutTestInfo {
}
};
- public abstract ChunkManager createChunkManager(boolean sync);
+ public abstract ChunkManager createChunkManager(boolean sync,
+ BlockManager manager);
public abstract void validateFileCount(File dir, long blockCount,
long chunkCount);
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainerCheck.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainerCheck.java
index 41ae57d..cb8ef34 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainerCheck.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainerCheck.java
@@ -109,7 +109,7 @@ import static org.junit.Assert.assertFalse;
conf.set(OZONE_METADATA_STORE_IMPL, storeImpl);
chunkManagerTestInfo.updateConfig(conf);
volumeSet = new MutableVolumeSet(UUID.randomUUID().toString(), conf);
- chunkManager = chunkManagerTestInfo.createChunkManager(true);
+ chunkManager = chunkManagerTestInfo.createChunkManager(true, null);
}
@After public void teardown() {
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/impl/AbstractTestChunkManager.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/impl/AbstractTestChunkManager.java
index 705bf14..c1ab19f 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/impl/AbstractTestChunkManager.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/impl/AbstractTestChunkManager.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
import org.apache.hadoop.ozone.container.keyvalue.ChunkLayoutTestInfo;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
+import org.apache.hadoop.ozone.container.keyvalue.interfaces.BlockManager;
import org.apache.hadoop.ozone.container.keyvalue.interfaces.ChunkManager;
import org.junit.Before;
import org.junit.Rule;
@@ -62,6 +63,7 @@ public abstract class AbstractTestChunkManager {
private ChunkInfo chunkInfo;
private ByteBuffer data;
private byte[] header;
+ private BlockManager blockManager;
@Rule
public TemporaryFolder folder = new TemporaryFolder();
@@ -69,7 +71,8 @@ public abstract class AbstractTestChunkManager {
protected abstract ChunkLayoutTestInfo getStrategy();
protected ChunkManager createTestSubject() {
- return getStrategy().createChunkManager(true);
+ blockManager = new BlockManagerImpl(new OzoneConfiguration());
+ return getStrategy().createChunkManager(true, blockManager);
}
@Before
@@ -167,4 +170,7 @@ public abstract class AbstractTestChunkManager {
return data;
}
+ protected BlockManager getBlockManager() {
+ return blockManager;
+ }
}
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/impl/CommonChunkManagerTestCases.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/impl/CommonChunkManagerTestCases.java
index c621028..ca9c7b8 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/impl/CommonChunkManagerTestCases.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/impl/CommonChunkManagerTestCases.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.ozone.container.keyvalue.impl;
import org.apache.hadoop.hdds.client.BlockID;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
+import org.apache.hadoop.ozone.container.common.helpers.BlockData;
import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
import org.apache.hadoop.ozone.container.common.transport.server.ratis.DispatcherContext;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer;
@@ -96,6 +97,9 @@ public abstract class CommonChunkManagerTestCases
checkWriteIOStats(chunkInfo.getLen(), 1);
checkReadIOStats(0, 0);
+ BlockData blockData = new BlockData(blockID);
+ blockData.addChunk(chunkInfo.getProtoBufMessage());
+ getBlockManager().putBlock(container, blockData);
ByteBuffer expectedData = chunkManager
.readChunk(container, blockID, chunkInfo, dispatcherContext)
@@ -175,13 +179,16 @@ public abstract class CommonChunkManagerTestCases
ByteBuffer data = getData();
DispatcherContext context = getDispatcherContext();
+ BlockData blockData = new BlockData(blockID);
// WHEN
for (int i = 0; i< count; i++) {
ChunkInfo info = new ChunkInfo(String.format("%d.data.%d", localID, i),
i * len, len);
chunkManager.writeChunk(container, blockID, info, data, context);
rewindBufferToDataStart();
+ blockData.addChunk(info.getProtoBufMessage());
}
+ getBlockManager().putBlock(container, blockData);
// THEN
checkWriteIOStats(len * count, count);
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestKeyInputStream.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestKeyInputStream.java
index d8c4b3b..8ab176d 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestKeyInputStream.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestKeyInputStream.java
@@ -21,6 +21,7 @@ import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.XceiverClientManager;
import org.apache.hadoop.hdds.scm.XceiverClientMetrics;
import org.apache.hadoop.ozone.MiniOzoneCluster;
@@ -32,15 +33,20 @@ import org.apache.hadoop.ozone.client.io.KeyInputStream;
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
import org.apache.hadoop.ozone.container.ContainerTestHelper;
import org.apache.hadoop.ozone.container.TestHelper;
-import org.junit.AfterClass;
+import org.apache.hadoop.ozone.container.keyvalue.ChunkLayoutTestInfo;
+import org.junit.After;
import org.junit.Assert;
-import org.junit.BeforeClass;
+import org.junit.Before;
import org.junit.Test;
import org.junit.Rule;
import org.junit.rules.Timeout;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
import java.util.Random;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
@@ -52,6 +58,7 @@ import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_STALENODE_INTER
/**
* Tests {@link KeyInputStream}.
*/
+@RunWith(Parameterized.class)
public class TestKeyInputStream {
private static MiniOzoneCluster cluster;
private static OzoneConfiguration conf = new OzoneConfiguration();
@@ -64,7 +71,19 @@ public class TestKeyInputStream {
private static String volumeName;
private static String bucketName;
private static String keyString;
+ private static ChunkLayoutTestInfo chunkLayout;
+
+ @Parameterized.Parameters
+ public static Collection<Object[]> layouts() {
+ return Arrays.asList(new Object[][] {
+ {ChunkLayoutTestInfo.FILE_PER_CHUNK},
+ {ChunkLayoutTestInfo.FILE_PER_BLOCK}
+ });
+ }
+ public TestKeyInputStream(ChunkLayoutTestInfo layout) {
+ this.chunkLayout = layout;
+ }
/**
* Create a MiniDFSCluster for testing.
* <p>
@@ -72,17 +91,20 @@ public class TestKeyInputStream {
*
* @throws IOException
*/
- @BeforeClass
- public static void init() throws Exception {
- chunkSize = 100;
- flushSize = 4 * chunkSize;
+ @Before
+ public void init() throws Exception {
+ chunkSize = 256 * 1024 * 2;
+ flushSize = 2 * chunkSize;
maxFlushSize = 2 * flushSize;
blockSize = 2 * maxFlushSize;
conf.setTimeDuration(HDDS_SCM_WATCHER_TIMEOUT, 1000, TimeUnit.MILLISECONDS);
conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 3, TimeUnit.SECONDS);
conf.setQuietMode(false);
- conf.setStorageSize(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE, 4,
+ conf.setStorageSize(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE, 64,
StorageUnit.MB);
+ conf.set(ScmConfigKeys.OZONE_SCM_CHUNK_LAYOUT_KEY, chunkLayout.name());
+ conf.setStorageSize(
+ OzoneConfigKeys.OZONE_CLIENT_BYTES_PER_CHECKSUM, 256, StorageUnit.KB);
cluster = MiniOzoneCluster.newBuilder(conf)
.setNumDatanodes(3)
.setTotalPipelineNumLimit(5)
@@ -109,8 +131,8 @@ public class TestKeyInputStream {
/**
* Shutdown MiniDFSCluster.
*/
- @AfterClass
- public static void shutdown() {
+ @After
+ public void shutdown() {
if (cluster != null) {
cluster.shutdown();
}
@@ -331,4 +353,46 @@ public class TestKeyInputStream {
}
}
}
+
+ @Test
+ public void testReadChunk() throws Exception {
+ String keyName = getKeyName();
+ OzoneOutputStream key = TestHelper.createKey(keyName,
+ ReplicationType.RATIS, 0, objectStore, volumeName, bucketName);
+
+ // write data spanning multiple chunks
+ int dataLength = 2 * blockSize + (blockSize / 2);
+ byte[] originData = new byte[dataLength];
+ Random r = new Random();
+ r.nextBytes(originData);
+ key.write(originData);
+ key.close();
+
+ // read chunk data
+ KeyInputStream keyInputStream = (KeyInputStream) objectStore
+ .getVolume(volumeName).getBucket(bucketName).readKey(keyName)
+ .getInputStream();
+
+ int[] bufferSizeList = {chunkSize / 4, chunkSize / 2, chunkSize - 1,
+ chunkSize, chunkSize + 1, blockSize - 1, blockSize, blockSize + 1,
+ blockSize * 2};
+ for (int bufferSize : bufferSizeList) {
+ byte[] data = new byte[bufferSize];
+ int totalRead = 0;
+ while (totalRead < dataLength) {
+ int numBytesRead = keyInputStream.read(data);
+ if (numBytesRead == -1 || numBytesRead == 0) {
+ break;
+ }
+ byte[] tmp1 =
+ Arrays.copyOfRange(originData, totalRead, totalRead + numBytesRead);
+ byte[] tmp2 =
+ Arrays.copyOfRange(data, 0, numBytesRead);
+ Assert.assertArrayEquals(tmp1, tmp2);
+ totalRead += numBytesRead;
+ }
+ keyInputStream.seek(0);
+ }
+ keyInputStream.close();
+ }
}
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ChunkManagerDiskWrite.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ChunkManagerDiskWrite.java
index 4b9b9ba..43a5af3 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ChunkManagerDiskWrite.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ChunkManagerDiskWrite.java
@@ -134,7 +134,8 @@ public class ChunkManagerDiskWrite extends BaseFreonGenerator implements
blockSize = chunkSize * chunksPerBlock;
data = randomAscii(chunkSize).getBytes(UTF_8);
- chunkManager = ChunkManagerFactory.createChunkManager(ozoneConfiguration);
+ chunkManager = ChunkManagerFactory.createChunkManager(ozoneConfiguration,
+ null);
timer = getMetrics().timer("chunk-write");
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchmarkChunkManager.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchmarkChunkManager.java
index c9d2b5e..76cd037 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchmarkChunkManager.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchmarkChunkManager.java
@@ -131,7 +131,7 @@ public class BenchmarkChunkManager {
public void writeMultipleFiles(BenchmarkState state, Blackhole sink)
throws StorageContainerException {
- ChunkManager chunkManager = new FilePerChunkStrategy(true);
+ ChunkManager chunkManager = new FilePerChunkStrategy(true, null);
benchmark(chunkManager, FILE_PER_CHUNK, state, sink);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org