You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@ozone.apache.org by "Mark Gui (Jira)" <ji...@apache.org> on 2022/02/15 07:11:00 UTC
[jira] [Updated] (HDDS-6319) Fix read big file failure with EC policy 10+4.
[ https://issues.apache.org/jira/browse/HDDS-6319?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Mark Gui updated HDDS-6319:
---------------------------
Description:
Steps to reproduce:
./bin/ozone sh volume create vol1
./bin/ozone sh bucket create vol1/bucket1 --layout=FILE_SYSTEM_OPTIMIZED --replication=rs-10-4-1024k --type EC
./bin/ozone sh key put /vol1/bucket1/dd.2G dd.2G
./bin/ozone sh key get /vol1/bucket1/dd.2G down.2G
output:
{code:java}
java.lang.IndexOutOfBoundsException
at java.nio.ByteBuffer.wrap(ByteBuffer.java:375)
at org.apache.hadoop.ozone.client.io.ECBlockInputStreamProxy.read(ECBlockInputStreamProxy.java:143)
at org.apache.hadoop.hdds.scm.storage.ByteArrayReader.readFromBlock(ByteArrayReader.java:57)
at org.apache.hadoop.ozone.client.io.KeyInputStream.readWithStrategy(KeyInputStream.java:268)
at org.apache.hadoop.ozone.client.io.KeyInputStream.read(KeyInputStream.java:235)
at org.apache.hadoop.ozone.client.io.OzoneInputStream.read(OzoneInputStream.java:56)
at java.io.InputStream.read(InputStream.java:101)
at org.apache.hadoop.io.IOUtils.copyBytes(IOUtils.java:94)
at org.apache.hadoop.ozone.shell.keys.GetKeyHandler.execute(GetKeyHandler.java:88)
at org.apache.hadoop.ozone.shell.Handler.call(Handler.java:98)
at org.apache.hadoop.ozone.shell.Handler.call(Handler.java:44)
at picocli.CommandLine.executeUserObject(CommandLine.java:1953)
at picocli.CommandLine.access$1300(CommandLine.java:145)
at picocli.CommandLine$RunLast.executeUserObjectOfLastSubcommandWithSameParent(CommandLine.java:2352)
at picocli.CommandLine$RunLast.handle(CommandLine.java:2346)
at picocli.CommandLine$RunLast.handle(CommandLine.java:2311)
at picocli.CommandLine$AbstractParseResultHandler.handleParseResult(CommandLine.java:2172)
at picocli.CommandLine.parseWithHandlers(CommandLine.java:2550)
at picocli.CommandLine.parseWithHandler(CommandLine.java:2485)
at org.apache.hadoop.hdds.cli.GenericCli.execute(GenericCli.java:96)
at org.apache.hadoop.ozone.shell.OzoneShell.lambda$execute$17(OzoneShell.java:55)
at org.apache.hadoop.hdds.tracing.TracingUtil.executeInNewSpan(TracingUtil.java:159)
at org.apache.hadoop.ozone.shell.OzoneShell.execute(OzoneShell.java:53)
at org.apache.hadoop.hdds.cli.GenericCli.run(GenericCli.java:87)
at org.apache.hadoop.ozone.shell.OzoneShell.main(OzoneShell.java:47) {code}
This is due to an int overflow in KeyInputStream:
{code:java}
protected synchronized int readWithStrategy(ByteReaderStrategy strategy)
throws IOException {
Preconditions.checkArgument(strategy != null);
checkOpen();
int buffLen = strategy.getTargetLength();
int totalReadLen = 0;
while (buffLen > 0) {
// if we are at the last block and have read the entire block, return
if (blockStreams.size() == 0 ||
(blockStreams.size() - 1 <= blockIndex &&
blockStreams.get(blockIndex)
.getRemaining() == 0)) {
return totalReadLen == 0 ? EOF : totalReadLen;
}
// Get the current blockStream and read data from it
BlockExtendedInputStream current = blockStreams.get(blockIndex);
int numBytesToRead = Math.min(buffLen, (int)current.getRemaining()); <-- int overflow
int numBytesRead = strategy.readFromBlock(current, numBytesToRead);
if (numBytesRead != numBytesToRead) {
// This implies that there is either data loss or corruption in the
// chunk entries. Even EOF in the current stream would be covered in
// this case.
throw new IOException(String.format("Inconsistent read for blockID=%s "
+ "length=%d numBytesToRead=%d numBytesRead=%d",
current.getBlockID(), current.getLength(), numBytesToRead,
numBytesRead));
}
totalReadLen += numBytesRead;
buffLen -= numBytesRead;
if (current.getRemaining() <= 0 &&
((blockIndex + 1) < blockStreams.size())) {
blockIndex += 1;
}
}
return totalReadLen;
} {code}
KeyInputStream is common path for both replicate read and ec write, but ECBlockInputStream getLength() is the length of the whole big block group which is easily > INT_MAX under an EC policy of 10+4.
> Fix read big file failure with EC policy 10+4.
> ----------------------------------------------
>
> Key: HDDS-6319
> URL: https://issues.apache.org/jira/browse/HDDS-6319
> Project: Apache Ozone
> Issue Type: Bug
> Reporter: Mark Gui
> Assignee: Mark Gui
> Priority: Major
>
> Steps to reproduce:
> ./bin/ozone sh volume create vol1
> ./bin/ozone sh bucket create vol1/bucket1 --layout=FILE_SYSTEM_OPTIMIZED --replication=rs-10-4-1024k --type EC
> ./bin/ozone sh key put /vol1/bucket1/dd.2G dd.2G
> ./bin/ozone sh key get /vol1/bucket1/dd.2G down.2G
> output:
> {code:java}
> java.lang.IndexOutOfBoundsException
> at java.nio.ByteBuffer.wrap(ByteBuffer.java:375)
> at org.apache.hadoop.ozone.client.io.ECBlockInputStreamProxy.read(ECBlockInputStreamProxy.java:143)
> at org.apache.hadoop.hdds.scm.storage.ByteArrayReader.readFromBlock(ByteArrayReader.java:57)
> at org.apache.hadoop.ozone.client.io.KeyInputStream.readWithStrategy(KeyInputStream.java:268)
> at org.apache.hadoop.ozone.client.io.KeyInputStream.read(KeyInputStream.java:235)
> at org.apache.hadoop.ozone.client.io.OzoneInputStream.read(OzoneInputStream.java:56)
> at java.io.InputStream.read(InputStream.java:101)
> at org.apache.hadoop.io.IOUtils.copyBytes(IOUtils.java:94)
> at org.apache.hadoop.ozone.shell.keys.GetKeyHandler.execute(GetKeyHandler.java:88)
> at org.apache.hadoop.ozone.shell.Handler.call(Handler.java:98)
> at org.apache.hadoop.ozone.shell.Handler.call(Handler.java:44)
> at picocli.CommandLine.executeUserObject(CommandLine.java:1953)
> at picocli.CommandLine.access$1300(CommandLine.java:145)
> at picocli.CommandLine$RunLast.executeUserObjectOfLastSubcommandWithSameParent(CommandLine.java:2352)
> at picocli.CommandLine$RunLast.handle(CommandLine.java:2346)
> at picocli.CommandLine$RunLast.handle(CommandLine.java:2311)
> at picocli.CommandLine$AbstractParseResultHandler.handleParseResult(CommandLine.java:2172)
> at picocli.CommandLine.parseWithHandlers(CommandLine.java:2550)
> at picocli.CommandLine.parseWithHandler(CommandLine.java:2485)
> at org.apache.hadoop.hdds.cli.GenericCli.execute(GenericCli.java:96)
> at org.apache.hadoop.ozone.shell.OzoneShell.lambda$execute$17(OzoneShell.java:55)
> at org.apache.hadoop.hdds.tracing.TracingUtil.executeInNewSpan(TracingUtil.java:159)
> at org.apache.hadoop.ozone.shell.OzoneShell.execute(OzoneShell.java:53)
> at org.apache.hadoop.hdds.cli.GenericCli.run(GenericCli.java:87)
> at org.apache.hadoop.ozone.shell.OzoneShell.main(OzoneShell.java:47) {code}
> This is due to an int overflow in KeyInputStream:
> {code:java}
> protected synchronized int readWithStrategy(ByteReaderStrategy strategy)
> throws IOException {
> Preconditions.checkArgument(strategy != null);
> checkOpen();
> int buffLen = strategy.getTargetLength();
> int totalReadLen = 0;
> while (buffLen > 0) {
> // if we are at the last block and have read the entire block, return
> if (blockStreams.size() == 0 ||
> (blockStreams.size() - 1 <= blockIndex &&
> blockStreams.get(blockIndex)
> .getRemaining() == 0)) {
> return totalReadLen == 0 ? EOF : totalReadLen;
> }
> // Get the current blockStream and read data from it
> BlockExtendedInputStream current = blockStreams.get(blockIndex);
> int numBytesToRead = Math.min(buffLen, (int)current.getRemaining()); <-- int overflow
> int numBytesRead = strategy.readFromBlock(current, numBytesToRead);
> if (numBytesRead != numBytesToRead) {
> // This implies that there is either data loss or corruption in the
> // chunk entries. Even EOF in the current stream would be covered in
> // this case.
> throw new IOException(String.format("Inconsistent read for blockID=%s "
> + "length=%d numBytesToRead=%d numBytesRead=%d",
> current.getBlockID(), current.getLength(), numBytesToRead,
> numBytesRead));
> }
> totalReadLen += numBytesRead;
> buffLen -= numBytesRead;
> if (current.getRemaining() <= 0 &&
> ((blockIndex + 1) < blockStreams.size())) {
> blockIndex += 1;
> }
> }
> return totalReadLen;
> } {code}
> KeyInputStream is common path for both replicate read and ec write, but ECBlockInputStream getLength() is the length of the whole big block group which is easily > INT_MAX under an EC policy of 10+4.
--
This message was sent by Atlassian Jira
(v8.20.1#820001)
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org