You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by ad...@apache.org on 2023/04/22 06:48:39 UTC
[ozone] branch master updated: HDDS-8421. Use FileChannel.size() in FilePerBlockStrategy to reduce file operations (#4570)
This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 7573c0934c HDDS-8421. Use FileChannel.size() in FilePerBlockStrategy to reduce file operations (#4570)
7573c0934c is described below
commit 7573c0934c0a8c9fa2e654503219dd44f79e999b
Author: XiChen <32...@users.noreply.github.com>
AuthorDate: Sat Apr 22 14:48:33 2023 +0800
HDDS-8421. Use FileChannel.size() in FilePerBlockStrategy to reduce file operations (#4570)
---
.../container/keyvalue/helpers/ChunkUtils.java | 51 ++++++++++++++++++++++
.../keyvalue/impl/FilePerBlockStrategy.java | 12 ++---
.../container/keyvalue/helpers/TestChunkUtils.java | 13 ++++++
3 files changed, 71 insertions(+), 5 deletions(-)
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/ChunkUtils.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/ChunkUtils.java
index 8b42355d21..73d1f9070b 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/ChunkUtils.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/ChunkUtils.java
@@ -259,6 +259,33 @@ public final class ChunkUtils {
return false;
}
+
+ /**
+ * Validates chunk data and returns a boolean value that indicates if the
+ * chunk data should be overwritten.
+ *
+ * @param chunkFile - FileChannel of the chunkFile to write data into.
+ * @param info - chunk info.
+ * @return true if the chunkOffset is less than the chunkFile length,
+ * false otherwise.
+ */
+ public static boolean validateChunkForOverwrite(FileChannel chunkFile,
+ ChunkInfo info) {
+
+ if (isOverWriteRequested(chunkFile, info)) {
+ if (!isOverWritePermitted(info)) {
+ LOG.warn("Duplicate write chunk request. Chunk overwrite " +
+ "without explicit request. {}", info);
+ }
+ return true;
+ }
+
+ // TODO: when overwriting a chunk, we should ensure that the new chunk
+ // size is same as the old chunk size
+
+ return false;
+ }
+
/**
* Checks if we are getting a request to overwrite an existing range of
* chunk.
@@ -278,6 +305,30 @@ public final class ChunkUtils {
return offset < chunkFile.length();
}
+ /**
+ * Checks if a request to overwrite an existing range of a chunk has been
+ * received.
+ *
+ * @param channel - FileChannel of the file to check
+ * @param chunkInfo - Chunk information containing the offset
+ * @return true if the offset is less than the file length, indicating
+ * a request to overwrite an existing range; false otherwise
+ */
+ public static boolean isOverWriteRequested(FileChannel channel, ChunkInfo
+ chunkInfo) {
+ long fileLen;
+ try {
+ fileLen = channel.size();
+ } catch (IOException e) {
+ String msg = "IO error encountered while getting the file size";
+ LOG.error(msg, e.getMessage());
+ throw new UncheckedIOException("IO error encountered while " +
+ "getting the file size for ", e);
+ }
+ long offset = chunkInfo.getOffset();
+ return offset < fileLen;
+ }
+
/**
* Overwrite is permitted if an only if the user explicitly asks for it. We
* permit this iff the key/value pair contains a flag called
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/FilePerBlockStrategy.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/FilePerBlockStrategy.java
index 23db342da0..0e1f81ad6a 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/FilePerBlockStrategy.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/FilePerBlockStrategy.java
@@ -137,24 +137,26 @@ public class FilePerBlockStrategy implements ChunkManager {
.getContainerData();
File chunkFile = getChunkFile(container, blockID, info);
- boolean overwrite = validateChunkForOverwrite(chunkFile, info);
long len = info.getLen();
long offset = info.getOffset();
- if (LOG.isDebugEnabled()) {
- LOG.debug("Writing chunk {} (overwrite: {}) in stage {} to file {}",
- info, overwrite, stage, chunkFile);
- }
HddsVolume volume = containerData.getVolume();
FileChannel channel = null;
+ boolean overwrite;
try {
channel = files.getChannel(chunkFile, doSyncWrite);
+ overwrite = validateChunkForOverwrite(channel, info);
} catch (IOException e) {
onFailure(volume);
throw e;
}
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Writing chunk {} (overwrite: {}) in stage {} to file {}",
+ info, overwrite, stage, chunkFile);
+ }
+
// check whether offset matches block file length if its an overwrite
if (!overwrite) {
ChunkUtils.validateChunkSize(channel, info, chunkFile.getName());
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/helpers/TestChunkUtils.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/helpers/TestChunkUtils.java
index faf88d220b..592d6f3249 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/helpers/TestChunkUtils.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/helpers/TestChunkUtils.java
@@ -20,8 +20,10 @@ package org.apache.hadoop.ozone.container.keyvalue.helpers;
import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
import java.nio.file.Files;
import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
import java.util.Arrays;
import java.util.LinkedList;
import java.util.List;
@@ -198,6 +200,17 @@ public class TestChunkUtils {
Assertions.assertFalse(
ChunkUtils.validateChunkForOverwrite(tempFile.toFile(),
new ChunkInfo("chunk", 5, 5)));
+
+ try (FileChannel fileChannel =
+ FileChannel.open(tempFile, StandardOpenOption.READ)) {
+ Assertions.assertTrue(
+ ChunkUtils.validateChunkForOverwrite(fileChannel,
+ new ChunkInfo("chunk", 3, 5)));
+
+ Assertions.assertFalse(
+ ChunkUtils.validateChunkForOverwrite(fileChannel,
+ new ChunkInfo("chunk", 5, 5)));
+ }
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org