You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by ck...@apache.org on 2022/10/27 02:27:53 UTC
[ozone] branch ozone-1.3 updated (141ca24816 -> 355377f952)
This is an automated email from the ASF dual-hosted git repository.
ckj pushed a change to branch ozone-1.3
in repository https://gitbox.apache.org/repos/asf/ozone.git
from 141ca24816 HDDS-7422. Bump woodstox-core from 5.0.3 to 5.4.0 (#3886)
new bd389a9daa HDDS-7413. Fix logging while marking container state unhealthy (#3887)
new 355377f952 HDDS-7407. EC: Block allocation should not be stripped across the EC group (#3882)
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../hadoop/ozone/container/keyvalue/KeyValueContainer.java | 5 +++--
.../hadoop/hdds/scm/pipeline/WritableECContainerProvider.java | 11 ++++-------
.../hdds/scm/pipeline/TestWritableECContainerProvider.java | 5 ++---
3 files changed, 9 insertions(+), 12 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
[ozone] 02/02: HDDS-7407. EC: Block allocation should not be stripped across the EC group (#3882)
Posted by ck...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ckj pushed a commit to branch ozone-1.3
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 355377f9525fb5b6af136c1d1803cca65a66d54b
Author: Kaijie Chen <ck...@apache.org>
AuthorDate: Thu Oct 27 10:14:14 2022 +0800
HDDS-7407. EC: Block allocation should not be stripped across the EC group (#3882)
---
.../hadoop/hdds/scm/pipeline/WritableECContainerProvider.java | 11 ++++-------
.../hdds/scm/pipeline/TestWritableECContainerProvider.java | 5 ++---
2 files changed, 6 insertions(+), 10 deletions(-)
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/WritableECContainerProvider.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/WritableECContainerProvider.java
index 76e31d1524..7833f150f4 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/WritableECContainerProvider.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/WritableECContainerProvider.java
@@ -93,16 +93,13 @@ public class WritableECContainerProvider
public ContainerInfo getContainer(final long size,
ECReplicationConfig repConfig, String owner, ExcludeList excludeList)
throws IOException, TimeoutException {
- // Bound this at a minimum of 1 byte in case a request is made for a very
- // small size, which when divided by EC DataNum is zero.
- long requiredSpace = Math.max(1, size / repConfig.getData());
synchronized (this) {
int openPipelineCount = pipelineManager.getPipelineCount(repConfig,
Pipeline.PipelineState.OPEN);
if (openPipelineCount < providerConfig.getMinimumPipelines()) {
try {
return allocateContainer(
- repConfig, requiredSpace, owner, excludeList);
+ repConfig, size, owner, excludeList);
} catch (IOException e) {
LOG.warn("Unable to allocate a container for {} with {} existing "
+ "containers", repConfig, openPipelineCount, e);
@@ -115,7 +112,7 @@ public class WritableECContainerProvider
PipelineRequestInformation pri =
PipelineRequestInformation.Builder.getBuilder()
- .setSize(requiredSpace)
+ .setSize(size)
.build();
while (existingPipelines.size() > 0) {
Pipeline pipeline =
@@ -129,7 +126,7 @@ public class WritableECContainerProvider
try {
ContainerInfo containerInfo = getContainerFromPipeline(pipeline);
if (containerInfo == null
- || !containerHasSpace(containerInfo, requiredSpace)) {
+ || !containerHasSpace(containerInfo, size)) {
// This is O(n), which isn't great if there are a lot of pipelines
// and we keep finding pipelines without enough space.
existingPipelines.remove(pipeline);
@@ -153,7 +150,7 @@ public class WritableECContainerProvider
// allocate a new one and usePipelineManagerV2Impl.java it.
try {
synchronized (this) {
- return allocateContainer(repConfig, requiredSpace, owner, excludeList);
+ return allocateContainer(repConfig, size, owner, excludeList);
}
} catch (IOException e) {
LOG.error("Unable to allocate a container for {} after trying all "
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestWritableECContainerProvider.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestWritableECContainerProvider.java
index 09ae77581f..3296188707 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestWritableECContainerProvider.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestWritableECContainerProvider.java
@@ -298,12 +298,11 @@ public class TestWritableECContainerProvider {
// Update all the containers to make them nearly full, but with enough space
// for an EC block to be striped across them.
for (ContainerInfo c : allocatedContainers) {
- c.setUsedBytes(getMaxContainerSize() - 30 * 1024 * 1024);
+ c.setUsedBytes(getMaxContainerSize() - 90 * 1024 * 1024);
}
// Get a new container of size 50 and ensure it is one of the original set.
- // We ask for a space of 50, but as it is stripped across the EC group it
- // will actually need 50 / dataNum space
+ // We ask for a space of 50 MB, and will actually need 50 MB space.
ContainerInfo newContainer =
provider.getContainer(50 * 1024 * 1024, repConfig, OWNER,
new ExcludeList());
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
[ozone] 01/02: HDDS-7413. Fix logging while marking container state unhealthy (#3887)
Posted by ck...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ckj pushed a commit to branch ozone-1.3
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit bd389a9daa51ae6c6d8446527b66f8e27dbb66c1
Author: Swaminathan Balachandran <47...@users.noreply.github.com>
AuthorDate: Wed Oct 26 00:26:45 2022 -0700
HDDS-7413. Fix logging while marking container state unhealthy (#3887)
---
.../apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java
index 9fc8c44662..7412e766d0 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java
@@ -328,6 +328,7 @@ public class KeyValueContainer implements Container<KeyValueContainerData> {
@Override
public void markContainerUnhealthy() throws StorageContainerException {
writeLock();
+ ContainerDataProto.State prevState = containerData.getState();
try {
updateContainerData(() ->
containerData.setState(ContainerDataProto.State.UNHEALTHY));
@@ -335,9 +336,9 @@ public class KeyValueContainer implements Container<KeyValueContainerData> {
} finally {
writeUnlock();
}
- LOG.warn("Moving container {} to state UNHEALTHY from state:{} Trace:{}",
+ LOG.warn("Moving container {} to state {} from state:{} Trace:{}",
containerData.getContainerPath(), containerData.getState(),
- StringUtils.getStackTrace(Thread.currentThread()));
+ prevState, StringUtils.getStackTrace(Thread.currentThread()));
}
@Override
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org