You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by so...@apache.org on 2020/07/01 10:24:09 UTC
[hadoop-ozone] branch master updated: HDDS-3831. Enhance Recon
ContainerEndPoint to report on different unhealty container states (#1148)
This is an automated email from the ASF dual-hosted git repository.
sodonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 8ff5c8e HDDS-3831. Enhance Recon ContainerEndPoint to report on different unhealty container states (#1148)
8ff5c8e is described below
commit 8ff5c8eaa222ae4afb3ee90bc4e683599ab117b5
Author: Stephen O'Donnell <st...@gmail.com>
AuthorDate: Wed Jul 1 11:23:52 2020 +0100
HDDS-3831. Enhance Recon ContainerEndPoint to report on different unhealty container states (#1148)
---
.../apache/hadoop/ozone/recon/TestReconTasks.java | 9 +-
.../apache/hadoop/ozone/recon/ReconConstants.java | 2 +
.../hadoop/ozone/recon/api/ContainerEndpoint.java | 124 ++++++++++--
.../api/types/UnhealthyContainerMetadata.java | 119 +++++++++++
.../api/types/UnhealthyContainersResponse.java | 98 +++++++++
.../api/types/UnhealthyContainersSummary.java | 41 ++++
.../recon/persistence/ContainerSchemaManager.java | 50 ++++-
.../recon/spi/ContainerDBServiceProvider.java | 8 -
.../spi/impl/ContainerDBServiceProviderImpl.java | 6 -
.../ozone/recon/api/TestContainerEndpoint.java | 224 ++++++++++++++++++++-
10 files changed, 645 insertions(+), 36 deletions(-)
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconTasks.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconTasks.java
index 0e5a721..84fa45d 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconTasks.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconTasks.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.recon.scm.ReconContainerManager;
import org.apache.hadoop.ozone.recon.scm.ReconStorageContainerManagerFacade;
import org.apache.hadoop.test.LambdaTestUtils;
+import org.hadoop.ozone.recon.schema.ContainerSchemaDefinition;
import org.hadoop.ozone.recon.schema.tables.pojos.UnhealthyContainers;
import org.junit.After;
import org.junit.Before;
@@ -116,7 +117,9 @@ public class TestReconTasks {
LambdaTestUtils.await(120000, 10000, () -> {
List<UnhealthyContainers> allMissingContainers =
reconContainerManager.getContainerSchemaManager()
- .getAllMissingContainers();
+ .getUnhealthyContainers(
+ ContainerSchemaDefinition.UnHealthyContainerStates.MISSING,
+ 0, 1000);
return (allMissingContainers.size() == 1);
});
@@ -125,7 +128,9 @@ public class TestReconTasks {
LambdaTestUtils.await(120000, 10000, () -> {
List<UnhealthyContainers> allMissingContainers =
reconContainerManager.getContainerSchemaManager()
- .getAllMissingContainers();
+ .getUnhealthyContainers(
+ ContainerSchemaDefinition.UnHealthyContainerStates.MISSING,
+ 0, 1000);
return (allMissingContainers.isEmpty());
});
}
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java
index b0a5c0b..972399f 100644
--- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java
@@ -45,6 +45,8 @@ public final class ReconConstants {
// By default, limit the number of results returned
public static final String DEFAULT_FETCH_COUNT = "1000";
+ public static final String DEFAULT_BATCH_NUMBER = "1";
+ public static final String RECON_QUERY_BATCH_PARAM = "batchNum";
public static final String RECON_QUERY_PREVKEY = "prevKey";
public static final String PREV_CONTAINER_ID_DEFAULT_VALUE = "0";
public static final String RECON_QUERY_LIMIT = "limit";
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/ContainerEndpoint.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/ContainerEndpoint.java
index b4f9792..c534062 100644
--- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/ContainerEndpoint.java
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/ContainerEndpoint.java
@@ -52,14 +52,21 @@ import org.apache.hadoop.ozone.recon.api.types.KeyMetadata.ContainerBlockMetadat
import org.apache.hadoop.ozone.recon.api.types.KeysResponse;
import org.apache.hadoop.ozone.recon.api.types.MissingContainerMetadata;
import org.apache.hadoop.ozone.recon.api.types.MissingContainersResponse;
+import org.apache.hadoop.ozone.recon.api.types.UnhealthyContainerMetadata;
+import org.apache.hadoop.ozone.recon.api.types.UnhealthyContainersResponse;
+import org.apache.hadoop.ozone.recon.api.types.UnhealthyContainersSummary;
import org.apache.hadoop.ozone.recon.persistence.ContainerSchemaManager;
import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager;
import org.apache.hadoop.ozone.recon.scm.ReconContainerManager;
import org.apache.hadoop.ozone.recon.spi.ContainerDBServiceProvider;
+import org.hadoop.ozone.recon.schema.ContainerSchemaDefinition.UnHealthyContainerStates;
import org.hadoop.ozone.recon.schema.tables.pojos.ContainerHistory;
+import org.hadoop.ozone.recon.schema.tables.pojos.UnhealthyContainers;
+import static org.apache.hadoop.ozone.recon.ReconConstants.DEFAULT_BATCH_NUMBER;
import static org.apache.hadoop.ozone.recon.ReconConstants.DEFAULT_FETCH_COUNT;
import static org.apache.hadoop.ozone.recon.ReconConstants.PREV_CONTAINER_ID_DEFAULT_VALUE;
+import static org.apache.hadoop.ozone.recon.ReconConstants.RECON_QUERY_BATCH_PARAM;
import static org.apache.hadoop.ozone.recon.ReconConstants.RECON_QUERY_LIMIT;
import static org.apache.hadoop.ozone.recon.ReconConstants.RECON_QUERY_PREVKEY;
@@ -233,31 +240,120 @@ public class ContainerEndpoint {
@Path("/missing")
public Response getMissingContainers() {
List<MissingContainerMetadata> missingContainers = new ArrayList<>();
- containerDBServiceProvider.getMissingContainers().forEach(container -> {
- long containerID = container.getContainerId();
- try {
+ containerSchemaManager.getUnhealthyContainers(
+ UnHealthyContainerStates.MISSING, 0, Integer.MAX_VALUE)
+ .forEach(container -> {
+ long containerID = container.getContainerId();
+ try {
+ ContainerInfo containerInfo =
+ containerManager.getContainer(new ContainerID(containerID));
+ long keyCount = containerInfo.getNumberOfKeys();
+ UUID pipelineID = containerInfo.getPipelineID().getId();
+
+ List<ContainerHistory> datanodes =
+ containerSchemaManager.getLatestContainerHistory(containerID,
+ containerInfo.getReplicationFactor().getNumber());
+ missingContainers.add(new MissingContainerMetadata(containerID,
+ container.getInStateSince(), keyCount, pipelineID, datanodes));
+ } catch (IOException ioEx) {
+ throw new WebApplicationException(ioEx,
+ Response.Status.INTERNAL_SERVER_ERROR);
+ }
+ });
+ MissingContainersResponse response =
+ new MissingContainersResponse(missingContainers.size(),
+ missingContainers);
+ return Response.ok(response).build();
+ }
+
+ /**
+ * Return
+ * {@link org.apache.hadoop.ozone.recon.api.types.UnhealthyContainerMetadata}
+ * for all unhealthy containers.
+ *
+ * @param state Return only containers matching the given unhealthy state,
+ * eg UNDER_REPLICATED, MIS_REPLICATED, OVER_REPLICATED or
+ * MISSING. Passing null returns all containers.
+ * @param limit The limit of unhealthy containers to return.
+ * @param batchNum The batch number (like "page number") of results to return.
+ * Passing 1, will return records 1 to limit. 2 will return
+ * limit + 1 to 2 * limit, etc.
+ * @return {@link Response}
+ */
+ @GET
+ @Path("/unhealthy/{state}")
+ public Response getUnhealthyContainers(
+ @PathParam("state") String state,
+ @DefaultValue(DEFAULT_FETCH_COUNT) @QueryParam(RECON_QUERY_LIMIT)
+ int limit,
+ @DefaultValue(DEFAULT_BATCH_NUMBER)
+ @QueryParam(RECON_QUERY_BATCH_PARAM) int batchNum) {
+ int offset = Math.max(((batchNum - 1) * limit), 0);
+
+ List<UnhealthyContainerMetadata> unhealthyMeta = new ArrayList<>();
+ List<UnhealthyContainersSummary> summary;
+ try {
+ UnHealthyContainerStates internalState = null;
+
+ if (state != null) {
+ // If an invalid state is passed in, this will throw
+ // illegalArgumentException and fail the request
+ internalState = UnHealthyContainerStates.valueOf(state);
+ }
+
+ summary = containerSchemaManager.getUnhealthyContainersSummary();
+ List<UnhealthyContainers> containers = containerSchemaManager
+ .getUnhealthyContainers(internalState, offset, limit);
+ for (UnhealthyContainers c : containers) {
+ long containerID = c.getContainerId();
ContainerInfo containerInfo =
containerManager.getContainer(new ContainerID(containerID));
long keyCount = containerInfo.getNumberOfKeys();
UUID pipelineID = containerInfo.getPipelineID().getId();
-
List<ContainerHistory> datanodes =
containerSchemaManager.getLatestContainerHistory(
- containerID, containerInfo.getReplicationFactor().getNumber());
- missingContainers.add(new MissingContainerMetadata(containerID,
- container.getInStateSince(), keyCount, pipelineID, datanodes));
- } catch (IOException ioEx) {
- throw new WebApplicationException(ioEx,
- Response.Status.INTERNAL_SERVER_ERROR);
+ containerID,
+ containerInfo.getReplicationFactor().getNumber());
+ unhealthyMeta.add(new UnhealthyContainerMetadata(
+ c, datanodes, pipelineID, keyCount));
}
- });
- MissingContainersResponse response =
- new MissingContainersResponse(missingContainers.size(),
- missingContainers);
+ } catch (IOException ex) {
+ throw new WebApplicationException(ex,
+ Response.Status.INTERNAL_SERVER_ERROR);
+ } catch (IllegalArgumentException e) {
+ throw new WebApplicationException(e, Response.Status.BAD_REQUEST);
+ }
+
+ UnhealthyContainersResponse response =
+ new UnhealthyContainersResponse(unhealthyMeta);
+ for (UnhealthyContainersSummary s : summary) {
+ response.setSummaryCount(s.getContainerState(), s.getCount());
+ }
return Response.ok(response).build();
}
/**
+ * Return
+ * {@link org.apache.hadoop.ozone.recon.api.types.UnhealthyContainerMetadata}
+ * for all unhealthy containers.
+
+ * @param limit The limit of unhealthy containers to return.
+ * @param batchNum The batch number (like "page number") of results to return.
+ * Passing 1, will return records 1 to limit. 2 will return
+ * limit + 1 to 2 * limit, etc.
+ * @return {@link Response}
+ */
+ @GET
+ @Path("/unhealthy")
+ public Response getUnhealthyContainers(
+ @DefaultValue(DEFAULT_FETCH_COUNT) @QueryParam(RECON_QUERY_LIMIT)
+ int limit,
+ @DefaultValue(DEFAULT_BATCH_NUMBER)
+ @QueryParam(RECON_QUERY_BATCH_PARAM) int batchNum) {
+ return getUnhealthyContainers(null, limit, batchNum);
+ }
+
+ /**
* Helper function to extract the blocks for a given container from a given
* OM Key.
* @param matchedKeys List of OM Key Info locations
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/UnhealthyContainerMetadata.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/UnhealthyContainerMetadata.java
new file mode 100644
index 0000000..370c2a6
--- /dev/null
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/UnhealthyContainerMetadata.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.recon.api.types;
+
+import org.hadoop.ozone.recon.schema.tables.pojos.ContainerHistory;
+import org.hadoop.ozone.recon.schema.tables.pojos.UnhealthyContainers;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+import java.util.List;
+import java.util.UUID;
+
+/**
+ * Metadata object that represents an unhealthy Container.
+ */
+@XmlAccessorType(XmlAccessType.FIELD)
+public class UnhealthyContainerMetadata {
+
+ @XmlElement(name = "containerID")
+ private long containerID;
+
+ @XmlElement(name = "containerState")
+ private String containerState;
+
+ @XmlElement(name = "unhealthySince")
+ private long unhealthySince;
+
+ @XmlElement(name = "expectedReplicaCount")
+ private long expectedReplicaCount = 0;
+
+ @XmlElement(name = "actualReplicaCount")
+ private long actualReplicaCount = 0;
+
+ @XmlElement(name = "replicaDeltaCount")
+ private long replicaDeltaCount = 0;
+
+ @XmlElement(name = "reason")
+ private String reason;
+
+ @XmlElement(name = "keys")
+ private long keys;
+
+ @XmlElement(name = "pipelineID")
+ private UUID pipelineID;
+
+ @XmlElement(name = "replicas")
+ private List<ContainerHistory> replicas;
+
+ public UnhealthyContainerMetadata(UnhealthyContainers rec,
+ List<ContainerHistory> replicas, UUID pipelineID, long keyCount) {
+ this.containerID = rec.getContainerId();
+ this.containerState = rec.getContainerState();
+ this.unhealthySince = rec.getInStateSince();
+ this.actualReplicaCount = rec.getActualReplicaCount();
+ this.expectedReplicaCount = rec.getExpectedReplicaCount();
+ this.replicaDeltaCount = rec.getReplicaDelta();
+ this.reason = rec.getReason();
+ this.replicas = replicas;
+ this.pipelineID = pipelineID;
+ this.keys = keyCount;
+ }
+
+ public long getContainerID() {
+ return containerID;
+ }
+
+ public long getKeys() {
+ return keys;
+ }
+
+ public List<ContainerHistory> getReplicas() {
+ return replicas;
+ }
+
+ public String getContainerState() {
+ return containerState;
+ }
+
+ public long getExpectedReplicaCount() {
+ return expectedReplicaCount;
+ }
+
+ public long getActualReplicaCount() {
+ return actualReplicaCount;
+ }
+
+ public long getReplicaDeltaCount() {
+ return replicaDeltaCount;
+ }
+
+ public String getReason() {
+ return reason;
+ }
+
+ public long getUnhealthySince() {
+ return unhealthySince;
+ }
+
+ public UUID getPipelineID() {
+ return pipelineID;
+ }
+
+}
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/UnhealthyContainersResponse.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/UnhealthyContainersResponse.java
new file mode 100644
index 0000000..ef40329
--- /dev/null
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/UnhealthyContainersResponse.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.recon.api.types;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.hadoop.ozone.recon.schema.ContainerSchemaDefinition.UnHealthyContainerStates;
+
+import java.util.Collection;
+
+/**
+ * Class that represents the API Response structure of Unhealthy Containers.
+ */
+public class UnhealthyContainersResponse {
+ /**
+ * Total count of the missing containers.
+ */
+ @JsonProperty("missingCount")
+ private long missingCount = 0;
+
+ /**
+ * Total count of under replicated containers.
+ */
+ @JsonProperty("underReplicatedCount")
+ private long underReplicatedCount = 0;
+
+ /**
+ * Total count of over replicated containers.
+ */
+ @JsonProperty("overReplicatedCount")
+ private long overReplicatedCount = 0;
+
+ /**
+ * Total count of mis-replicated containers.
+ */
+ @JsonProperty("misReplicatedCount")
+ private long misReplicatedCount = 0;
+
+ /**
+ * A collection of unhealthy containers.
+ */
+ @JsonProperty("containers")
+ private Collection<UnhealthyContainerMetadata> containers;
+
+ public UnhealthyContainersResponse(Collection<UnhealthyContainerMetadata>
+ containers) {
+ this.containers = containers;
+ }
+
+ public void setSummaryCount(String state, long count) {
+ if (state.equals(UnHealthyContainerStates.MISSING.toString())) {
+ this.missingCount = count;
+ } else if (state.equals(
+ UnHealthyContainerStates.OVER_REPLICATED.toString())) {
+ this.overReplicatedCount = count;
+ } else if (state.equals(
+ UnHealthyContainerStates.UNDER_REPLICATED.toString())) {
+ this.underReplicatedCount = count;
+ } else if (state.equals(
+ UnHealthyContainerStates.MIS_REPLICATED.toString())) {
+ this.misReplicatedCount = count;
+ }
+ }
+
+ public long getMissingCount() {
+ return missingCount;
+ }
+
+ public long getUnderReplicatedCount() {
+ return underReplicatedCount;
+ }
+
+ public long getOverReplicatedCount() {
+ return overReplicatedCount;
+ }
+
+ public long getMisReplicatedCount() {
+ return misReplicatedCount;
+ }
+
+ public Collection<UnhealthyContainerMetadata> getContainers() {
+ return containers;
+ }
+}
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/UnhealthyContainersSummary.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/UnhealthyContainersSummary.java
new file mode 100644
index 0000000..2ff8538
--- /dev/null
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/UnhealthyContainersSummary.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.recon.api.types;
+
+/**
+ * Simple POJO to receive the results of a Jooq query.
+ */
+public class UnhealthyContainersSummary {
+
+ private Integer count;
+ private String containerState;
+
+ public UnhealthyContainersSummary(String containerState, Integer cnt) {
+ this.containerState = containerState;
+ this.count = cnt;
+ }
+
+ public String getContainerState() {
+ return containerState;
+ }
+
+ public long getCount() {
+ return count.longValue();
+ }
+
+}
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/persistence/ContainerSchemaManager.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/persistence/ContainerSchemaManager.java
index 74183f8..bf37c34 100644
--- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/persistence/ContainerSchemaManager.java
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/persistence/ContainerSchemaManager.java
@@ -19,9 +19,11 @@ package org.apache.hadoop.ozone.recon.persistence;
import static org.hadoop.ozone.recon.schema.tables.ContainerHistoryTable.CONTAINER_HISTORY;
import static org.hadoop.ozone.recon.schema.tables.UnhealthyContainersTable.UNHEALTHY_CONTAINERS;
+import static org.jooq.impl.DSL.count;
import com.google.inject.Inject;
import com.google.inject.Singleton;
+import org.apache.hadoop.ozone.recon.api.types.UnhealthyContainersSummary;
import org.hadoop.ozone.recon.schema.ContainerSchemaDefinition;
import org.hadoop.ozone.recon.schema.ContainerSchemaDefinition.UnHealthyContainerStates;
import org.hadoop.ozone.recon.schema.tables.daos.ContainerHistoryDao;
@@ -31,7 +33,9 @@ import org.hadoop.ozone.recon.schema.tables.pojos.UnhealthyContainers;
import org.hadoop.ozone.recon.schema.tables.records.UnhealthyContainersRecord;
import org.jooq.Cursor;
import org.jooq.DSLContext;
+import org.jooq.Record;
import org.jooq.Record2;
+import org.jooq.SelectQuery;
import java.util.List;
/**
@@ -52,9 +56,49 @@ public class ContainerSchemaManager {
this.containerSchemaDefinition = containerSchemaDefinition;
}
- public List<UnhealthyContainers> getAllMissingContainers() {
- return unhealthyContainersDao
- .fetchByContainerState(UnHealthyContainerStates.MISSING.toString());
+ /**
+ * Get a batch of unhealthy containers, starting at offset and returning
+ * limit records. If a null value is passed for state, then unhealthy
+ * containers in all states will be returned. Otherwise, only containers
+ * matching the given state will be returned.
+ * @param state Return only containers in this state, or all containers if
+ * null
+ * @param offset The starting record to return in the result set. The first
+ * record is at zero.
+ * @param limit The total records to return
+ * @return List of unhealthy containers.
+ */
+ public List<UnhealthyContainers> getUnhealthyContainers(
+ UnHealthyContainerStates state, int offset, int limit) {
+ DSLContext dslContext = containerSchemaDefinition.getDSLContext();
+ SelectQuery<Record> query = dslContext.selectQuery();
+ query.addFrom(UNHEALTHY_CONTAINERS);
+ if (state != null) {
+ query.addConditions(
+ UNHEALTHY_CONTAINERS.CONTAINER_STATE.eq(state.toString()));
+ }
+ query.addOrderBy(UNHEALTHY_CONTAINERS.CONTAINER_ID.asc(),
+ UNHEALTHY_CONTAINERS.CONTAINER_STATE.asc());
+ query.addOffset(offset);
+ query.addLimit(limit);
+
+ return query.fetchInto(UnhealthyContainers.class);
+ }
+
+ /**
+ * Obtain a count of all containers in each state. If there are no unhealthy
+ * containers an empty list will be returned. If there are unhealthy
+ * containers for a certain state, no entry will be returned for it.
+ * @return Count of unhealthy containers in each state
+ */
+ public List<UnhealthyContainersSummary> getUnhealthyContainersSummary() {
+ DSLContext dslContext = containerSchemaDefinition.getDSLContext();
+ return dslContext
+ .select(UNHEALTHY_CONTAINERS.CONTAINER_STATE.as("containerState"),
+ count().as("cnt"))
+ .from(UNHEALTHY_CONTAINERS)
+ .groupBy(UNHEALTHY_CONTAINERS.CONTAINER_STATE)
+ .fetchInto(UnhealthyContainersSummary.class);
}
public Cursor<UnhealthyContainersRecord> getAllUnhealthyRecordsCursor() {
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/ContainerDBServiceProvider.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/ContainerDBServiceProvider.java
index 26b971c..8e7267d 100644
--- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/ContainerDBServiceProvider.java
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/ContainerDBServiceProvider.java
@@ -19,14 +19,12 @@
package org.apache.hadoop.ozone.recon.spi;
import java.io.IOException;
-import java.util.List;
import java.util.Map;
import org.apache.hadoop.hdds.annotation.InterfaceStability;
import org.apache.hadoop.ozone.recon.api.types.ContainerKeyPrefix;
import org.apache.hadoop.ozone.recon.api.types.ContainerMetadata;
import org.apache.hadoop.hdds.utils.db.TableIterator;
-import org.hadoop.ozone.recon.schema.tables.pojos.UnhealthyContainers;
/**
* The Recon Container DB Service interface.
@@ -164,10 +162,4 @@ public interface ContainerDBServiceProvider {
*/
void incrementContainerCountBy(long count);
- /**
- * Get all the missing containers.
- *
- * @return List of MissingContainers.
- */
- List<UnhealthyContainers> getMissingContainers();
}
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ContainerDBServiceProviderImpl.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ContainerDBServiceProviderImpl.java
index c196745..ec87352 100644
--- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ContainerDBServiceProviderImpl.java
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ContainerDBServiceProviderImpl.java
@@ -30,7 +30,6 @@ import java.io.File;
import java.io.IOException;
import java.sql.Timestamp;
import java.util.LinkedHashMap;
-import java.util.List;
import java.util.Map;
import javax.inject.Inject;
@@ -50,7 +49,6 @@ import org.apache.hadoop.hdds.utils.db.Table.KeyValue;
import org.apache.hadoop.hdds.utils.db.TableIterator;
import org.hadoop.ozone.recon.schema.tables.daos.GlobalStatsDao;
import org.hadoop.ozone.recon.schema.tables.pojos.GlobalStats;
-import org.hadoop.ozone.recon.schema.tables.pojos.UnhealthyContainers;
import org.jooq.Configuration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -358,10 +356,6 @@ public class ContainerDBServiceProviderImpl
return containers;
}
- public List<UnhealthyContainers> getMissingContainers() {
- return containerSchemaManager.getAllMissingContainers();
- }
-
@Override
public void deleteContainerMapping(ContainerKeyPrefix containerKeyPrefix)
throws IOException {
diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerEndpoint.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerEndpoint.java
index 2650ce7..6ba6f56 100644
--- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerEndpoint.java
+++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerEndpoint.java
@@ -24,7 +24,9 @@ import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getTestRe
import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.initializeNewOmMetadataManager;
import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeDataToOm;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -37,6 +39,8 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.stream.Collectors;
+import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.Response;
import org.apache.commons.lang3.StringUtils;
@@ -58,6 +62,8 @@ import org.apache.hadoop.ozone.recon.api.types.KeyMetadata;
import org.apache.hadoop.ozone.recon.api.types.KeysResponse;
import org.apache.hadoop.ozone.recon.api.types.MissingContainerMetadata;
import org.apache.hadoop.ozone.recon.api.types.MissingContainersResponse;
+import org.apache.hadoop.ozone.recon.api.types.UnhealthyContainerMetadata;
+import org.apache.hadoop.ozone.recon.api.types.UnhealthyContainersResponse;
import org.apache.hadoop.ozone.recon.persistence.ContainerSchemaManager;
import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager;
import org.apache.hadoop.ozone.recon.scm.ReconContainerManager;
@@ -69,6 +75,7 @@ import org.apache.hadoop.ozone.recon.spi.impl.StorageContainerServiceProviderImp
import org.apache.hadoop.ozone.recon.tasks.ContainerKeyMapperTask;
import org.apache.hadoop.hdds.utils.db.Table;
import org.hadoop.ozone.recon.schema.ContainerSchemaDefinition;
+import org.hadoop.ozone.recon.schema.ContainerSchemaDefinition.UnHealthyContainerStates;
import org.hadoop.ozone.recon.schema.tables.pojos.ContainerHistory;
import org.hadoop.ozone.recon.schema.tables.pojos.UnhealthyContainers;
import org.junit.Assert;
@@ -76,6 +83,7 @@ import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
+import org.mockito.Mockito;
/**
* Test for container endpoint.
@@ -108,14 +116,14 @@ public class TestContainerEndpoint {
ContainerManager mockContainerManager =
mock(ReconContainerManager.class);
- when(mockContainerManager.getContainer(containerID)).thenReturn(
+ when(mockContainerManager.getContainer(Mockito.any(ContainerID.class)))
+ .thenReturn(
new ContainerInfo.Builder()
.setContainerID(containerID.getId())
.setNumberOfKeys(keyCount)
.setReplicationFactor(ReplicationFactor.THREE)
.setPipelineID(pipelineID)
- .build()
- );
+ .build());
when(mockReconSCM.getContainerManager())
.thenReturn(mockContainerManager);
@@ -447,6 +455,169 @@ public class TestContainerEndpoint {
}
@Test
+ public void testUnhealthyContainers() {
+ Response response = containerEndpoint.getUnhealthyContainers(1000, 1);
+
+ UnhealthyContainersResponse responseObject =
+ (UnhealthyContainersResponse) response.getEntity();
+
+ assertEquals(0, responseObject.getMissingCount());
+ assertEquals(0, responseObject.getOverReplicatedCount());
+ assertEquals(0, responseObject.getUnderReplicatedCount());
+ assertEquals(0, responseObject.getMisReplicatedCount());
+
+ assertEquals(Collections.EMPTY_LIST, responseObject.getContainers());
+
+ createUnhealthyRecords(5, 4, 3, 2);
+
+ response = containerEndpoint.getUnhealthyContainers(1000, 1);
+
+ responseObject = (UnhealthyContainersResponse) response.getEntity();
+ assertEquals(5, responseObject.getMissingCount());
+ assertEquals(4, responseObject.getOverReplicatedCount());
+ assertEquals(3, responseObject.getUnderReplicatedCount());
+ assertEquals(2, responseObject.getMisReplicatedCount());
+
+ Collection<UnhealthyContainerMetadata> records
+ = responseObject.getContainers();
+ List<UnhealthyContainerMetadata> missing = records
+ .stream()
+ .filter(r -> r.getContainerState()
+ .equals(UnHealthyContainerStates.MISSING.toString()))
+ .collect(Collectors.toList());
+ assertEquals(5, missing.size());
+ assertEquals(3, missing.get(0).getExpectedReplicaCount());
+ assertEquals(0, missing.get(0).getActualReplicaCount());
+ assertEquals(3, missing.get(0).getReplicaDeltaCount());
+ assertEquals(12345L, missing.get(0).getUnhealthySince());
+ assertEquals(1L, missing.get(0).getContainerID());
+ assertEquals(keyCount, missing.get(0).getKeys());
+ assertEquals(pipelineID.getId(), missing.get(0).getPipelineID());
+ assertEquals(3, missing.get(0).getReplicas().size());
+ assertNull(missing.get(0).getReason());
+
+ Set<String> datanodes = Collections.unmodifiableSet(
+ new HashSet<>(Arrays.asList("host2", "host3", "host4")));
+ List<ContainerHistory> containerReplicas = missing.get(0).getReplicas();
+ containerReplicas.forEach(history -> {
+ Assert.assertTrue(datanodes.contains(history.getDatanodeHost()));
+ });
+
+ List<UnhealthyContainerMetadata> overRep = records
+ .stream()
+ .filter(r -> r.getContainerState()
+ .equals(UnHealthyContainerStates.OVER_REPLICATED.toString()))
+ .collect(Collectors.toList());
+ assertEquals(4, overRep.size());
+ assertEquals(3, overRep.get(0).getExpectedReplicaCount());
+ assertEquals(5, overRep.get(0).getActualReplicaCount());
+ assertEquals(-2, overRep.get(0).getReplicaDeltaCount());
+ assertEquals(12345L, overRep.get(0).getUnhealthySince());
+ assertEquals(6L, overRep.get(0).getContainerID());
+ assertNull(overRep.get(0).getReason());
+
+ List<UnhealthyContainerMetadata> underRep = records
+ .stream()
+ .filter(r -> r.getContainerState()
+ .equals(UnHealthyContainerStates.UNDER_REPLICATED.toString()))
+ .collect(Collectors.toList());
+ assertEquals(3, underRep.size());
+ assertEquals(3, underRep.get(0).getExpectedReplicaCount());
+ assertEquals(1, underRep.get(0).getActualReplicaCount());
+ assertEquals(2, underRep.get(0).getReplicaDeltaCount());
+ assertEquals(12345L, underRep.get(0).getUnhealthySince());
+ assertEquals(10L, underRep.get(0).getContainerID());
+ assertNull(underRep.get(0).getReason());
+
+ List<UnhealthyContainerMetadata> misRep = records
+ .stream()
+ .filter(r -> r.getContainerState()
+ .equals(UnHealthyContainerStates.MIS_REPLICATED.toString()))
+ .collect(Collectors.toList());
+ assertEquals(2, misRep.size());
+ assertEquals(2, misRep.get(0).getExpectedReplicaCount());
+ assertEquals(1, misRep.get(0).getActualReplicaCount());
+ assertEquals(1, misRep.get(0).getReplicaDeltaCount());
+ assertEquals(12345L, misRep.get(0).getUnhealthySince());
+ assertEquals(13L, misRep.get(0).getContainerID());
+ assertEquals("some reason", misRep.get(0).getReason());
+ }
+
+ @Test
+ public void testUnhealthyContainersFilteredResponse() {
+ String missing = UnHealthyContainerStates.MISSING.toString();
+
+ Response response = containerEndpoint
+ .getUnhealthyContainers(missing, 1000, 1);
+
+ UnhealthyContainersResponse responseObject =
+ (UnhealthyContainersResponse) response.getEntity();
+
+ assertEquals(0, responseObject.getMissingCount());
+ assertEquals(0, responseObject.getOverReplicatedCount());
+ assertEquals(0, responseObject.getUnderReplicatedCount());
+ assertEquals(0, responseObject.getMisReplicatedCount());
+ assertEquals(Collections.EMPTY_LIST, responseObject.getContainers());
+
+ createUnhealthyRecords(5, 4, 3, 2);
+
+ response = containerEndpoint.getUnhealthyContainers(missing, 1000, 1);
+
+ responseObject = (UnhealthyContainersResponse) response.getEntity();
+ // Summary should have the count for all unhealthy:
+ assertEquals(5, responseObject.getMissingCount());
+ assertEquals(4, responseObject.getOverReplicatedCount());
+ assertEquals(3, responseObject.getUnderReplicatedCount());
+ assertEquals(2, responseObject.getMisReplicatedCount());
+
+ Collection<UnhealthyContainerMetadata> records
+ = responseObject.getContainers();
+
+ // There should only be 5 missing containers and no others as we asked for
+ // only missing.
+ assertEquals(5, records.size());
+ for (UnhealthyContainerMetadata r : records) {
+ assertEquals(missing, r.getContainerState());
+ }
+ }
+
+ @Test
+ public void testUnhealthyContainersInvalidState() {
+ try {
+ containerEndpoint.getUnhealthyContainers("invalid", 1000, 1);
+ fail("Expected exception to be raised");
+ } catch (WebApplicationException e) {
+ assertEquals("HTTP 400 Bad Request", e.getMessage());
+ }
+ }
+
+ @Test
+ public void testUnhealthyContainersPaging() {
+ createUnhealthyRecords(5, 4, 3, 2);
+ UnhealthyContainersResponse firstBatch =
+ (UnhealthyContainersResponse) containerEndpoint.getUnhealthyContainers(
+ 3, 1).getEntity();
+
+ UnhealthyContainersResponse secondBatch =
+ (UnhealthyContainersResponse) containerEndpoint.getUnhealthyContainers(
+ 3, 2).getEntity();
+
+ ArrayList<UnhealthyContainerMetadata> records
+ = new ArrayList<>(firstBatch.getContainers());
+ assertEquals(3, records.size());
+ assertEquals(1L, records.get(0).getContainerID());
+ assertEquals(2L, records.get(1).getContainerID());
+ assertEquals(3L, records.get(2).getContainerID());
+
+ records
+ = new ArrayList<>(secondBatch.getContainers());
+ assertEquals(3, records.size());
+ assertEquals(4L, records.get(0).getContainerID());
+ assertEquals(5L, records.get(1).getContainerID());
+ assertEquals(6L, records.get(2).getContainerID());
+ }
+
+ @Test
public void testGetReplicaHistoryForContainer() {
// Add container history for id 1
containerSchemaManager.upsertContainerHistory(1L, "host1", 1L);
@@ -469,4 +640,51 @@ public class TestContainerEndpoint {
}
});
}
+
+ private void createUnhealthyRecords(int missing, int overRep, int underRep,
+ int misRep) {
+ int cid = 0;
+ for (int i=0; i<missing; i++) {
+ createUnhealthyRecord(++cid,
+ UnHealthyContainerStates.MISSING.toString(), 3, 0, 3, null);
+ }
+ for (int i=0; i<overRep; i++) {
+ createUnhealthyRecord(++cid,
+ UnHealthyContainerStates.OVER_REPLICATED.toString(),
+ 3, 5, -2, null);
+ }
+ for (int i=0; i<underRep; i++) {
+ createUnhealthyRecord(++cid,
+ UnHealthyContainerStates.UNDER_REPLICATED.toString(),
+ 3, 1, 2, null);
+ }
+ for (int i=0; i<misRep; i++) {
+ createUnhealthyRecord(++cid,
+ UnHealthyContainerStates.MIS_REPLICATED.toString(),
+ 2, 1, 1, "some reason");
+ }
+ }
+
+ private void createUnhealthyRecord(int id, String state, int expected,
+ int actual, int delta, String reason) {
+ long cID = Integer.toUnsignedLong(id);
+ UnhealthyContainers missing = new UnhealthyContainers();
+ missing.setContainerId(cID);
+ missing.setContainerState(state);
+ missing.setInStateSince(12345L);
+ missing.setActualReplicaCount(actual);
+ missing.setExpectedReplicaCount(expected);
+ missing.setReplicaDelta(delta);
+ missing.setReason(reason);
+
+ ArrayList<UnhealthyContainers> missingList =
+ new ArrayList<UnhealthyContainers>();
+ missingList.add(missing);
+ containerSchemaManager.insertUnhealthyContainerRecords(missingList);
+
+ containerSchemaManager.upsertContainerHistory(cID, "host1", 1L);
+ containerSchemaManager.upsertContainerHistory(cID, "host2", 2L);
+ containerSchemaManager.upsertContainerHistory(cID, "host3", 3L);
+ containerSchemaManager.upsertContainerHistory(cID, "host4", 4L);
+ }
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org