You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@ozone.apache.org by GitBox <gi...@apache.org> on 2020/05/29 19:42:14 UTC
[GitHub] [hadoop-ozone] vivekratnavel commented on a change in pull request #994: HDDS-3082. Refactor recon missing containers task to detect under, over and mis-replicated containers.
vivekratnavel commented on a change in pull request #994:
URL: https://github.com/apache/hadoop-ozone/pull/994#discussion_r432689814
##########
File path: hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/fsck/ContainerHealthTask.java
##########
@@ -0,0 +1,345 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.recon.fsck;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerManager;
+import org.apache.hadoop.hdds.scm.container.ContainerNotFoundException;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.ozone.recon.persistence.ContainerSchemaManager;
+import org.apache.hadoop.ozone.recon.scm.ReconScmTask;
+import org.apache.hadoop.ozone.recon.tasks.ReconTaskConfig;
+import org.apache.hadoop.util.Time;
+import org.hadoop.ozone.recon.schema.ContainerSchemaDefinition.UnHealthyContainerStates;
+import org.hadoop.ozone.recon.schema.tables.daos.ReconTaskStatusDao;
+import org.hadoop.ozone.recon.schema.tables.pojos.UnhealthyContainers;
+import org.hadoop.ozone.recon.schema.tables.records.UnhealthyContainersRecord;
+import org.jooq.Cursor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Class that scans the list of containers and keeps track of containers with
+ * no replicas in a SQL table.
+ */
+public class ContainerHealthTask extends ReconScmTask {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ContainerHealthTask.class);
+
+ private ContainerManager containerManager;
+ private ContainerSchemaManager containerSchemaManager;
+ private PlacementPolicy placementPolicy;
+ private final long interval;
+ private Set<ContainerInfo> processedContainers = new HashSet<>();
+
+ public ContainerHealthTask(
+ ContainerManager containerManager,
+ ReconTaskStatusDao reconTaskStatusDao,
+ ContainerSchemaManager containerSchemaManager,
+ PlacementPolicy placementPolicy,
+ ReconTaskConfig reconTaskConfig) {
+ super(reconTaskStatusDao);
+ this.containerSchemaManager = containerSchemaManager;
+ this.placementPolicy = placementPolicy;
+ this.containerManager = containerManager;
+ this.interval = TimeUnit.SECONDS.toMillis(
+ reconTaskConfig.getMissingContainerTaskInterval());
+ }
+
+ public synchronized void run() {
+ try {
+ while (canRun()) {
+ long start = Time.monotonicNow();
+ long currentTime = System.currentTimeMillis();
+ long existingCount = processExistingDBRecords(currentTime);
+ LOG.info("Missing Container task thread took {} milliseconds to" +
+ " process {} existing database records.",
+ Time.monotonicNow() - start, existingCount);
+ start = Time.monotonicNow();
+ final List<ContainerInfo> containers = containerManager.getContainers();
+ containers.forEach(container ->
+ processContainer(container, currentTime));
+ recordSingleRunCompletion();
+ LOG.info("Missing Container task Thread took {} milliseconds for" +
+ " processing {} containers.", Time.monotonicNow() - start,
+ containers.size());
+ processedContainers.clear();
+ wait(interval);
+ }
+ } catch (Throwable t) {
+ LOG.error("Exception in Missing Container task Thread.", t);
+ }
+ }
+
+ private ContainerHealthStatus setCurrentContainer(long recordId)
+ throws ContainerNotFoundException {
+ ContainerInfo container =
+ containerManager.getContainer(new ContainerID(recordId));
+ Set<ContainerReplica> replicas =
+ containerManager.getContainerReplicas(container.containerID());
+ return new ContainerHealthStatus(container, replicas, placementPolicy);
+ }
+
+ private void completeProcessingContainer(ContainerHealthStatus container,
+ Set<String> existingRecords, long currentTime) {
+ if (container != null) {
+ containerSchemaManager.insertUnhealthyContainerRecords(
+ ContainerHealthRecords.generateUnhealthyRecords(
+ container, existingRecords, currentTime));
+ processedContainers.add(container.getContainer());
+ }
+ }
+
+ private long processExistingDBRecords(long currentTime) {
+ long recordCount = 0;
+ try (Cursor<UnhealthyContainersRecord> cursor =
+ containerSchemaManager.getAllUnhealthyRecordsCursor()) {
+ ContainerHealthStatus currentContainer = null;
+ Set<String> existingRecords = new HashSet<>();
+ while(cursor.hasNext()) {
+ recordCount++;
+ UnhealthyContainersRecord rec = cursor.fetchNext();
+ try {
+ if (currentContainer == null
+ || currentContainer.getContainerID() != rec.getContainerId()) {
+ completeProcessingContainer(
+ currentContainer, existingRecords, currentTime);
+ existingRecords.clear();
+ currentContainer = setCurrentContainer(rec.getContainerId());
+ }
+ if (ContainerHealthRecords
+ .retainOrUpdateRecord(currentContainer, rec)) {
+ existingRecords.add(rec.getContainerState());
+ if (rec.changed()) {
+ rec.update();
+ }
+ } else {
+ rec.delete();
+ }
+ } catch (ContainerNotFoundException cnf) {
+ rec.delete();
+ currentContainer = null;
+ }
+ }
+ // Remember to finish processing the last container
+ completeProcessingContainer(
+ currentContainer, existingRecords, currentTime);
+ }
+ return recordCount;
+ }
+
+ private void processContainer(ContainerInfo container, long currentTime) {
+ try {
+ if (processedContainers.contains(container)) {
+ // Was already handled when processing the existing DB records, so skip
+ return;
+ }
+ Set<ContainerReplica> containerReplicas =
+ containerManager.getContainerReplicas(container.containerID());
+ ContainerHealthStatus h = new ContainerHealthStatus(
+ container, containerReplicas, placementPolicy);
+ if (h.isHealthy()) {
+ return;
+ }
+ containerSchemaManager.insertUnhealthyContainerRecords(
+ ContainerHealthRecords.generateUnhealthyRecords(h, currentTime));
+ } catch (ContainerNotFoundException e) {
+ LOG.error("Container not found while finding missing containers", e);
Review comment:
```suggestion
LOG.error("Container not found while processing container in Container Health task", e);
```
##########
File path: hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/fsck/ContainerHealthTask.java
##########
@@ -0,0 +1,345 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.recon.fsck;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerManager;
+import org.apache.hadoop.hdds.scm.container.ContainerNotFoundException;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.ozone.recon.persistence.ContainerSchemaManager;
+import org.apache.hadoop.ozone.recon.scm.ReconScmTask;
+import org.apache.hadoop.ozone.recon.tasks.ReconTaskConfig;
+import org.apache.hadoop.util.Time;
+import org.hadoop.ozone.recon.schema.ContainerSchemaDefinition.UnHealthyContainerStates;
+import org.hadoop.ozone.recon.schema.tables.daos.ReconTaskStatusDao;
+import org.hadoop.ozone.recon.schema.tables.pojos.UnhealthyContainers;
+import org.hadoop.ozone.recon.schema.tables.records.UnhealthyContainersRecord;
+import org.jooq.Cursor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Class that scans the list of containers and keeps track of containers with
+ * no replicas in a SQL table.
+ */
+public class ContainerHealthTask extends ReconScmTask {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ContainerHealthTask.class);
+
+ private ContainerManager containerManager;
+ private ContainerSchemaManager containerSchemaManager;
+ private PlacementPolicy placementPolicy;
+ private final long interval;
+ private Set<ContainerInfo> processedContainers = new HashSet<>();
+
+ public ContainerHealthTask(
+ ContainerManager containerManager,
+ ReconTaskStatusDao reconTaskStatusDao,
+ ContainerSchemaManager containerSchemaManager,
+ PlacementPolicy placementPolicy,
+ ReconTaskConfig reconTaskConfig) {
+ super(reconTaskStatusDao);
+ this.containerSchemaManager = containerSchemaManager;
+ this.placementPolicy = placementPolicy;
+ this.containerManager = containerManager;
+ this.interval = TimeUnit.SECONDS.toMillis(
+ reconTaskConfig.getMissingContainerTaskInterval());
+ }
+
+ public synchronized void run() {
+ try {
+ while (canRun()) {
+ long start = Time.monotonicNow();
+ long currentTime = System.currentTimeMillis();
+ long existingCount = processExistingDBRecords(currentTime);
+ LOG.info("Missing Container task thread took {} milliseconds to" +
Review comment:
```suggestion
LOG.info("Container Health task thread took {} milliseconds to" +
```
##########
File path: hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/fsck/ContainerHealthTask.java
##########
@@ -0,0 +1,345 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.recon.fsck;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerManager;
+import org.apache.hadoop.hdds.scm.container.ContainerNotFoundException;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.ozone.recon.persistence.ContainerSchemaManager;
+import org.apache.hadoop.ozone.recon.scm.ReconScmTask;
+import org.apache.hadoop.ozone.recon.tasks.ReconTaskConfig;
+import org.apache.hadoop.util.Time;
+import org.hadoop.ozone.recon.schema.ContainerSchemaDefinition.UnHealthyContainerStates;
+import org.hadoop.ozone.recon.schema.tables.daos.ReconTaskStatusDao;
+import org.hadoop.ozone.recon.schema.tables.pojos.UnhealthyContainers;
+import org.hadoop.ozone.recon.schema.tables.records.UnhealthyContainersRecord;
+import org.jooq.Cursor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Class that scans the list of containers and keeps track of containers with
+ * no replicas in a SQL table.
+ */
+public class ContainerHealthTask extends ReconScmTask {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ContainerHealthTask.class);
+
+ private ContainerManager containerManager;
+ private ContainerSchemaManager containerSchemaManager;
+ private PlacementPolicy placementPolicy;
+ private final long interval;
+ private Set<ContainerInfo> processedContainers = new HashSet<>();
+
+ public ContainerHealthTask(
+ ContainerManager containerManager,
+ ReconTaskStatusDao reconTaskStatusDao,
+ ContainerSchemaManager containerSchemaManager,
+ PlacementPolicy placementPolicy,
+ ReconTaskConfig reconTaskConfig) {
+ super(reconTaskStatusDao);
+ this.containerSchemaManager = containerSchemaManager;
+ this.placementPolicy = placementPolicy;
+ this.containerManager = containerManager;
+ this.interval = TimeUnit.SECONDS.toMillis(
+ reconTaskConfig.getMissingContainerTaskInterval());
+ }
+
+ public synchronized void run() {
+ try {
+ while (canRun()) {
+ long start = Time.monotonicNow();
+ long currentTime = System.currentTimeMillis();
+ long existingCount = processExistingDBRecords(currentTime);
+ LOG.info("Missing Container task thread took {} milliseconds to" +
+ " process {} existing database records.",
+ Time.monotonicNow() - start, existingCount);
+ start = Time.monotonicNow();
+ final List<ContainerInfo> containers = containerManager.getContainers();
+ containers.forEach(container ->
+ processContainer(container, currentTime));
+ recordSingleRunCompletion();
+ LOG.info("Missing Container task Thread took {} milliseconds for" +
Review comment:
```suggestion
LOG.info("Container Health task thread took {} milliseconds for" +
```
##########
File path: hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/fsck/ContainerHealthTask.java
##########
@@ -0,0 +1,345 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.recon.fsck;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerManager;
+import org.apache.hadoop.hdds.scm.container.ContainerNotFoundException;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.ozone.recon.persistence.ContainerSchemaManager;
+import org.apache.hadoop.ozone.recon.scm.ReconScmTask;
+import org.apache.hadoop.ozone.recon.tasks.ReconTaskConfig;
+import org.apache.hadoop.util.Time;
+import org.hadoop.ozone.recon.schema.ContainerSchemaDefinition.UnHealthyContainerStates;
+import org.hadoop.ozone.recon.schema.tables.daos.ReconTaskStatusDao;
+import org.hadoop.ozone.recon.schema.tables.pojos.UnhealthyContainers;
+import org.hadoop.ozone.recon.schema.tables.records.UnhealthyContainersRecord;
+import org.jooq.Cursor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Class that scans the list of containers and keeps track of containers with
+ * no replicas in a SQL table.
+ */
+public class ContainerHealthTask extends ReconScmTask {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ContainerHealthTask.class);
+
+ private ContainerManager containerManager;
+ private ContainerSchemaManager containerSchemaManager;
+ private PlacementPolicy placementPolicy;
+ private final long interval;
+ private Set<ContainerInfo> processedContainers = new HashSet<>();
+
+ public ContainerHealthTask(
+ ContainerManager containerManager,
+ ReconTaskStatusDao reconTaskStatusDao,
+ ContainerSchemaManager containerSchemaManager,
+ PlacementPolicy placementPolicy,
+ ReconTaskConfig reconTaskConfig) {
+ super(reconTaskStatusDao);
+ this.containerSchemaManager = containerSchemaManager;
+ this.placementPolicy = placementPolicy;
+ this.containerManager = containerManager;
+ this.interval = TimeUnit.SECONDS.toMillis(
+ reconTaskConfig.getMissingContainerTaskInterval());
+ }
+
+ public synchronized void run() {
+ try {
+ while (canRun()) {
+ long start = Time.monotonicNow();
+ long currentTime = System.currentTimeMillis();
+ long existingCount = processExistingDBRecords(currentTime);
+ LOG.info("Missing Container task thread took {} milliseconds to" +
+ " process {} existing database records.",
+ Time.monotonicNow() - start, existingCount);
+ start = Time.monotonicNow();
+ final List<ContainerInfo> containers = containerManager.getContainers();
+ containers.forEach(container ->
+ processContainer(container, currentTime));
+ recordSingleRunCompletion();
+ LOG.info("Missing Container task Thread took {} milliseconds for" +
+ " processing {} containers.", Time.monotonicNow() - start,
+ containers.size());
+ processedContainers.clear();
+ wait(interval);
+ }
+ } catch (Throwable t) {
+ LOG.error("Exception in Missing Container task Thread.", t);
+ }
+ }
+
+ private ContainerHealthStatus setCurrentContainer(long recordId)
+ throws ContainerNotFoundException {
+ ContainerInfo container =
+ containerManager.getContainer(new ContainerID(recordId));
+ Set<ContainerReplica> replicas =
+ containerManager.getContainerReplicas(container.containerID());
+ return new ContainerHealthStatus(container, replicas, placementPolicy);
+ }
+
+ private void completeProcessingContainer(ContainerHealthStatus container,
+ Set<String> existingRecords, long currentTime) {
+ if (container != null) {
+ containerSchemaManager.insertUnhealthyContainerRecords(
+ ContainerHealthRecords.generateUnhealthyRecords(
+ container, existingRecords, currentTime));
+ processedContainers.add(container.getContainer());
+ }
+ }
+
+ private long processExistingDBRecords(long currentTime) {
+ long recordCount = 0;
+ try (Cursor<UnhealthyContainersRecord> cursor =
+ containerSchemaManager.getAllUnhealthyRecordsCursor()) {
+ ContainerHealthStatus currentContainer = null;
+ Set<String> existingRecords = new HashSet<>();
+ while(cursor.hasNext()) {
+ recordCount++;
+ UnhealthyContainersRecord rec = cursor.fetchNext();
+ try {
+ if (currentContainer == null
+ || currentContainer.getContainerID() != rec.getContainerId()) {
+ completeProcessingContainer(
+ currentContainer, existingRecords, currentTime);
+ existingRecords.clear();
+ currentContainer = setCurrentContainer(rec.getContainerId());
+ }
+ if (ContainerHealthRecords
+ .retainOrUpdateRecord(currentContainer, rec)) {
+ existingRecords.add(rec.getContainerState());
+ if (rec.changed()) {
+ rec.update();
+ }
+ } else {
+ rec.delete();
+ }
+ } catch (ContainerNotFoundException cnf) {
+ rec.delete();
+ currentContainer = null;
+ }
+ }
+ // Remember to finish processing the last container
+ completeProcessingContainer(
+ currentContainer, existingRecords, currentTime);
+ }
+ return recordCount;
+ }
+
+ private void processContainer(ContainerInfo container, long currentTime) {
+ try {
+ if (processedContainers.contains(container)) {
+ // Was already handled when processing the existing DB records, so skip
+ return;
+ }
+ Set<ContainerReplica> containerReplicas =
+ containerManager.getContainerReplicas(container.containerID());
+ ContainerHealthStatus h = new ContainerHealthStatus(
+ container, containerReplicas, placementPolicy);
+ if (h.isHealthy()) {
+ return;
+ }
+ containerSchemaManager.insertUnhealthyContainerRecords(
+ ContainerHealthRecords.generateUnhealthyRecords(h, currentTime));
+ } catch (ContainerNotFoundException e) {
+ LOG.error("Container not found while finding missing containers", e);
+ }
+ }
+
+ /**
+ * Helper methods to generate and update the required database records for
+ * unhealthy containers.
+ */
+ static public class ContainerHealthRecords {
+
+ static public boolean retainOrUpdateRecord(
Review comment:
Can you please add JavaDoc for this method?
##########
File path: hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/fsck/TestContainerHealthStatus.java
##########
@@ -0,0 +1,181 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.recon.fsck;
+
+import org.apache.hadoop.hdds.protocol.MockDatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.placement.algorithms.ContainerPlacementStatusDefault;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mockito;
+import java.util.HashSet;
+import java.util.Set;
+
+import static junit.framework.TestCase.assertEquals;
+import static junit.framework.TestCase.assertTrue;
+import static org.junit.Assert.assertFalse;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * Test to ensure the correct container state is return by a
+ * ContainerHealthyStatus instance.
Review comment:
```suggestion
* Test to ensure the correct container state is returned by a
* ContainerHealthStatus instance.
```
##########
File path: hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/fsck/ContainerHealthTask.java
##########
@@ -0,0 +1,345 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.recon.fsck;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerManager;
+import org.apache.hadoop.hdds.scm.container.ContainerNotFoundException;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.ozone.recon.persistence.ContainerSchemaManager;
+import org.apache.hadoop.ozone.recon.scm.ReconScmTask;
+import org.apache.hadoop.ozone.recon.tasks.ReconTaskConfig;
+import org.apache.hadoop.util.Time;
+import org.hadoop.ozone.recon.schema.ContainerSchemaDefinition.UnHealthyContainerStates;
+import org.hadoop.ozone.recon.schema.tables.daos.ReconTaskStatusDao;
+import org.hadoop.ozone.recon.schema.tables.pojos.UnhealthyContainers;
+import org.hadoop.ozone.recon.schema.tables.records.UnhealthyContainersRecord;
+import org.jooq.Cursor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Class that scans the list of containers and keeps track of containers with
+ * no replicas in a SQL table.
+ */
+public class ContainerHealthTask extends ReconScmTask {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ContainerHealthTask.class);
+
+ private ContainerManager containerManager;
+ private ContainerSchemaManager containerSchemaManager;
+ private PlacementPolicy placementPolicy;
+ private final long interval;
+ private Set<ContainerInfo> processedContainers = new HashSet<>();
+
+ public ContainerHealthTask(
+ ContainerManager containerManager,
+ ReconTaskStatusDao reconTaskStatusDao,
+ ContainerSchemaManager containerSchemaManager,
+ PlacementPolicy placementPolicy,
+ ReconTaskConfig reconTaskConfig) {
+ super(reconTaskStatusDao);
+ this.containerSchemaManager = containerSchemaManager;
+ this.placementPolicy = placementPolicy;
+ this.containerManager = containerManager;
+ this.interval = TimeUnit.SECONDS.toMillis(
+ reconTaskConfig.getMissingContainerTaskInterval());
+ }
+
+ public synchronized void run() {
+ try {
+ while (canRun()) {
+ long start = Time.monotonicNow();
+ long currentTime = System.currentTimeMillis();
+ long existingCount = processExistingDBRecords(currentTime);
+ LOG.info("Missing Container task thread took {} milliseconds to" +
+ " process {} existing database records.",
+ Time.monotonicNow() - start, existingCount);
+ start = Time.monotonicNow();
+ final List<ContainerInfo> containers = containerManager.getContainers();
+ containers.forEach(container ->
+ processContainer(container, currentTime));
+ recordSingleRunCompletion();
+ LOG.info("Missing Container task Thread took {} milliseconds for" +
+ " processing {} containers.", Time.monotonicNow() - start,
+ containers.size());
+ processedContainers.clear();
+ wait(interval);
+ }
+ } catch (Throwable t) {
+ LOG.error("Exception in Missing Container task Thread.", t);
+ }
+ }
+
+ private ContainerHealthStatus setCurrentContainer(long recordId)
+ throws ContainerNotFoundException {
+ ContainerInfo container =
+ containerManager.getContainer(new ContainerID(recordId));
+ Set<ContainerReplica> replicas =
+ containerManager.getContainerReplicas(container.containerID());
+ return new ContainerHealthStatus(container, replicas, placementPolicy);
+ }
+
+ private void completeProcessingContainer(ContainerHealthStatus container,
+ Set<String> existingRecords, long currentTime) {
+ if (container != null) {
+ containerSchemaManager.insertUnhealthyContainerRecords(
+ ContainerHealthRecords.generateUnhealthyRecords(
+ container, existingRecords, currentTime));
+ processedContainers.add(container.getContainer());
+ }
+ }
+
+ private long processExistingDBRecords(long currentTime) {
+ long recordCount = 0;
+ try (Cursor<UnhealthyContainersRecord> cursor =
+ containerSchemaManager.getAllUnhealthyRecordsCursor()) {
+ ContainerHealthStatus currentContainer = null;
+ Set<String> existingRecords = new HashSet<>();
+ while(cursor.hasNext()) {
+ recordCount++;
+ UnhealthyContainersRecord rec = cursor.fetchNext();
+ try {
+ if (currentContainer == null
+ || currentContainer.getContainerID() != rec.getContainerId()) {
+ completeProcessingContainer(
+ currentContainer, existingRecords, currentTime);
+ existingRecords.clear();
+ currentContainer = setCurrentContainer(rec.getContainerId());
+ }
+ if (ContainerHealthRecords
+ .retainOrUpdateRecord(currentContainer, rec)) {
+ existingRecords.add(rec.getContainerState());
+ if (rec.changed()) {
+ rec.update();
+ }
+ } else {
+ rec.delete();
+ }
+ } catch (ContainerNotFoundException cnf) {
+ rec.delete();
+ currentContainer = null;
+ }
+ }
+ // Remember to finish processing the last container
+ completeProcessingContainer(
+ currentContainer, existingRecords, currentTime);
+ }
+ return recordCount;
+ }
+
+ private void processContainer(ContainerInfo container, long currentTime) {
+ try {
+ if (processedContainers.contains(container)) {
+ // Was already handled when processing the existing DB records, so skip
+ return;
+ }
+ Set<ContainerReplica> containerReplicas =
+ containerManager.getContainerReplicas(container.containerID());
+ ContainerHealthStatus h = new ContainerHealthStatus(
+ container, containerReplicas, placementPolicy);
+ if (h.isHealthy()) {
+ return;
+ }
+ containerSchemaManager.insertUnhealthyContainerRecords(
+ ContainerHealthRecords.generateUnhealthyRecords(h, currentTime));
+ } catch (ContainerNotFoundException e) {
+ LOG.error("Container not found while finding missing containers", e);
+ }
+ }
+
+ /**
+ * Helper methods to generate and update the required database records for
+ * unhealthy containers.
+ */
+ static public class ContainerHealthRecords {
+
+ static public boolean retainOrUpdateRecord(
+ ContainerHealthStatus container, UnhealthyContainersRecord rec) {
+ boolean returnValue = false;
+ switch(UnHealthyContainerStates.valueOf(rec.getContainerState())) {
+ case MISSING:
+ returnValue = container.isMissing();
+ break;
+ case MIS_REPLICATED:
+ returnValue = keepMisReplicatedRecord(container, rec);
+ break;
+ case UNDER_REPLICATED:
+ returnValue = keepUnderReplicatedRecord(container, rec);
+ break;
+ case OVER_REPLICATED:
+ returnValue = keepOverReplicatedRecord(container, rec);
+ break;
+ default:
+ returnValue = false;
+ }
+ return returnValue;
+ }
+
+ static public List<UnhealthyContainers> generateUnhealthyRecords(
+ ContainerHealthStatus container, long time) {
+ return generateUnhealthyRecords(container, new HashSet<>(), time);
+ }
+
+ /**
+ * Check the status of the container and generate any database records that
+ * need to be recorded. This method also considers the records seen by the
+ * method retainOrUpdateRecord. If a record has been seen by that method
+ * then it will not be emitted here. Therefore this method returns only the
+ * missing records which have not been seen already.
+ * @return List of UnhealthyContainer records to be stored in the DB
+ */
+ static public List<UnhealthyContainers> generateUnhealthyRecords(
+ ContainerHealthStatus container, Set<String> recordForStateExists,
+ long time) {
+ List<UnhealthyContainers> records = new ArrayList<>();
+ if (container.isHealthy()) {
+ return records;
+ }
+
+ if (container.isMissing()
+ && !recordForStateExists.contains(
+ UnHealthyContainerStates.MISSING.toString())) {
+ records.add(
+ recordForState(container, UnHealthyContainerStates.MISSING, time));
+ }
+
+ if (container.isUnderReplicated()
+ && !recordForStateExists.contains(
+ UnHealthyContainerStates.UNDER_REPLICATED.toString())) {
+ records.add(recordForState(
+ container, UnHealthyContainerStates.UNDER_REPLICATED, time));
+ }
+
+ if (container.isOverReplicated()
+ && !recordForStateExists.contains(
+ UnHealthyContainerStates.OVER_REPLICATED.toString())) {
+ records.add(recordForState(
+ container, UnHealthyContainerStates.OVER_REPLICATED, time));
+ }
+
+ if (container.isMisReplicated()
+ && !recordForStateExists.contains(
+ UnHealthyContainerStates.MIS_REPLICATED.toString())) {
+ records.add(recordForState(
+ container, UnHealthyContainerStates.MIS_REPLICATED, time));
+ }
+ return records;
+ }
+
+ static private UnhealthyContainers recordForState(
+ ContainerHealthStatus container, UnHealthyContainerStates state,
+ long time) {
+ UnhealthyContainers rec = new UnhealthyContainers();
+ rec.setContainerId(container.getContainerID());
+ if (state == UnHealthyContainerStates.MIS_REPLICATED) {
+ rec.setExpectedReplicaCount(container.expectedPlacementCount());
+ rec.setActualReplicaCount(container.actualPlacementCount());
+ rec.setReplicaDelta(container.misReplicatedDelta());
+ rec.setReason(container.misReplicatedReason());
+ } else {
+ rec.setExpectedReplicaCount(container.getReplicationFactor());
+ rec.setActualReplicaCount(container.getReplicaCount());
+ rec.setReplicaDelta(container.replicaDelta());
+ }
+ rec.setContainerState(state.toString());
+ rec.setInStateSince(time);
+ return rec;
+ }
+
+ static private boolean keepOverReplicatedRecord(
+ ContainerHealthStatus container, UnhealthyContainersRecord rec) {
+ if (container.isOverReplicated()) {
+ updateExpectedReplicaCount(rec, container.getReplicationFactor());
+ updateActualReplicaCount(rec, container.getReplicaCount());
+ updateReplicaDelta(rec, container.replicaDelta());
+ return true;
+ }
+ return false;
+ }
+
+ static private boolean keepUnderReplicatedRecord(
+ ContainerHealthStatus container, UnhealthyContainersRecord rec) {
+ if (container.isUnderReplicated()) {
+ updateExpectedReplicaCount(rec, container.getReplicationFactor());
+ updateActualReplicaCount(rec, container.getReplicaCount());
+ updateReplicaDelta(rec, container.replicaDelta());
+ return true;
+ }
+ return false;
+ }
+
+ static private boolean keepMisReplicatedRecord(
+ ContainerHealthStatus container, UnhealthyContainersRecord rec) {
+ if (container.isMisReplicated()) {
+ updateExpectedReplicaCount(rec, container.expectedPlacementCount());
+ updateActualReplicaCount(rec, container.actualPlacementCount());
+ updateReplicaDelta(rec, container.misReplicatedDelta());
+ updateReason(rec, container.misReplicatedReason());
+ return true;
+ }
+ return false;
+ }
+
+ /**
+ * With a Jooq record, if you update any field in the record, the record
+ * is marked as changed, even if you updated it to the same value as it is
+ * already set to. We only need to run a DB update statement if the record
+ * has really changed. The methods below ensure we do not update the Jooq
+ * record unless the values have changed and hence save a DB execution
+ * when
Review comment:
Please fix the dangling statement
##########
File path: hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/fsck/TestContainerHealthTask.java
##########
@@ -0,0 +1,280 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.recon.fsck;
+
+import static junit.framework.TestCase.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.UUID;
+
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.MockDatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State;
+import org.apache.hadoop.hdds.scm.ContainerPlacementStatus;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerManager;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.placement.algorithms.ContainerPlacementStatusDefault;
+import org.apache.hadoop.ozone.recon.persistence.ContainerSchemaManager;
+import org.apache.hadoop.ozone.recon.scm.ReconStorageContainerManagerFacade;
+import org.apache.hadoop.ozone.recon.tasks.ReconTaskConfig;
+import org.apache.hadoop.test.LambdaTestUtils;
+import org.hadoop.ozone.recon.schema.ContainerSchemaDefinition;
+import org.hadoop.ozone.recon.schema.tables.daos.ContainerHistoryDao;
+import org.apache.hadoop.ozone.recon.persistence.AbstractReconSqlDBTest;
+import org.hadoop.ozone.recon.schema.tables.daos.ReconTaskStatusDao;
+import org.hadoop.ozone.recon.schema.tables.daos.UnhealthyContainersDao;
+import org.hadoop.ozone.recon.schema.tables.pojos.ReconTaskStatus;
+import org.hadoop.ozone.recon.schema.tables.pojos.UnhealthyContainers;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Class to test single run of Missing Container Task.
Review comment:
```suggestion
* Class to test single run of Container Health Task.
```
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org