You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by so...@apache.org on 2022/04/26 15:57:32 UTC
[ozone] branch master updated: HDDS-6635. ReportSubCommand should provide a JSON output option (#3344)
This is an automated email from the ASF dual-hosted git repository.
sodonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new b4e8e7ac8d HDDS-6635. ReportSubCommand should provide a JSON output option (#3344)
b4e8e7ac8d is described below
commit b4e8e7ac8d382c88c44bbd5d42a9648c2e9feb70
Author: Stephen O'Donnell <st...@gmail.com>
AuthorDate: Tue Apr 26 16:57:27 2022 +0100
HDDS-6635. ReportSubCommand should provide a JSON output option (#3344)
---
hadoop-hdds/common/pom.xml | 4 ++
.../scm/container/ReplicationManagerReport.java | 30 +++++++++++-
.../org/apache/hadoop/hdds/server/JsonUtils.java | 0
.../container/TestReplicationManagerReport.java | 54 ++++++++++++++++++++++
.../hdds/scm/cli/container/ReportSubcommand.java | 12 +++++
.../scm/cli/container/TestReportSubCommand.java | 24 ++++++++++
6 files changed, 123 insertions(+), 1 deletion(-)
diff --git a/hadoop-hdds/common/pom.xml b/hadoop-hdds/common/pom.xml
index 58c07881f9..cf65d8b1d5 100644
--- a/hadoop-hdds/common/pom.xml
+++ b/hadoop-hdds/common/pom.xml
@@ -70,6 +70,10 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
</dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.datatype</groupId>
+ <artifactId>jackson-datatype-jsr310</artifactId>
+ </dependency>
<dependency>
<groupId>org.apache.ozone</groupId>
<artifactId>hdds-config</artifactId>
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManagerReport.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManagerReport.java
index 2f2a7bf3e5..b47b2d6029 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManagerReport.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManagerReport.java
@@ -107,7 +107,7 @@ public class ReplicationManagerReport {
for (HddsProtos.KeyContainerIDList sample : proto.getStatSampleList()) {
report.setSample(sample.getKey(), sample.getContainerList()
.stream()
- .map(c -> ContainerID.getFromProtobuf(c))
+ .map(ContainerID::getFromProtobuf)
.collect(Collectors.toList()));
}
return report;
@@ -146,6 +146,34 @@ public class ReplicationManagerReport {
return reportTimeStamp;
}
+ /**
+ * Return a map of all stats and their value as a long.
+ * @return
+ */
+ public Map<String, Long> getStats() {
+ Map<String, Long> result = new HashMap<>();
+ for (Map.Entry<String, LongAdder> e : stats.entrySet()) {
+ result.put(e.getKey(), e.getValue().longValue());
+ }
+ return result;
+ }
+
+ /**
+ * Return a map of all samples, with the stat as the key and the samples
+ * for the stat as a List of Long.
+ * @return
+ */
+ public Map<String, List<Long>> getSamples() {
+ Map<String, List<Long>> result = new HashMap<>();
+ for (Map.Entry<String, List<ContainerID>> e : containerSample.entrySet()) {
+ result.put(e.getKey(),
+ e.getValue().stream()
+ .map(c -> c.getId())
+ .collect(Collectors.toList()));
+ }
+ return result;
+ }
+
/**
* Get the stat for the given LifeCycleState. If there is no stat available
* for that stat -1 is returned.
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/JsonUtils.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/server/JsonUtils.java
similarity index 100%
rename from hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/JsonUtils.java
rename to hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/server/JsonUtils.java
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/container/TestReplicationManagerReport.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/container/TestReplicationManagerReport.java
index a05f9abb2b..eee266f4c0 100644
--- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/container/TestReplicationManagerReport.java
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/container/TestReplicationManagerReport.java
@@ -17,16 +17,22 @@
*/
package org.apache.hadoop.hdds.scm.container;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.server.JsonUtils;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;
import java.util.concurrent.ThreadLocalRandom;
+import static com.fasterxml.jackson.databind.node.JsonNodeType.ARRAY;
+
/**
* Tests for the ReplicationManagerReport class.
*/
@@ -64,6 +70,54 @@ public class TestReplicationManagerReport {
report.getStat(HddsProtos.LifeCycleState.QUASI_CLOSED));
}
+
+ @Test
+ public void testJsonOutput() throws IOException {
+ report.increment(HddsProtos.LifeCycleState.OPEN);
+ report.increment(HddsProtos.LifeCycleState.CLOSED);
+ report.increment(HddsProtos.LifeCycleState.CLOSED);
+
+ report.incrementAndSample(
+ ReplicationManagerReport.HealthState.UNDER_REPLICATED,
+ new ContainerID(1));
+ report.incrementAndSample(
+ ReplicationManagerReport.HealthState.UNDER_REPLICATED,
+ new ContainerID(2));
+ report.incrementAndSample(
+ ReplicationManagerReport.HealthState.OVER_REPLICATED,
+ new ContainerID(3));
+ report.setComplete();
+
+ String jsonString = JsonUtils.toJsonStringWithDefaultPrettyPrinter(report);
+
+ ObjectMapper mapper = new ObjectMapper();
+ JsonNode json = mapper.readTree(jsonString);
+
+ Assert.assertTrue(json.get("reportTimeStamp").longValue() > 0);
+ JsonNode stats = json.get("stats");
+ Assert.assertEquals(1, stats.get("OPEN").longValue());
+ Assert.assertEquals(0, stats.get("CLOSING").longValue());
+ Assert.assertEquals(0, stats.get("QUASI_CLOSED").longValue());
+ Assert.assertEquals(2, stats.get("CLOSED").longValue());
+ Assert.assertEquals(0, stats.get("DELETING").longValue());
+ Assert.assertEquals(0, stats.get("DELETED").longValue());
+
+ Assert.assertEquals(2, stats.get("UNDER_REPLICATED").longValue());
+ Assert.assertEquals(1, stats.get("OVER_REPLICATED").longValue());
+ Assert.assertEquals(0, stats.get("MIS_REPLICATED").longValue());
+ Assert.assertEquals(0, stats.get("MISSING").longValue());
+ Assert.assertEquals(0, stats.get("UNHEALTHY").longValue());
+ Assert.assertEquals(0, stats.get("EMPTY").longValue());
+ Assert.assertEquals(0, stats.get("OPEN_UNHEALTHY").longValue());
+ Assert.assertEquals(0, stats.get("QUASI_CLOSED_STUCK").longValue());
+
+ JsonNode samples = json.get("samples");
+ Assert.assertEquals(ARRAY, samples.get("UNDER_REPLICATED").getNodeType());
+ Assert.assertEquals(1, samples.get("UNDER_REPLICATED").get(0).longValue());
+ Assert.assertEquals(2, samples.get("UNDER_REPLICATED").get(1).longValue());
+ Assert.assertEquals(3, samples.get("OVER_REPLICATED").get(0).longValue());
+ }
+
@Test
public void testContainerIDsCanBeSampled() {
report.incrementAndSample(
diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ReportSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ReportSubcommand.java
index 89b1a11cba..554316c2e9 100644
--- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ReportSubcommand.java
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ReportSubcommand.java
@@ -23,6 +23,7 @@ import org.apache.hadoop.hdds.scm.cli.ScmSubcommand;
import org.apache.hadoop.hdds.scm.client.ScmClient;
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.container.ReplicationManagerReport;
+import org.apache.hadoop.hdds.server.JsonUtils;
import picocli.CommandLine;
import java.io.IOException;
@@ -43,9 +44,20 @@ public class ReportSubcommand extends ScmSubcommand {
@CommandLine.Spec
private CommandLine.Model.CommandSpec spec;
+ @CommandLine.Option(names = { "--json" },
+ defaultValue = "false",
+ description = "Format output as JSON")
+ private boolean json;
+
@Override
public void execute(ScmClient scmClient) throws IOException {
ReplicationManagerReport report = scmClient.getReplicationManagerReport();
+
+ if (json) {
+ output(JsonUtils.toJsonStringWithDefaultPrettyPrinter(report));
+ return;
+ }
+
outputHeader(report.getReportTimeStamp());
blankLine();
outputContainerStats(report);
diff --git a/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/container/TestReportSubCommand.java b/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/container/TestReportSubCommand.java
index be0e2c8fb6..df8dd953fb 100644
--- a/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/container/TestReportSubCommand.java
+++ b/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/container/TestReportSubCommand.java
@@ -17,14 +17,18 @@
*/
package org.apache.hadoop.hdds.scm.cli.container;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.client.ScmClient;
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.container.ReplicationManagerReport;
import org.junit.After;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.mockito.Mockito;
+import picocli.CommandLine;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
@@ -87,6 +91,26 @@ public class TestReportSubCommand {
}
}
+ @Test
+ public void testValidJsonOutput() throws IOException {
+ // More complete testing of the Report JSON output is in
+ // TestReplicationManagerReport.
+ ScmClient scmClient = mock(ScmClient.class);
+ Mockito.when(scmClient.getReplicationManagerReport())
+ .thenAnswer(invocation -> new ReplicationManagerReport());
+
+ CommandLine c = new CommandLine(cmd);
+ c.parseArgs("--json");
+ cmd.execute(scmClient);
+
+ ObjectMapper mapper = new ObjectMapper();
+ JsonNode json = mapper.readTree(outContent.toString("UTF-8"));
+
+ Assert.assertTrue(json.get("reportTimeStamp") != null);
+ Assert.assertTrue(json.get("stats") != null);
+ Assert.assertTrue(json.get("samples") != null);
+ }
+
@Test
public void testCorrectValuesAppearInReport() throws IOException {
ScmClient scmClient = mock(ScmClient.class);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org