You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by ad...@apache.org on 2023/03/12 09:36:03 UTC
[ozone] branch master updated: HDDS-8133. Create ozone sh key checksum command (#4375)
This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 3d036f461b HDDS-8133. Create ozone sh key checksum command (#4375)
3d036f461b is described below
commit 3d036f461bb1d06b3fe5249f7b4d90a074e0f339
Author: Stephen O'Donnell <st...@gmail.com>
AuthorDate: Sun Mar 12 09:35:57 2023 +0000
HDDS-8133. Create ozone sh key checksum command (#4375)
---
.../ozone/shell/keys/ChecksumKeyHandler.java | 100 +++++++++++++++++
.../hadoop/ozone/shell/keys/KeyCommands.java | 3 +-
.../ozone/shell/keys/TestChecksumKeyHandler.java | 122 +++++++++++++++++++++
.../hadoop/ozone/shell/keys/package-info.java | 21 ++++
4 files changed, 245 insertions(+), 1 deletion(-)
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/ChecksumKeyHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/ChecksumKeyHandler.java
new file mode 100644
index 0000000000..b8acf783d6
--- /dev/null
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/ChecksumKeyHandler.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.shell.keys;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import org.apache.hadoop.fs.FileChecksum;
+import org.apache.hadoop.hdds.scm.OzoneClientConfig;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientException;
+import org.apache.hadoop.ozone.client.OzoneKeyDetails;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
+import org.apache.hadoop.ozone.shell.OzoneAddress;
+import picocli.CommandLine;
+
+import java.io.IOException;
+
+import static org.apache.hadoop.fs.ozone.OzoneClientUtils.getFileChecksumWithCombineMode;
+
+/**
+ * Class to display checksum information about an existing key.
+ */
+@CommandLine.Command(name = "checksum",
+ description = "returns checksum information about an existing key")
+public class ChecksumKeyHandler extends KeyHandler {
+
+ @CommandLine.Option(
+ names = {"-c", "--combine-mode"},
+ description = "Method of combining the chunk checksums. Valid values are "
+ + "COMPOSITE_CRC and MD5MD5CRC. Defaults to COMPOSITE_CRC.")
+ private OzoneClientConfig.ChecksumCombineMode mode =
+ OzoneClientConfig.ChecksumCombineMode.COMPOSITE_CRC;
+
+ @Override
+ protected void execute(OzoneClient client, OzoneAddress address)
+ throws IOException, OzoneClientException {
+
+ ChecksumInfo checksumInfo = new ChecksumInfo(address, client, mode);
+ printObjectAsJson(checksumInfo);
+ }
+
+ /**
+ * Wrapper to the checksum computer to allow it to be override in tests.
+ */
+ protected FileChecksum getFileChecksum(OzoneVolume vol, OzoneBucket bucket,
+ String keyName, long dataSize, ClientProtocol clientProxy)
+ throws IOException {
+ return getFileChecksumWithCombineMode(vol, bucket, keyName, dataSize, mode,
+ clientProxy);
+ }
+
+ /**
+ * Wrapper class to allow checksum information to be printed as JSON.
+ */
+ @JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY)
+ private class ChecksumInfo {
+
+ private final String volumeName;
+ private final String bucketName;
+ private final String name;
+ private final long dataSize;
+ private final String algorithm;
+ private final String checksum;
+
+ ChecksumInfo(OzoneAddress address, OzoneClient client,
+ OzoneClientConfig.ChecksumCombineMode mode) throws IOException {
+ volumeName = address.getVolumeName();
+ bucketName = address.getBucketName();
+ name = address.getKeyName();
+
+ OzoneVolume vol = client.getObjectStore().getVolume(volumeName);
+ OzoneBucket bucket = vol.getBucket(bucketName);
+ OzoneKeyDetails key = bucket.getKey(name);
+ dataSize = key.getDataSize();
+
+ FileChecksum fileChecksum = getFileChecksum(vol, bucket,
+ name, dataSize, client.getObjectStore().getClientProxy());
+
+ this.algorithm = fileChecksum.getAlgorithmName();
+ this.checksum = javax.xml.bind.DatatypeConverter.printHexBinary(
+ fileChecksum.getBytes());
+ }
+ }
+}
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/KeyCommands.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/KeyCommands.java
index 28af82fab0..726ab8be4a 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/KeyCommands.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/KeyCommands.java
@@ -49,7 +49,8 @@ import picocli.CommandLine.ParentCommand;
AddAclKeyHandler.class,
RemoveAclKeyHandler.class,
SetAclKeyHandler.class,
- GetAclKeyHandler.class
+ GetAclKeyHandler.class,
+ ChecksumKeyHandler.class
},
mixinStandardHelpOptions = true,
versionProvider = HddsVersionProvider.class)
diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/shell/keys/TestChecksumKeyHandler.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/shell/keys/TestChecksumKeyHandler.java
new file mode 100644
index 0000000000..1279adcc8a
--- /dev/null
+++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/shell/keys/TestChecksumKeyHandler.java
@@ -0,0 +1,122 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.shell.keys;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.hadoop.fs.CompositeCrcFileChecksum;
+import org.apache.hadoop.fs.FileChecksum;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientException;
+import org.apache.hadoop.ozone.client.OzoneKeyDetails;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
+import org.apache.hadoop.ozone.shell.OzoneAddress;
+import org.apache.hadoop.util.CrcUtil;
+import org.apache.hadoop.util.DataChecksum;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.io.UnsupportedEncodingException;
+import java.nio.charset.StandardCharsets;
+
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.mock;
+
+/**
+ * Tests for ChecksumKeyHandler.
+ */
+public class TestChecksumKeyHandler {
+
+ private ChecksumKeyHandler cmd;
+ private final ByteArrayOutputStream outContent = new ByteArrayOutputStream();
+ private final ByteArrayOutputStream errContent = new ByteArrayOutputStream();
+ private final PrintStream originalOut = System.out;
+ private final PrintStream originalErr = System.err;
+ private static final String DEFAULT_ENCODING = StandardCharsets.UTF_8.name();
+ private static final int CHECKSUM = 123456;
+
+ @BeforeEach
+ public void setup() throws UnsupportedEncodingException {
+
+ cmd = new ChecksumKeyHandler() {
+ // Just return a known checksum for testing, as it reduces the amount of
+ // mocking needed.
+ @Override
+ protected FileChecksum getFileChecksum(OzoneVolume vol,
+ OzoneBucket bucket, String keyName, long dataSize,
+ ClientProtocol clientProxy) {
+ return new CompositeCrcFileChecksum(
+ CHECKSUM, DataChecksum.Type.CRC32, 512);
+ }
+ };
+ System.setOut(new PrintStream(outContent, false, DEFAULT_ENCODING));
+ System.setErr(new PrintStream(errContent, false, DEFAULT_ENCODING));
+ }
+
+ @AfterEach
+ public void tearDown() {
+ System.setOut(originalOut);
+ System.setErr(originalErr);
+ }
+
+ @Test
+ public void testChecksumKeyHandler()
+ throws OzoneClientException, IOException {
+ OzoneAddress address = new OzoneAddress("o3://ozone1/volume/bucket/key");
+ long keySize = 1024L;
+
+ ObjectStore objectStore = mock(ObjectStore.class);
+ OzoneClient client = mock(OzoneClient.class);
+ Mockito.when(client.getObjectStore()).thenReturn(objectStore);
+
+ OzoneVolume volume = mock(OzoneVolume.class);
+ OzoneBucket bucket = mock(OzoneBucket.class);
+ OzoneKeyDetails key = mock(OzoneKeyDetails.class);
+
+ Mockito.when(volume.getBucket(anyString())).thenReturn(bucket);
+ Mockito.when(bucket.getKey(anyString()))
+ .thenReturn(key);
+ Mockito.when(objectStore.getVolume(anyString())).
+ thenReturn(volume);
+ Mockito.when(key.getDataSize()).thenReturn(keySize);
+
+ cmd.execute(client, address);
+
+ ObjectMapper mapper = new ObjectMapper();
+ JsonNode json = mapper.readTree(outContent.toString("UTF-8"));
+ Assertions.assertEquals("volume", json.get("volumeName").asText());
+ Assertions.assertEquals("bucket", json.get("bucketName").asText());
+ Assertions.assertEquals("key", json.get("name").asText());
+ Assertions.assertEquals(keySize, json.get("dataSize").asLong());
+ Assertions.assertEquals("COMPOSITE-CRC32", json.get("algorithm").asText());
+
+ String expectedChecksum = javax.xml.bind.DatatypeConverter.printHexBinary(
+ CrcUtil.intToBytes(Integer.valueOf(CHECKSUM)));
+ Assertions.assertEquals(expectedChecksum, json.get("checksum").asText());
+ }
+
+}
diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/shell/keys/package-info.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/shell/keys/package-info.java
new file mode 100644
index 0000000000..40f30232bf
--- /dev/null
+++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/shell/keys/package-info.java
@@ -0,0 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.shell.keys;
+/**
+ * Tests for ozone shell key commands.
+ */
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org