You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by el...@apache.org on 2021/05/13 16:23:05 UTC
[ozone] branch master updated: HDDS-4976. Add container replica
related commands to debug (#2070)
This is an automated email from the ASF dual-hosted git repository.
elek pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 6b24c18 HDDS-4976. Add container replica related commands to debug (#2070)
6b24c18 is described below
commit 6b24c188df038f1ab404acace20a4060fe2114fd
Author: Gui Hecheng <ma...@tencent.com>
AuthorDate: Fri May 14 00:22:49 2021 +0800
HDDS-4976. Add container replica related commands to debug (#2070)
---
.../container/keyvalue/KeyValueContainerData.java | 2 +
.../org/apache/hadoop/ozone/debug/OzoneDebug.java | 10 ++
.../ContainerCommands.java} | 130 +++++++++++----------
.../ozone/debug/container/ExportSubcommand.java | 75 ++++++++++++
.../ozone/debug/container/InfoSubcommand.java | 53 +++++++++
.../ozone/debug/container/ListSubcommand.java | 50 ++++++++
.../hadoop/ozone/debug/container/package-info.java | 22 ++++
7 files changed, 283 insertions(+), 59 deletions(-)
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerData.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerData.java
index 9f93e50..e1a1f02 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerData.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerData.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.ozone.container.keyvalue;
+import com.fasterxml.jackson.annotation.JsonIgnore;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
@@ -251,6 +252,7 @@ public class KeyValueContainerData extends ContainerData {
* @return Protocol Buffer Message
*/
@Override
+ @JsonIgnore
public ContainerDataProto getProtoBufMessage() {
ContainerDataProto.Builder builder = ContainerDataProto.newBuilder();
builder.setContainerID(this.getContainerID());
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/OzoneDebug.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/OzoneDebug.java
index 9647396..41b96da 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/OzoneDebug.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/OzoneDebug.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.ozone.debug;
import org.apache.hadoop.hdds.cli.GenericCli;
import org.apache.hadoop.hdds.cli.HddsVersionProvider;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import picocli.CommandLine;
/**
@@ -32,10 +33,19 @@ import picocli.CommandLine;
mixinStandardHelpOptions = true)
public class OzoneDebug extends GenericCli {
+ private OzoneConfiguration ozoneConf;
+
public OzoneDebug() {
super(OzoneDebug.class);
}
+ public OzoneConfiguration getOzoneConf() {
+ if (ozoneConf == null) {
+ ozoneConf = createOzoneConfiguration();
+ }
+ return ozoneConf;
+ }
+
/**
* Main for the Ozone Debug shell Command handling.
*
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/ExportContainer.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/container/ContainerCommands.java
similarity index 69%
rename from hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/ExportContainer.java
rename to hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/container/ContainerCommands.java
index dab4c9c..fd8ebcc 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/ExportContainer.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/container/ContainerCommands.java
@@ -16,30 +16,22 @@
* limitations under the License.
*/
-package org.apache.hadoop.ozone.debug;
+package org.apache.hadoop.ozone.debug.container;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Properties;
-import java.util.concurrent.Callable;
-
-import org.apache.hadoop.hdds.cli.GenericParentCommand;
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.hdds.cli.GenericCli;
+import org.apache.hadoop.hdds.cli.HddsVersionProvider;
import org.apache.hadoop.hdds.cli.SubcommandWithParent;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerType;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
+import org.apache.hadoop.hdds.server.JsonUtils;
import org.apache.hadoop.hdfs.server.datanode.StorageLocation;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.common.InconsistentStorageStateException;
import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics;
import org.apache.hadoop.ozone.container.common.helpers.DatanodeVersionFile;
+import org.apache.hadoop.ozone.container.common.impl.ContainerData;
import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
import org.apache.hadoop.ozone.container.common.interfaces.Handler;
import org.apache.hadoop.ozone.container.common.utils.HddsVolumeUtil;
@@ -47,46 +39,70 @@ import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet;
import org.apache.hadoop.ozone.container.ozoneimpl.ContainerController;
import org.apache.hadoop.ozone.container.ozoneimpl.ContainerReader;
-import org.apache.hadoop.ozone.container.replication.ContainerReplicationSource;
-import org.apache.hadoop.ozone.container.replication.OnDemandContainerReplicationSource;
-
-import com.google.common.base.Preconditions;
+import org.apache.hadoop.ozone.debug.OzoneDebug;
import org.kohsuke.MetaInfServices;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import picocli.CommandLine;
import picocli.CommandLine.Command;
import picocli.CommandLine.ParentCommand;
+import picocli.CommandLine.Model.CommandSpec;
+import picocli.CommandLine.Spec;
-@Command(name = "export-container",
- description = "Export one container to a tarball")
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.Callable;
+
+/**
+ * Subcommand to group container replica related operations.
+ */
+@Command(
+ name = "container",
+ description = "Container replica specific operations" +
+ " to be executed on datanodes only",
+ mixinStandardHelpOptions = true,
+ versionProvider = HddsVersionProvider.class,
+ subcommands = {
+ ListSubcommand.class,
+ InfoSubcommand.class,
+ ExportSubcommand.class,
+ })
@MetaInfServices(SubcommandWithParent.class)
-public class ExportContainer implements SubcommandWithParent, Callable<Void> {
+public class ContainerCommands implements Callable<Void>, SubcommandWithParent {
private static final Logger LOG =
- LoggerFactory.getLogger(ExportContainer.class);
+ LoggerFactory.getLogger(ContainerCommands.class);
+
@ParentCommand
- private GenericParentCommand parent;
+ private OzoneDebug parent;
- @CommandLine.Option(names = {"--container"},
- required = true,
- description = "Container Id")
- private long containerId;
+ @Spec
+ private CommandSpec spec;
- @CommandLine.Option(names = {"--dest"},
- defaultValue = "/tmp",
- description = "Destination directory")
- private String destination;
+ private MutableVolumeSet volumeSet;
+
+ private ContainerController controller;
@Override
- public Class<?> getParentType() {
- return OzoneDebug.class;
+ public Void call() throws Exception {
+ GenericCli.missingSubcommand(spec);
+ return null;
}
@Override
- public Void call() throws Exception {
+ public Class<?> getParentType() {
+ return OzoneDebug.class;
+ }
- ConfigurationSource conf = parent.createOzoneConfiguration();
+ public void loadContainersFromVolumes() throws IOException {
+ OzoneConfiguration conf = parent.getOzoneConf();
ContainerSet containerSet = new ContainerSet();
@@ -98,11 +114,12 @@ public class ExportContainer implements SubcommandWithParent, Callable<Void> {
String clusterId = getClusterId(firstStorageDir);
- MutableVolumeSet volumeSet = new MutableVolumeSet(datanodeUuid, conf);
+ volumeSet = new MutableVolumeSet(datanodeUuid, conf);
- Map<ContainerType, Handler> handlers = new HashMap<>();
+ Map<ContainerProtos.ContainerType, Handler> handlers = new HashMap<>();
- for (ContainerType containerType : ContainerType.values()) {
+ for (ContainerProtos.ContainerType containerType
+ : ContainerProtos.ContainerType.values()) {
final Handler handler =
Handler.getHandlerForContainerType(
containerType,
@@ -117,11 +134,7 @@ public class ExportContainer implements SubcommandWithParent, Callable<Void> {
handlers.put(containerType, handler);
}
- ContainerController controller =
- new ContainerController(containerSet, handlers);
-
- final ContainerReplicationSource replicationSource =
- new OnDemandContainerReplicationSource(controller);
+ controller = new ContainerController(containerSet, handlers);
Iterator<HddsVolume> volumeSetIterator = volumeSet.getVolumesList()
.iterator();
@@ -136,22 +149,18 @@ public class ExportContainer implements SubcommandWithParent, Callable<Void> {
reader.run();
}
- LOG.info("All the container metadata is loaded. Starting to replication");
-
- replicationSource.prepare(containerId);
- LOG.info("Preparation is done");
+ LOG.info("All the container metadata is loaded.");
+ }
- final File destinationFile =
- new File(destination, "container-" + containerId + ".tar.gz");
- try (FileOutputStream fos = new FileOutputStream(destinationFile)) {
- replicationSource.copyData(containerId, fos);
- }
- LOG.info("Container is exported to {}", destinationFile);
+ public MutableVolumeSet getVolumeSet() {
+ return this.volumeSet;
+ }
- return null;
+ public ContainerController getController() {
+ return this.controller;
}
- public String getClusterId(String storageDir) throws IOException {
+ private String getClusterId(String storageDir) throws IOException {
Preconditions.checkNotNull(storageDir);
final Path firstStorageDirPath = Files.list(Paths.get(storageDir, "hdds"))
.filter(Files::isDirectory)
@@ -163,7 +172,7 @@ public class ExportContainer implements SubcommandWithParent, Callable<Void> {
return firstStorageDirPath.toString();
}
- public String getDatanodeUUID(String storageDir, ConfigurationSource config)
+ private String getDatanodeUUID(String storageDir, ConfigurationSource config)
throws IOException {
final File versionFile = new File(storageDir, "hdds/VERSION");
@@ -188,4 +197,7 @@ public class ExportContainer implements SubcommandWithParent, Callable<Void> {
.getUri().getPath();
}
+ public static void outputContainer(ContainerData data) throws IOException {
+ System.out.println(JsonUtils.toJsonStringWithDefaultPrettyPrinter(data));
+ }
}
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/container/ExportSubcommand.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/container/ExportSubcommand.java
new file mode 100644
index 0000000..edeeebd
--- /dev/null
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/container/ExportSubcommand.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.debug.container;
+
+import org.apache.hadoop.ozone.container.replication.ContainerReplicationSource;
+import org.apache.hadoop.ozone.container.replication.OnDemandContainerReplicationSource;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import picocli.CommandLine;
+import picocli.CommandLine.Command;
+import picocli.CommandLine.ParentCommand;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.util.concurrent.Callable;
+
+@Command(
+ name = "export",
+ description = "Export one container to a tarball")
+public class ExportSubcommand implements Callable<Void> {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ExportSubcommand.class);
+
+ @ParentCommand
+ private ContainerCommands parent;
+
+ @CommandLine.Option(names = {"--container"},
+ required = true,
+ description = "Container Id")
+ private long containerId;
+
+ @CommandLine.Option(names = {"--dest"},
+ defaultValue = "/tmp",
+ description = "Destination directory")
+ private String destination;
+
+ @Override
+ public Void call() throws Exception {
+ parent.loadContainersFromVolumes();
+
+ final ContainerReplicationSource replicationSource =
+ new OnDemandContainerReplicationSource(parent.getController());
+
+ LOG.info("Starting to replication");
+
+ replicationSource.prepare(containerId);
+ LOG.info("Preparation is done");
+
+ final File destinationFile =
+ new File(destination, "container-" + containerId + ".tar.gz");
+ try (FileOutputStream fos = new FileOutputStream(destinationFile)) {
+ replicationSource.copyData(containerId, fos);
+ }
+ LOG.info("Container is exported to {}", destinationFile);
+
+ return null;
+ }
+}
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/container/InfoSubcommand.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/container/InfoSubcommand.java
new file mode 100644
index 0000000..3456aeb
--- /dev/null
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/container/InfoSubcommand.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.debug.container;
+
+import org.apache.hadoop.ozone.container.common.interfaces.Container;
+import picocli.CommandLine;
+import picocli.CommandLine.Command;
+
+import java.util.concurrent.Callable;
+
+import static org.apache.hadoop.ozone.debug.container.ContainerCommands.outputContainer;
+
+@Command(
+ name = "info",
+ description = "Show container info of a container replica on datanode")
+public class InfoSubcommand implements Callable<Void> {
+
+ @CommandLine.ParentCommand
+ private ContainerCommands parent;
+
+ @CommandLine.Option(names = {"--container"},
+ required = true,
+ description = "Container Id")
+ private long containerId;
+
+ @Override
+ public Void call() throws Exception {
+ parent.loadContainersFromVolumes();
+
+ Container container = parent.getController().getContainer(containerId);
+ if (container != null) {
+ outputContainer(container.getContainerData());
+ }
+
+ return null;
+ }
+}
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/container/ListSubcommand.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/container/ListSubcommand.java
new file mode 100644
index 0000000..15bdb02
--- /dev/null
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/container/ListSubcommand.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.debug.container;
+
+import org.apache.hadoop.ozone.container.common.interfaces.Container;
+import picocli.CommandLine;
+import picocli.CommandLine.Command;
+
+import java.util.Iterator;
+import java.util.concurrent.Callable;
+
+import static org.apache.hadoop.ozone.debug.container.ContainerCommands.outputContainer;
+
+@Command(
+ name = "list",
+ description = "Show container info of all container replicas on datanode")
+public class ListSubcommand implements Callable<Void> {
+
+ @CommandLine.ParentCommand
+ private ContainerCommands parent;
+
+ @Override
+ public Void call() throws Exception {
+ parent.loadContainersFromVolumes();
+
+ Iterator<Container<?>> containerIt = parent.getController().getContainers();
+ while (containerIt.hasNext()) {
+ Container container = containerIt.next();
+ outputContainer(container.getContainerData());
+ }
+
+ return null;
+ }
+}
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/container/package-info.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/container/package-info.java
new file mode 100644
index 0000000..ba26a10
--- /dev/null
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/container/package-info.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Contains all of the datanode container replica related commands.
+ */
+package org.apache.hadoop.ozone.debug.container;
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org