You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by si...@apache.org on 2023/02/17 02:26:29 UTC
[hudi] branch master updated: [HUDI-1593] Add support for "show restores" and "show restore" commands in hudi-cli (#7868)
This is an automated email from the ASF dual-hosted git repository.
sivabalan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 45017036ecf [HUDI-1593] Add support for "show restores" and "show restore" commands in hudi-cli (#7868)
45017036ecf is described below
commit 45017036ecfa7ca70b7b9e5cc2435abb87f79e3e
Author: Pramod Biligiri <pr...@gmail.com>
AuthorDate: Fri Feb 17 07:56:22 2023 +0530
[HUDI-1593] Add support for "show restores" and "show restore" commands in hudi-cli (#7868)
This adds two commands to the hudi-cli: "show restores" and "show restore --instant INSTANT_VALUE".
---
.../apache/hudi/cli/HoodieTableHeaderFields.java | 6 +
.../apache/hudi/cli/commands/RestoresCommand.java | 172 +++++++++++++++++
.../hudi/cli/commands/TestRestoresCommand.java | 207 +++++++++++++++++++++
3 files changed, 385 insertions(+)
diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/HoodieTableHeaderFields.java b/hudi-cli/src/main/java/org/apache/hudi/cli/HoodieTableHeaderFields.java
index e6016e4cc1c..20829251ee2 100644
--- a/hudi-cli/src/main/java/org/apache/hudi/cli/HoodieTableHeaderFields.java
+++ b/hudi-cli/src/main/java/org/apache/hudi/cli/HoodieTableHeaderFields.java
@@ -101,6 +101,12 @@ public class HoodieTableHeaderFields {
public static final String HEADER_DELETED_FILE = "Deleted File";
public static final String HEADER_SUCCEEDED = "Succeeded";
+ /**
+ * Fields of Restore.
+ */
+ public static final String HEADER_RESTORE_INSTANT = "Restored " + HEADER_INSTANT;
+ public static final String HEADER_RESTORE_STATE = "Restore State";
+
/**
* Fields of Stats.
*/
diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RestoresCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RestoresCommand.java
new file mode 100644
index 00000000000..fb6c4b7a66c
--- /dev/null
+++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RestoresCommand.java
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.cli.commands;
+
+import org.apache.hudi.avro.model.HoodieInstantInfo;
+import org.apache.hudi.avro.model.HoodieRestoreMetadata;
+import org.apache.hudi.avro.model.HoodieRestorePlan;
+import org.apache.hudi.cli.HoodieCLI;
+import org.apache.hudi.cli.HoodiePrintHelper;
+import org.apache.hudi.cli.HoodieTableHeaderFields;
+import org.apache.hudi.cli.TableHeader;
+import org.apache.hudi.common.table.timeline.HoodieActiveTimeline;
+import org.apache.hudi.common.table.timeline.HoodieInstant;
+import org.apache.hudi.common.table.timeline.TimelineMetadataUtils;
+import org.apache.hudi.common.util.Option;
+import org.springframework.shell.standard.ShellComponent;
+import org.springframework.shell.standard.ShellMethod;
+import org.springframework.shell.standard.ShellOption;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+import static org.apache.hudi.common.table.timeline.HoodieTimeline.RESTORE_ACTION;
+
+/**
+ * CLI command to display info about restore actions.
+ */
+@ShellComponent
+public class RestoresCommand {
+
+ @ShellMethod(key = "show restores", value = "List all restore instants")
+ public String showRestores(
+ @ShellOption(value = {"--limit"}, help = "Limit #rows to be displayed", defaultValue = "10") Integer limit,
+ @ShellOption(value = {"--sortBy"}, help = "Sorting Field", defaultValue = "") final String sortByField,
+ @ShellOption(value = {"--desc"}, help = "Ordering", defaultValue = "false") final boolean descending,
+ @ShellOption(value = {"--headeronly"}, help = "Print Header Only",
+ defaultValue = "false") final boolean headerOnly,
+ @ShellOption(value = {"--includeInflights"}, help = "Also list restores that are in flight",
+ defaultValue = "false") final boolean includeInflights) {
+
+ HoodieActiveTimeline activeTimeline = HoodieCLI.getTableMetaClient().getActiveTimeline();
+ List<HoodieInstant> restoreInstants = getRestoreInstants(activeTimeline, includeInflights);
+
+ final List<Comparable[]> outputRows = new ArrayList<>();
+ for (HoodieInstant restoreInstant : restoreInstants) {
+ populateOutputFromRestoreInstant(restoreInstant, outputRows, activeTimeline);
+ }
+
+ TableHeader header = createResultHeader();
+ return HoodiePrintHelper.print(header, new HashMap<>(), sortByField, descending, limit, headerOnly, outputRows);
+ }
+
+ @ShellMethod(key = "show restore", value = "Show details of a restore instant")
+ public String showRestore(
+ @ShellOption(value = {"--instant"}, help = "Restore instant") String restoreInstant,
+ @ShellOption(value = {"--limit"}, help = "Limit #rows to be displayed", defaultValue = "10") Integer limit,
+ @ShellOption(value = {"--sortBy"}, help = "Sorting Field", defaultValue = "") final String sortByField,
+ @ShellOption(value = {"--desc"}, help = "Ordering", defaultValue = "false") final boolean descending,
+ @ShellOption(value = {"--headeronly"}, help = "Print Header Only",
+ defaultValue = "false") final boolean headerOnly) {
+
+ HoodieActiveTimeline activeTimeline = HoodieCLI.getTableMetaClient().getActiveTimeline();
+ List<HoodieInstant> matchingInstants = activeTimeline.filterCompletedInstants().filter(completed ->
+ completed.getTimestamp().equals(restoreInstant)).getInstants();
+ if (matchingInstants.isEmpty()) {
+ matchingInstants = activeTimeline.filterInflights().filter(inflight ->
+ inflight.getTimestamp().equals(restoreInstant)).getInstants();
+ }
+
+ // Assuming a single exact match is found in either completed or inflight instants
+ HoodieInstant instant = matchingInstants.get(0);
+ List<Comparable[]> outputRows = new ArrayList<>();
+ populateOutputFromRestoreInstant(instant, outputRows, activeTimeline);
+
+ TableHeader header = createResultHeader();
+ return HoodiePrintHelper.print(header, new HashMap<>(), sortByField, descending, limit, headerOnly, outputRows);
+ }
+
+ private void addDetailsOfCompletedRestore(HoodieActiveTimeline activeTimeline, List<Comparable[]> rows,
+ HoodieInstant restoreInstant) throws IOException {
+ HoodieRestoreMetadata instantMetadata;
+ Option<byte[]> instantDetails = activeTimeline.getInstantDetails(restoreInstant);
+ instantMetadata = TimelineMetadataUtils
+ .deserializeAvroMetadata(instantDetails.get(), HoodieRestoreMetadata.class);
+
+ for (String rolledbackInstant : instantMetadata.getInstantsToRollback()) {
+ Comparable[] row = createDataRow(instantMetadata.getStartRestoreTime(), rolledbackInstant,
+ instantMetadata.getTimeTakenInMillis(), restoreInstant.getState());
+ rows.add(row);
+ }
+ }
+
+ private void addDetailsOfInflightRestore(HoodieActiveTimeline activeTimeline, List<Comparable[]> rows,
+ HoodieInstant restoreInstant) throws IOException {
+ HoodieRestorePlan restorePlan = getRestorePlan(activeTimeline, restoreInstant);
+ for (HoodieInstantInfo instantToRollback : restorePlan.getInstantsToRollback()) {
+ Comparable[] dataRow = createDataRow(restoreInstant.getTimestamp(), instantToRollback.getCommitTime(), "",
+ restoreInstant.getState());
+ rows.add(dataRow);
+ }
+ }
+
+ private HoodieRestorePlan getRestorePlan(HoodieActiveTimeline activeTimeline, HoodieInstant restoreInstant) throws IOException {
+ HoodieInstant instantKey = new HoodieInstant(HoodieInstant.State.REQUESTED, RESTORE_ACTION,
+ restoreInstant.getTimestamp());
+ Option<byte[]> instantDetails = activeTimeline.getInstantDetails(instantKey);
+ HoodieRestorePlan restorePlan = TimelineMetadataUtils
+ .deserializeAvroMetadata(instantDetails.get(), HoodieRestorePlan.class);
+ return restorePlan;
+ }
+
+ private List<HoodieInstant> getRestoreInstants(HoodieActiveTimeline activeTimeline, boolean includeInFlight) {
+ List<HoodieInstant> restores = new ArrayList<>();
+ restores.addAll(activeTimeline.getRestoreTimeline().filterCompletedInstants().getInstants());
+
+ if (includeInFlight) {
+ restores.addAll(activeTimeline.getRestoreTimeline().filterInflights().getInstants());
+ }
+
+ return restores;
+ }
+
+ private TableHeader createResultHeader() {
+ return new TableHeader()
+ .addTableHeaderField(HoodieTableHeaderFields.HEADER_INSTANT)
+ .addTableHeaderField(HoodieTableHeaderFields.HEADER_RESTORE_INSTANT)
+ .addTableHeaderField(HoodieTableHeaderFields.HEADER_TIME_TOKEN_MILLIS)
+ .addTableHeaderField(HoodieTableHeaderFields.HEADER_RESTORE_STATE);
+ }
+
+ private void populateOutputFromRestoreInstant(HoodieInstant restoreInstant, List<Comparable[]> outputRows,
+ HoodieActiveTimeline activeTimeline) {
+ try {
+ if (restoreInstant.isInflight() || restoreInstant.isRequested()) {
+ addDetailsOfInflightRestore(activeTimeline, outputRows, restoreInstant);
+ } else if (restoreInstant.isCompleted()) {
+ addDetailsOfCompletedRestore(activeTimeline, outputRows, restoreInstant);
+ }
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+
+ private Comparable[] createDataRow(Comparable restoreInstantTimestamp, Comparable rolledbackInstantTimestamp,
+ Comparable timeInterval, Comparable state) {
+ Comparable[] row = new Comparable[4];
+ row[0] = restoreInstantTimestamp;
+ row[1] = rolledbackInstantTimestamp;
+ row[2] = timeInterval;
+ row[3] = state;
+ return row;
+ }
+
+}
diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRestoresCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRestoresCommand.java
new file mode 100644
index 00000000000..aa75ff29b8b
--- /dev/null
+++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRestoresCommand.java
@@ -0,0 +1,207 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.cli.commands;
+
+import org.apache.hudi.avro.model.HoodieRestoreMetadata;
+import org.apache.hudi.avro.model.HoodieSavepointMetadata;
+import org.apache.hudi.cli.HoodieCLI;
+import org.apache.hudi.cli.HoodiePrintHelper;
+import org.apache.hudi.cli.HoodieTableHeaderFields;
+import org.apache.hudi.cli.TableHeader;
+import org.apache.hudi.cli.functional.CLIFunctionalTestHarness;
+import org.apache.hudi.cli.testutils.ShellEvaluationResultUtil;
+import org.apache.hudi.client.BaseHoodieWriteClient;
+import org.apache.hudi.client.SparkRDDWriteClient;
+import org.apache.hudi.common.config.HoodieMetadataConfig;
+import org.apache.hudi.common.model.HoodieTableType;
+import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.common.table.timeline.HoodieActiveTimeline;
+import org.apache.hudi.common.table.timeline.HoodieInstant;
+import org.apache.hudi.common.table.timeline.TimelineMetadataUtils;
+import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion;
+import org.apache.hudi.common.testutils.HoodieMetadataTestTable;
+import org.apache.hudi.common.testutils.HoodieTestTable;
+import org.apache.hudi.config.HoodieIndexConfig;
+import org.apache.hudi.config.HoodieWriteConfig;
+import org.apache.hudi.index.HoodieIndex;
+import org.apache.hudi.metadata.SparkHoodieBackedTableMetadataWriter;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.api.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.shell.Shell;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Stream;
+
+import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH;
+import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_PARTITION_PATHS;
+import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH;
+import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_THIRD_PARTITION_PATH;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+@Tag("functional")
+@SpringBootTest(properties = {"spring.shell.interactive.enabled=false", "spring.shell.command.script.enabled=false"})
+public class TestRestoresCommand extends CLIFunctionalTestHarness {
+
+ @Autowired
+ private Shell shell;
+
+ @BeforeEach
+ public void init() throws Exception {
+ String tableName = tableName();
+ String tablePath = tablePath(tableName);
+ new TableCommand().createTable(
+ tablePath, tableName, HoodieTableType.MERGE_ON_READ.name(),
+ "", TimelineLayoutVersion.VERSION_1, "org.apache.hudi.common.model.HoodieAvroPayload");
+ HoodieTableMetaClient metaClient = HoodieTableMetaClient.reload(HoodieCLI.getTableMetaClient());
+ //Create some commits files and base files
+ Map<String, String> partitionAndFileId = new HashMap<String, String>() {
+ {
+ put(DEFAULT_FIRST_PARTITION_PATH, "file-1");
+ put(DEFAULT_SECOND_PARTITION_PATH, "file-2");
+ put(DEFAULT_THIRD_PARTITION_PATH, "file-3");
+ }
+ };
+
+ HoodieWriteConfig config = HoodieWriteConfig.newBuilder().withPath(tablePath)
+ .withMetadataConfig(
+ // Column Stats Index is disabled, since these tests construct tables which are
+ // not valid (empty commit metadata, etc)
+ HoodieMetadataConfig.newBuilder()
+ .withMetadataIndexColumnStats(false)
+ .build()
+ )
+ .withRollbackUsingMarkers(false)
+ .withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(HoodieIndex.IndexType.INMEMORY).build())
+ .build();
+
+ HoodieTestTable hoodieTestTable = HoodieMetadataTestTable.of(metaClient, SparkHoodieBackedTableMetadataWriter.create(
+ metaClient.getHadoopConf(), config, context))
+ .withPartitionMetaFiles(DEFAULT_PARTITION_PATHS)
+ .addCommit("100")
+ .withBaseFilesInPartitions(partitionAndFileId)
+ .addCommit("101");
+
+ hoodieTestTable.addCommit("102").withBaseFilesInPartitions(partitionAndFileId);
+ HoodieSavepointMetadata savepointMetadata2 = hoodieTestTable.doSavepoint("102");
+ hoodieTestTable.addSavepoint("102", savepointMetadata2);
+
+ hoodieTestTable.addCommit("103").withBaseFilesInPartitions(partitionAndFileId);
+
+ BaseHoodieWriteClient client = new SparkRDDWriteClient(context(), config);
+ client.rollback("103");
+ client.restoreToSavepoint("102");
+
+ hoodieTestTable.addCommit("105").withBaseFilesInPartitions(partitionAndFileId);
+ HoodieSavepointMetadata savepointMetadata = hoodieTestTable.doSavepoint("105");
+ hoodieTestTable.addSavepoint("105", savepointMetadata);
+
+ hoodieTestTable.addCommit("106").withBaseFilesInPartitions(partitionAndFileId);
+ client.rollback("106");
+ client.restoreToSavepoint("105");
+ client.close();
+ }
+
+ @Test
+ public void testShowRestores() {
+ Object result = shell.evaluate(() -> "show restores");
+ assertTrue(ShellEvaluationResultUtil.isSuccess(result));
+
+ // get restored instants
+ HoodieActiveTimeline activeTimeline = HoodieCLI.getTableMetaClient().getActiveTimeline();
+ Stream<HoodieInstant> restores = activeTimeline.getRestoreTimeline().filterCompletedInstants().getInstantsAsStream();
+
+ List<Comparable[]> rows = new ArrayList<>();
+ restores.sorted().forEach(instant -> {
+ try {
+ HoodieRestoreMetadata metadata = TimelineMetadataUtils
+ .deserializeAvroMetadata(activeTimeline.getInstantDetails(instant).get(), HoodieRestoreMetadata.class);
+ metadata.getInstantsToRollback().forEach(c -> {
+ Comparable[] row = new Comparable[4];
+ row[0] = metadata.getStartRestoreTime();
+ row[1] = c;
+ row[2] = metadata.getTimeTakenInMillis();
+ row[3] = HoodieInstant.State.COMPLETED.toString();
+ rows.add(row);
+ });
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ });
+
+ TableHeader header = new TableHeader()
+ .addTableHeaderField(HoodieTableHeaderFields.HEADER_INSTANT)
+ .addTableHeaderField(HoodieTableHeaderFields.HEADER_RESTORE_INSTANT)
+ .addTableHeaderField(HoodieTableHeaderFields.HEADER_TIME_TOKEN_MILLIS)
+ .addTableHeaderField(HoodieTableHeaderFields.HEADER_RESTORE_STATE);
+ String expected = HoodiePrintHelper.print(header, new HashMap<>(), "", false,
+ -1, false, rows);
+ expected = removeNonWordAndStripSpace(expected);
+ String got = removeNonWordAndStripSpace(result.toString());
+ assertEquals(expected, got);
+ }
+
+ @Test
+ public void testShowRestore() throws IOException {
+ // get instant
+ HoodieActiveTimeline activeTimeline = HoodieCLI.getTableMetaClient().getActiveTimeline();
+ Stream<HoodieInstant> restores = activeTimeline.getRestoreTimeline().filterCompletedInstants().getInstantsAsStream();
+ HoodieInstant instant = restores.findFirst().orElse(null);
+ assertNotNull(instant, "The instant can not be null.");
+
+ Object result = shell.evaluate(() -> "show restore --instant " + instant.getTimestamp());
+ assertTrue(ShellEvaluationResultUtil.isSuccess(result));
+
+ // get metadata of instant
+ HoodieRestoreMetadata instantMetadata = TimelineMetadataUtils.deserializeAvroMetadata(
+ activeTimeline.getInstantDetails(instant).get(), HoodieRestoreMetadata.class);
+
+ // generate expected result
+ TableHeader header = new TableHeader()
+ .addTableHeaderField(HoodieTableHeaderFields.HEADER_INSTANT)
+ .addTableHeaderField(HoodieTableHeaderFields.HEADER_RESTORE_INSTANT)
+ .addTableHeaderField(HoodieTableHeaderFields.HEADER_TIME_TOKEN_MILLIS)
+ .addTableHeaderField(HoodieTableHeaderFields.HEADER_RESTORE_STATE);
+
+ List<Comparable[]> rows = new ArrayList<>();
+ instantMetadata.getInstantsToRollback().forEach((String rolledbackInstant) -> {
+ Comparable[] row = new Comparable[4];
+ row[0] = instantMetadata.getStartRestoreTime();
+ row[1] = rolledbackInstant;
+ row[2] = instantMetadata.getTimeTakenInMillis();
+ row[3] = HoodieInstant.State.COMPLETED.toString();
+ rows.add(row);
+ });
+ String expected = HoodiePrintHelper.print(header, new HashMap<>(), "", false, -1,
+ false, rows);
+ expected = removeNonWordAndStripSpace(expected);
+ String got = removeNonWordAndStripSpace(result.toString());
+ assertEquals(expected, got);
+ }
+
+}