[incubator-pinot] 01/01: Add table config backup tool

[jf...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

jfim pushed a commit to branch add-config-backup-tool
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git

commit 5c75eff367043f8db0811e3138dd2d86b8dfc049
Author: Jean-Francois Im <je...@gmail.com>
AuthorDate: Mon Oct 1 09:58:36 2018 -0700

    Add table config backup tool
    
    Add a tool that allows saving table configurations to disk using the new
    configuration format.
---
 .../helix/ControllerRequestURLBuilder.java         |   4 +
 .../pinot/tools/admin/PinotAdministrator.java      |   4 +-
 .../admin/command/BackupTableConfigsCommand.java   | 125 +++++++++++++++++++++
 3 files changed, 132 insertions(+), 1 deletion(-)

diff --git a/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/ControllerRequestURLBuilder.java b/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/ControllerRequestURLBuilder.java
index 9a79cc3..cfb4a84 100644
--- a/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/ControllerRequestURLBuilder.java
+++ b/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/ControllerRequestURLBuilder.java
@@ -76,6 +76,10 @@ public class ControllerRequestURLBuilder {
     return StringUtil.join("/", StringUtils.chomp(_baseUrl, "/"), "instances");
   }
 
+  public String forTableList() {
+    return StringUtil.join("/", StringUtils.chomp(_baseUrl, "/"), "tables");
+  }
+
   public String forTablesFromTenant(String tenantName) {
     return StringUtil.join("/", StringUtils.chomp(_baseUrl, "/"), TENANTS, tenantName, TABLES);
   }
diff --git a/pinot-tools/src/main/java/com/linkedin/pinot/tools/admin/PinotAdministrator.java b/pinot-tools/src/main/java/com/linkedin/pinot/tools/admin/PinotAdministrator.java
index ac5054a..9a6bbeb 100644
--- a/pinot-tools/src/main/java/com/linkedin/pinot/tools/admin/PinotAdministrator.java
+++ b/pinot-tools/src/main/java/com/linkedin/pinot/tools/admin/PinotAdministrator.java
@@ -16,6 +16,7 @@
 package com.linkedin.pinot.tools.admin;
 
 import com.linkedin.pinot.tools.admin.command.ApplyTableConfigCommand;
+import com.linkedin.pinot.tools.admin.command.BackupTableConfigsCommand;
 import com.linkedin.pinot.tools.admin.command.MoveReplicaGroup;
 
 import com.linkedin.pinot.tools.admin.command.RealtimeProvisioningHelperCommand;
@@ -96,7 +97,8 @@ public class PinotAdministrator {
       @SubCommand(name = "BackfillSegmentColumn", impl = BackfillDateTimeColumnCommand.class),
       @SubCommand(name = "VerifyClusterState", impl = VerifyClusterStateCommand.class),
       @SubCommand(name = "ApplyTableConfig", impl = ApplyTableConfigCommand.class),
-      @SubCommand(name = "RealtimeProvisioningHelper", impl = RealtimeProvisioningHelperCommand.class)
+      @SubCommand(name = "RealtimeProvisioningHelper", impl = RealtimeProvisioningHelperCommand.class),
+      @SubCommand(name = "BackupTableConfigs", impl = BackupTableConfigsCommand.class)
   })
   Command _subCommand;
   // @formatter:on
diff --git a/pinot-tools/src/main/java/com/linkedin/pinot/tools/admin/command/BackupTableConfigsCommand.java b/pinot-tools/src/main/java/com/linkedin/pinot/tools/admin/command/BackupTableConfigsCommand.java
new file mode 100644
index 0000000..182aed5
--- /dev/null
+++ b/pinot-tools/src/main/java/com/linkedin/pinot/tools/admin/command/BackupTableConfigsCommand.java
@@ -0,0 +1,125 @@
+/**
+ * Copyright (C) 2014-2016 LinkedIn Corp. (pinot-core@linkedin.com)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.linkedin.pinot.tools.admin.command;
+
+import com.google.common.base.Charsets;
+import com.linkedin.pinot.controller.helix.ControllerRequestURLBuilder;
+import com.linkedin.pinot.tools.Command;
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+import org.apache.http.HttpResponse;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.HttpClients;
+import org.json.JSONArray;
+import org.json.JSONObject;
+import org.kohsuke.args4j.Option;
+
+
+/**
+ * Command to back up table configurations.
+ */
+public class BackupTableConfigsCommand extends AbstractBaseAdminCommand implements Command {
+  @Option(name = "-controllerUrl", required = true, metaVar = "<String>", usage = "Controller URL")
+  private String _controllerUrl;
+
+  @Option(name = "-tableName", required = false, usage = "Table name to back up. Optional, if not specified, defaults to backing up all tables")
+  private String _tableName = null;
+
+  @Option(name = "-outputDir", required = false, usage = "Directory in which table configs are placed. Optional, defaults to the current working directory. Directory is created if it does not exist.")
+  private String _outputDir = ".";
+
+  @Option(name = "-help", required = false, help = true, aliases = { "-h", "--h", "--help" },
+      usage = "Print this message.")
+  private boolean _help = false;
+
+  @Override
+  public boolean execute() throws Exception {
+    // Create the output directory if it does not exist
+    File outputDirectory = new File(_outputDir);
+    if (!outputDirectory.exists()) {
+      outputDirectory.mkdirs();
+    }
+
+    // Populate the list of tables to fetch, if applicable
+    ControllerRequestURLBuilder requestURLBuilder = ControllerRequestURLBuilder.baseUrl(_controllerUrl);
+    HttpClient client = HttpClients.createDefault();
+    List<String> tablesToFetch;
+    if (_tableName != null) {
+      tablesToFetch = Collections.singletonList(_tableName);
+    } else {
+      HttpResponse response = client.execute(new HttpGet(requestURLBuilder.forTableList()));
+      int statusCode = response.getStatusLine().getStatusCode();
+      String responseEntity = IOUtils.toString(response.getEntity().getContent());
+      if (statusCode / 100 == 2) {
+        JSONArray tableNameArray = new JSONObject(responseEntity).getJSONArray("tables");
+        tablesToFetch = new ArrayList<>(tableNameArray.length());
+        for (int i = 0; i < tableNameArray.length(); i++) {
+          tablesToFetch.add(tableNameArray.getString(i));
+        }
+      } else {
+        throw new RuntimeException("Failed to fetch the table list, got HTTP status " + statusCode + ": " + responseEntity);
+      }
+    }
+
+    System.out.println("Will fetch " + tablesToFetch.size() + " table(s)");
+
+    boolean allTablesSuccessful = true;
+    for (String tableName : tablesToFetch) {
+      HttpResponse response = client.execute(new HttpGet(requestURLBuilder.forNewUpdateTableConfig(tableName)));
+      int statusCode = response.getStatusLine().getStatusCode();
+      String responseEntity = IOUtils.toString(response.getEntity().getContent());
+      if (statusCode / 100 == 2) {
+        try {
+          File outputFile = new File(outputDirectory, tableName + ".conf");
+          FileUtils.writeStringToFile(outputFile, responseEntity, Charsets.UTF_8);
+          System.out.println("Saved table configuration for table " +  tableName);
+        } catch (IOException e) {
+          System.out.println("Failed to write table configuration for table " + tableName + ", got exception.");
+          e.printStackTrace();
+          allTablesSuccessful = false;
+        }
+      } else {
+        System.out.println("Failed to fetch table configuration for table " + tableName + ", got HTTP status " + statusCode + ": " + responseEntity);
+        allTablesSuccessful = false;
+      }
+    }
+
+    if (allTablesSuccessful) {
+      System.out.println("Successfully backed up " + tablesToFetch.size() + " tables");
+    } else {
+      System.out.println("Some tables failed to back up, see messages above for details.");
+    }
+
+    return allTablesSuccessful;
+  }
+
+  @Override
+  public String description() {
+    return "Backs up table configurations.";
+  }
+
+  @Override
+  public boolean getHelp() {
+    return _help;
+  }
+}


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org