You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ja...@apache.org on 2022/12/01 01:27:30 UTC
[pinot] branch master updated: #9834 Added Delete Schema/Table pinot admin commands (#9857)
This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 2dae95127d #9834 Added Delete Schema/Table pinot admin commands (#9857)
2dae95127d is described below
commit 2dae95127da06920b8d0a40ddc7f28d17cd1d24b
Author: Priyank Bagrecha <ba...@gmail.com>
AuthorDate: Wed Nov 30 17:27:24 2022 -0800
#9834 Added Delete Schema/Table pinot admin commands (#9857)
---
.../common/utils/FileUploadDownloadClient.java | 33 +++-
.../pinot/tools/admin/PinotAdministrator.java | 4 +
.../tools/admin/command/DeleteSchemaCommand.java | 160 ++++++++++++++++++
.../tools/admin/command/DeleteTableCommand.java | 183 +++++++++++++++++++++
4 files changed, 377 insertions(+), 3 deletions(-)
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/utils/FileUploadDownloadClient.java b/pinot-common/src/main/java/org/apache/pinot/common/utils/FileUploadDownloadClient.java
index c5feddcc02..9a07435bf1 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/utils/FileUploadDownloadClient.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/utils/FileUploadDownloadClient.java
@@ -119,6 +119,8 @@ public class FileUploadDownloadClient implements AutoCloseable {
private static final String FORCE_REVERT_PARAMETER = "&forceRevert=";
private static final String FORCE_CLEANUP_PARAMETER = "&forceCleanup=";
+ private static final String RETENTION_PARAMETER = "retention=";
+
private static final List<String> SUPPORTED_PROTOCOLS = Arrays.asList(HTTP, HTTPS);
private final HttpClient _httpClient;
@@ -236,6 +238,30 @@ public class FileUploadDownloadClient implements AutoCloseable {
return getURI(protocol, host, port, SCHEMA_PATH);
}
+ public static URI getDeleteSchemaURI(String protocol, String host, int port, String schemaName)
+ throws URISyntaxException {
+ return getURI(protocol, host, port, SCHEMA_PATH + "/" + schemaName);
+ }
+
+ public static URI getDeleteTableURI(String protocol, String host, int port, String tableName, String type,
+ String retention)
+ throws URISyntaxException {
+ StringBuilder sb = new StringBuilder();
+ if (StringUtils.isNotBlank(type)) {
+ sb.append(TYPE_DELIMITER);
+ sb.append(type);
+ }
+ if (StringUtils.isNotBlank(retention)) {
+ if (sb.length() > 0) {
+ sb.append("&");
+ }
+ sb.append(RETENTION_PARAMETER);
+ sb.append(retention);
+ }
+ String query = sb.length() == 0 ? null : sb.toString();
+ return getURI(protocol, host, port, TABLES_PATH + "/" + tableName, query);
+ }
+
public static URI getUploadSchemaURI(URI controllerURI)
throws URISyntaxException {
return getURI(controllerURI.getScheme(), controllerURI.getHost(), controllerURI.getPort(), SCHEMA_PATH);
@@ -759,7 +785,8 @@ public class FileUploadDownloadClient implements AutoCloseable {
* If tableType is left unspecified, both OFFLINE and REALTIME segments will be returned in the map.
*/
public Map<String, List<String>> getSegments(URI controllerUri, String rawTableName, @Nullable TableType tableType,
- boolean excludeReplacedSegments) throws Exception {
+ boolean excludeReplacedSegments)
+ throws Exception {
List<String> tableTypes;
if (tableType == null) {
tableTypes = Arrays.asList(TableType.OFFLINE.toString(), TableType.REALTIME.toString());
@@ -771,8 +798,8 @@ public class FileUploadDownloadClient implements AutoCloseable {
Map<String, List<String>> tableTypeToSegments = new HashMap<>();
for (String tableTypeToFilter : tableTypes) {
tableTypeToSegments.put(tableTypeToFilter, new ArrayList<>());
- String uri = controllerRequestURLBuilder.forSegmentListAPI(rawTableName,
- tableTypeToFilter, excludeReplacedSegments);
+ String uri =
+ controllerRequestURLBuilder.forSegmentListAPI(rawTableName, tableTypeToFilter, excludeReplacedSegments);
RequestBuilder requestBuilder = RequestBuilder.get(uri).setVersion(HttpVersion.HTTP_1_1);
HttpClient.setTimeout(requestBuilder, HttpClient.DEFAULT_SOCKET_TIMEOUT_MS);
RetryPolicies.exponentialBackoffRetryPolicy(5, 10_000L, 2.0).attempt(() -> {
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/PinotAdministrator.java b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/PinotAdministrator.java
index f32438adae..12c442fda9 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/PinotAdministrator.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/PinotAdministrator.java
@@ -34,6 +34,8 @@ import org.apache.pinot.tools.admin.command.ChangeTableState;
import org.apache.pinot.tools.admin.command.CreateSegmentCommand;
import org.apache.pinot.tools.admin.command.DataImportDryRunCommand;
import org.apache.pinot.tools.admin.command.DeleteClusterCommand;
+import org.apache.pinot.tools.admin.command.DeleteSchemaCommand;
+import org.apache.pinot.tools.admin.command.DeleteTableCommand;
import org.apache.pinot.tools.admin.command.FileSystemCommand;
import org.apache.pinot.tools.admin.command.GenerateDataCommand;
import org.apache.pinot.tools.admin.command.GitHubEventsQuickStartCommand;
@@ -107,9 +109,11 @@ public class PinotAdministrator {
SUBCOMMAND_MAP.put("StartMinion", new StartMinionCommand());
SUBCOMMAND_MAP.put("StartServiceManager", new StartServiceManagerCommand());
SUBCOMMAND_MAP.put("AddTable", new AddTableCommand());
+ SUBCOMMAND_MAP.put("DeleteTable", new DeleteTableCommand());
SUBCOMMAND_MAP.put("ChangeTableState", new ChangeTableState());
SUBCOMMAND_MAP.put("AddTenant", new AddTenantCommand());
SUBCOMMAND_MAP.put("AddSchema", new AddSchemaCommand());
+ SUBCOMMAND_MAP.put("DeleteSchema", new DeleteSchemaCommand());
SUBCOMMAND_MAP.put("DataImportDryRun", new DataImportDryRunCommand());
SUBCOMMAND_MAP.put("UpdateSchema", new AddSchemaCommand());
SUBCOMMAND_MAP.put("UploadSegment", new UploadSegmentCommand());
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/DeleteSchemaCommand.java b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/DeleteSchemaCommand.java
new file mode 100644
index 0000000000..1378a63f8f
--- /dev/null
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/DeleteSchemaCommand.java
@@ -0,0 +1,160 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.tools.admin.command;
+
+import java.util.Collections;
+import org.apache.pinot.common.utils.FileUploadDownloadClient;
+import org.apache.pinot.spi.auth.AuthProvider;
+import org.apache.pinot.spi.utils.CommonConstants;
+import org.apache.pinot.spi.utils.NetUtils;
+import org.apache.pinot.tools.Command;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import picocli.CommandLine;
+
+
+@CommandLine.Command(name = "DeleteSchema")
+public class DeleteSchemaCommand extends AbstractBaseAdminCommand implements Command {
+ private static final Logger LOGGER = LoggerFactory.getLogger(DeleteSchemaCommand.class);
+
+ @CommandLine.Option(names = {"-controllerHost"}, required = false, description = "host name for controller.")
+ private String _controllerHost;
+
+ @CommandLine.Option(names = {"-controllerPort"}, required = false, description = "port name for controller.")
+ private String _controllerPort = DEFAULT_CONTROLLER_PORT;
+
+ @CommandLine.Option(names = {"-controllerProtocol"}, required = false, description = "protocol for controller.")
+ private String _controllerProtocol = CommonConstants.HTTP_PROTOCOL;
+
+ @CommandLine.Option(names = {"-schemaName"}, required = true, description = "Schema name.")
+ private String _schemaName = null;
+
+ @CommandLine.Option(names = {"-exec"}, required = false, description = "Execute the command.")
+ private boolean _exec;
+
+ @CommandLine.Option(names = {"-user"}, required = false, description = "Username for basic auth.")
+ private String _user;
+
+ @CommandLine.Option(names = {"-password"}, required = false, description = "Password for basic auth.")
+ private String _password;
+
+ @CommandLine.Option(names = {"-authToken"}, required = false, description = "Http auth token.")
+ private String _authToken;
+
+ @CommandLine.Option(names = {"-authTokenUrl"}, required = false, description = "Http auth token url.")
+ private String _authTokenUrl;
+
+ @CommandLine.Option(names = {"-help", "-h", "--h", "--help"}, required = false, help = true, description = "Print "
+ + "this message.")
+ private boolean _help = false;
+
+ private AuthProvider _authProvider;
+
+ @Override
+ public boolean getHelp() {
+ return _help;
+ }
+
+ @Override
+ public String description() {
+ return "Delete schema specified via name";
+ }
+
+ @Override
+ public String getName() {
+ return "DeleteSchema";
+ }
+
+ @Override
+ public String toString() {
+ String retString =
+ ("DeleteSchema -controllerProtocol " + _controllerProtocol + " -controllerHost " + _controllerHost
+ + " -controllerPort " + _controllerPort + " -schemaName " + _schemaName + " -user " + _user + " -password "
+ + "[hidden]");
+
+ return ((_exec) ? (retString + " -exec") : retString);
+ }
+
+ @Override
+ public void cleanup() {
+ }
+
+ public DeleteSchemaCommand setControllerHost(String controllerHost) {
+ _controllerHost = controllerHost;
+ return this;
+ }
+
+ public DeleteSchemaCommand setControllerPort(String controllerPort) {
+ _controllerPort = controllerPort;
+ return this;
+ }
+
+ public DeleteSchemaCommand setControllerProtocol(String controllerProtocol) {
+ _controllerProtocol = controllerProtocol;
+ return this;
+ }
+
+ public DeleteSchemaCommand setSchemaName(String schemaName) {
+ _schemaName = schemaName;
+ return this;
+ }
+
+ public void setUser(String user) {
+ _user = user;
+ }
+
+ public void setPassword(String password) {
+ _password = password;
+ }
+
+ public void setAuthProvider(AuthProvider authProvider) {
+ _authProvider = authProvider;
+ }
+
+ public DeleteSchemaCommand setExecute(boolean exec) {
+ _exec = exec;
+ return this;
+ }
+
+ @Override
+ public boolean execute()
+ throws Exception {
+ if (_controllerHost == null) {
+ _controllerHost = NetUtils.getHostAddress();
+ }
+
+ if (!_exec) {
+ LOGGER.warn("Dry Running Command: " + toString());
+ LOGGER.warn("Use the -exec option to actually execute the command.");
+ return true;
+ }
+
+ LOGGER.info("Executing command: " + toString());
+ try (FileUploadDownloadClient fileUploadDownloadClient = new FileUploadDownloadClient()) {
+ fileUploadDownloadClient.getHttpClient().sendDeleteRequest(
+ FileUploadDownloadClient.getDeleteSchemaURI(_controllerProtocol, _controllerHost,
+ Integer.parseInt(_controllerPort), _schemaName), Collections.emptyMap(),
+ makeAuthProvider(_authProvider, _authTokenUrl, _authToken, _user, _password));
+ } catch (Exception e) {
+ LOGGER.error("Got Exception while deleting Pinot Schema: " + _schemaName, e);
+ return false;
+ }
+ return true;
+ }
+}
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/DeleteTableCommand.java b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/DeleteTableCommand.java
new file mode 100644
index 0000000000..54c8709d6c
--- /dev/null
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/DeleteTableCommand.java
@@ -0,0 +1,183 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.tools.admin.command;
+
+import java.util.Collections;
+import org.apache.pinot.common.utils.FileUploadDownloadClient;
+import org.apache.pinot.spi.auth.AuthProvider;
+import org.apache.pinot.spi.utils.CommonConstants;
+import org.apache.pinot.spi.utils.NetUtils;
+import org.apache.pinot.tools.Command;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import picocli.CommandLine;
+
+
+@CommandLine.Command(name = "DeleteTable")
+public class DeleteTableCommand extends AbstractBaseAdminCommand implements Command {
+ private static final Logger LOGGER = LoggerFactory.getLogger(DeleteTableCommand.class);
+
+ @CommandLine.Option(names = {"-tableName"}, required = true, description = "Name of the table to delete.")
+ private String _tableName;
+
+ @CommandLine.Option(names = {"-type"}, required = false, description = "realtime|offline.")
+ private String _type;
+
+ @CommandLine.Option(names = {"-retention"}, required = false, description = "Retention period for the table "
+ + "segments (e.g. 12h, 3d); If not set, the retention period will default to the first config that's not null: "
+ + "the cluster setting, then '7d'. Using 0d or -1d will instantly delete segments without retention.")
+ private String _retention;
+
+ @CommandLine.Option(names = {"-controllerHost"}, required = false, description = "host name for controller.")
+ private String _controllerHost;
+
+ @CommandLine.Option(names = {"-controllerPort"}, required = false, description = "Port number to start the "
+ + "controller at.")
+ private String _controllerPort = DEFAULT_CONTROLLER_PORT;
+
+ @CommandLine.Option(names = {"-controllerProtocol"}, required = false, description = "protocol for controller.")
+ private String _controllerProtocol = CommonConstants.HTTP_PROTOCOL;
+
+ @CommandLine.Option(names = {"-exec"}, required = false, description = "Execute the command.")
+ private boolean _exec;
+
+ @CommandLine.Option(names = {"-user"}, required = false, description = "Username for basic auth.")
+ private String _user;
+
+ @CommandLine.Option(names = {"-password"}, required = false, description = "Password for basic auth.")
+ private String _password;
+
+ @CommandLine.Option(names = {"-authToken"}, required = false, description = "Http auth token.")
+ private String _authToken;
+
+ @CommandLine.Option(names = {"-authTokenUrl"}, required = false, description = "Http auth token url.")
+ private String _authTokenUrl;
+
+ @CommandLine.Option(names = {"-help", "-h", "--h", "--help"}, required = false, help = true, description = "Print "
+ + "this message.")
+ private boolean _help = false;
+
+ private String _controllerAddress;
+
+ private AuthProvider _authProvider;
+
+ @Override
+ public boolean getHelp() {
+ return _help;
+ }
+
+ @Override
+ public String getName() {
+ return "DeleteTable";
+ }
+
+ @Override
+ public String description() {
+ return "Delete a Pinot table";
+ }
+
+ @Override
+ public String toString() {
+ String retString =
+ ("DeleteTable -tableName " + _tableName + " -type " + _type + " -retention " + _retention
+ + " -controllerProtocol " + _controllerProtocol + " -controllerHost " + _controllerHost
+ + " -controllerPort " + _controllerPort + " -user " + _user + " -password " + "[hidden]");
+ return ((_exec) ? (retString + " -exec") : retString);
+ }
+
+ @Override
+ public void cleanup() {
+ }
+
+ public DeleteTableCommand setTableName(String tableName) {
+ _tableName = tableName;
+ return this;
+ }
+
+ public DeleteTableCommand setType(String type) {
+ _type = type;
+ return this;
+ }
+
+ public DeleteTableCommand setRetention(String retention) {
+ _retention = retention;
+ return this;
+ }
+
+ public DeleteTableCommand setControllerHost(String controllerHost) {
+ _controllerHost = controllerHost;
+ return this;
+ }
+
+ public DeleteTableCommand setControllerPort(String controllerPort) {
+ _controllerPort = controllerPort;
+ return this;
+ }
+
+ public DeleteTableCommand setControllerProtocol(String controllerProtocol) {
+ _controllerProtocol = controllerProtocol;
+ return this;
+ }
+
+ public DeleteTableCommand setUser(String user) {
+ _user = user;
+ return this;
+ }
+
+ public DeleteTableCommand setPassword(String password) {
+ _password = password;
+ return this;
+ }
+
+ public DeleteTableCommand setExecute(boolean exec) {
+ _exec = exec;
+ return this;
+ }
+
+ public DeleteTableCommand setAuthProvider(AuthProvider authProvider) {
+ _authProvider = authProvider;
+ return this;
+ }
+
+ @Override
+ public boolean execute()
+ throws Exception {
+ if (_controllerHost == null) {
+ _controllerHost = NetUtils.getHostAddress();
+ }
+
+ if (!_exec) {
+ LOGGER.warn("Dry Running Command: " + toString());
+ LOGGER.warn("Use the -exec option to actually execute the command.");
+ return true;
+ }
+
+ LOGGER.info("Executing command: " + toString());
+ try (FileUploadDownloadClient fileUploadDownloadClient = new FileUploadDownloadClient()) {
+ fileUploadDownloadClient.getHttpClient().sendDeleteRequest(FileUploadDownloadClient
+ .getDeleteTableURI(_controllerProtocol, _controllerHost, Integer.parseInt(_controllerPort),
+ _tableName, _type, _retention), Collections.emptyMap(), makeAuthProvider(_authProvider,
+ _authTokenUrl, _authToken, _user, _password));
+ } catch (Exception e) {
+ LOGGER.error("Got Exception while deleting Pinot Table: " + _tableName, e);
+ return false;
+ }
+ return true;
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org