You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by xi...@apache.org on 2020/01/05 08:20:17 UTC
[incubator-pinot] 01/01: Update AddTableCommand to take schema file
in order to creation pinot table in one line
This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch new-pinot-admin-cmd
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit 35d5d0a7da90440f9b0763446869a8784a3956c0
Author: Xiang Fu <fx...@gmail.com>
AuthorDate: Sat Jan 4 23:55:02 2020 -0800
Update AddTableCommand to take schema file in order to creation pinot table in one line
---
.../org/apache/pinot/tools/HybridQuickstart.java | 2 -
.../java/org/apache/pinot/tools/Quickstart.java | 2 -
.../org/apache/pinot/tools/RealtimeQuickStart.java | 2 -
.../pinot/tools/admin/PinotAdministrator.java | 1 +
.../pinot/tools/admin/command/AddTableCommand.java | 82 +++++++++++++++-------
.../tools/admin/command/GenerateDataCommand.java | 2 +-
.../tools/admin/command/QuickstartRunner.java | 11 +--
.../tools/query/comparison/ClusterStarter.java | 3 +-
8 files changed, 61 insertions(+), 44 deletions(-)
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/HybridQuickstart.java b/pinot-tools/src/main/java/org/apache/pinot/tools/HybridQuickstart.java
index 68717cb..aa33073 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/HybridQuickstart.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/HybridQuickstart.java
@@ -130,8 +130,6 @@ public class HybridQuickstart {
runner.createServerTenantWith(1, 1, "airline");
printStatus(Color.YELLOW, "***** Creating a broker tenant with name 'airline_broker' *****");
runner.createBrokerTenantWith(2, "airline_broker");
- printStatus(Color.YELLOW, "***** Adding airlineStats schema *****");
- runner.addSchema();
printStatus(Color.YELLOW, "***** Adding airlineStats offline and realtime table *****");
runner.addTable();
printStatus(Color.YELLOW, "***** Building index segment for airlineStats *****");
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/Quickstart.java b/pinot-tools/src/main/java/org/apache/pinot/tools/Quickstart.java
index d7d0bf5..9427b52 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/Quickstart.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/Quickstart.java
@@ -146,8 +146,6 @@ public class Quickstart {
printStatus(Color.CYAN, "***** Starting Zookeeper, controller, broker and server *****");
runner.startAll();
- printStatus(Color.CYAN, "***** Adding baseballStats schema *****");
- runner.addSchema();
printStatus(Color.CYAN, "***** Adding baseballStats table *****");
runner.addTable();
printStatus(Color.CYAN, "***** Building index segment for baseballStats *****");
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/RealtimeQuickStart.java b/pinot-tools/src/main/java/org/apache/pinot/tools/RealtimeQuickStart.java
index 1492355..82f76e2 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/RealtimeQuickStart.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/RealtimeQuickStart.java
@@ -84,8 +84,6 @@ public class RealtimeQuickStart {
printStatus(Color.CYAN, "***** Starting Zookeeper, controller, server and broker *****");
runner.startAll();
- printStatus(Color.CYAN, "***** Adding meetupRSVP schema *****");
- runner.addSchema();
printStatus(Color.CYAN, "***** Adding meetupRSVP table *****");
runner.addTable();
printStatus(Color.CYAN, "***** Starting meetup data stream and publishing to Kafka *****");
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/PinotAdministrator.java b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/PinotAdministrator.java
index 9684cca..5fd2ccb 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/PinotAdministrator.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/PinotAdministrator.java
@@ -97,6 +97,7 @@ public class PinotAdministrator {
@SubCommand(name = "ChangeTableState", impl = ChangeTableState.class),
@SubCommand(name = "AddTenant", impl = AddTenantCommand.class),
@SubCommand(name = "AddSchema", impl = AddSchemaCommand.class),
+ @SubCommand(name = "UpdateSchema", impl = AddSchemaCommand.class),
@SubCommand(name = "UploadSegment", impl = UploadSegmentCommand.class),
@SubCommand(name = "PostQuery", impl = PostQueryCommand.class),
@SubCommand(name = "StopProcess", impl = StopProcessCommand.class),
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/AddTableCommand.java b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/AddTableCommand.java
index d035f10..4106e47 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/AddTableCommand.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/AddTableCommand.java
@@ -20,10 +20,13 @@ package org.apache.pinot.tools.admin.command;
import com.fasterxml.jackson.databind.JsonNode;
import java.io.File;
+import java.io.FileNotFoundException;
import java.io.IOException;
-import org.apache.pinot.spi.utils.JsonUtils;
+import org.apache.pinot.common.utils.FileUploadDownloadClient;
import org.apache.pinot.common.utils.NetUtil;
import org.apache.pinot.controller.helix.ControllerRequestURLBuilder;
+import org.apache.pinot.spi.data.Schema;
+import org.apache.pinot.spi.utils.JsonUtils;
import org.apache.pinot.tools.Command;
import org.kohsuke.args4j.Option;
import org.slf4j.Logger;
@@ -37,8 +40,11 @@ import org.slf4j.LoggerFactory;
public class AddTableCommand extends AbstractBaseAdminCommand implements Command {
private static final Logger LOGGER = LoggerFactory.getLogger(AddTableCommand.class);
- @Option(name = "-filePath", required = true, metaVar = "<string>", usage = "Path to the request.json file")
- private String _filePath;
+ @Option(name = "-tableConfigFile", required = true, metaVar = "<string>", aliases = {"-tableConf", "-tableConfig", "-filePath"}, usage = "Path to table config file.")
+ private String _tableConfigFile;
+
+ @Option(name = "-schemaFile", required = false, metaVar = "<string>", aliases = {"-schema"}, usage = "Path to table schema file.")
+ private String _schemaFile = null;
@Option(name = "-controllerHost", required = false, metaVar = "<String>", usage = "host name for controller.")
private String _controllerHost;
@@ -61,29 +67,33 @@ public class AddTableCommand extends AbstractBaseAdminCommand implements Command
@Override
public String getName() {
- return "AddTable";
+ return "AddTableV2";
}
@Override
public String description() {
- return "Add table specified in the json file, to the controller.";
+ return "Create Pinot table with table config and schema.";
}
@Override
public String toString() {
- String retString = ("AddTable -filePath " + _filePath + " -controllerHost " + _controllerHost + " -controllerPort "
- + _controllerPort);
-
+ String retString =
+ ("AddTable -tableConfigFile " + _tableConfigFile + " -schemaFile " + _schemaFile + " -controllerHost "
+ + _controllerHost + " -controllerPort " + _controllerPort);
return ((_exec) ? (retString + " -exec") : retString);
}
@Override
public void cleanup() {
+ }
+ public AddTableCommand setTableConfigFile(String tableConfigFile) {
+ _tableConfigFile = tableConfigFile;
+ return this;
}
- public AddTableCommand setFilePath(String filePath) {
- _filePath = filePath;
+ public AddTableCommand setSchemaFile(String schemaFile) {
+ _schemaFile = schemaFile;
return this;
}
@@ -102,33 +112,51 @@ public class AddTableCommand extends AbstractBaseAdminCommand implements Command
return this;
}
- public boolean execute(JsonNode node)
+ public boolean sendTableCreationRequest(JsonNode node)
throws IOException {
- if (_controllerHost == null) {
- _controllerHost = NetUtil.getHostAddress();
- }
- _controllerAddress = "http://" + _controllerHost + ":" + _controllerPort;
+ String res = AbstractBaseAdminCommand
+ .sendPostRequest(ControllerRequestURLBuilder.baseUrl(_controllerAddress).forTableCreate(), node.toString());
+ LOGGER.info(res);
+ return res.contains("succesfully added");
+ }
+ @Override
+ public boolean execute()
+ throws Exception {
if (!_exec) {
LOGGER.warn("Dry Running Command: " + toString());
LOGGER.warn("Use the -exec option to actually execute the command.");
return true;
}
- LOGGER.info("Executing command: " + toString());
- String res = AbstractBaseAdminCommand
- .sendPostRequest(ControllerRequestURLBuilder.baseUrl(_controllerAddress).forTableCreate(), node.toString());
+ if (_controllerHost == null) {
+ _controllerHost = NetUtil.getHostAddress();
+ }
+ _controllerAddress = "http://" + _controllerHost + ":" + _controllerPort;
- LOGGER.info(res);
- if (res.contains("succesfully added")) {
- return true;
+ LOGGER.info("Executing command: " + toString());
+ // Backward compatible
+ if (_schemaFile != null) {
+ File schemaFile = new File(_schemaFile);
+ if (!schemaFile.exists()) {
+ throw new FileNotFoundException("schema file does not exist: " + _schemaFile);
+ }
+
+ Schema schema = Schema.fromFile(schemaFile);
+ try (FileUploadDownloadClient fileUploadDownloadClient = new FileUploadDownloadClient()) {
+ fileUploadDownloadClient.addSchema(
+ FileUploadDownloadClient.getUploadSchemaHttpURI(_controllerHost, Integer.parseInt(_controllerPort)),
+ schema.getSchemaName(), schemaFile);
+ } catch (Exception e) {
+ LOGGER.error("Got Exception to upload Pinot Schema: " + schema.getSchemaName(), e);
+ throw e;
+ }
+ }
+ File tableConfigFile = new File(_tableConfigFile);
+ if (!tableConfigFile.exists()) {
+ throw new FileNotFoundException("table config file does not exist: " + _tableConfigFile);
}
- return false;
- }
- @Override
- public boolean execute()
- throws Exception {
- return execute(JsonUtils.fileToJsonNode(new File(_filePath)));
+ return sendTableCreationRequest(JsonUtils.fileToJsonNode(tableConfigFile));
}
}
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/GenerateDataCommand.java b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/GenerateDataCommand.java
index e46dcae..19ff1ce 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/GenerateDataCommand.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/GenerateDataCommand.java
@@ -101,7 +101,7 @@ public class GenerateDataCommand extends AbstractBaseAdminCommand implements Com
@Override
public String description() {
- return "Generate random data as per the provided scema";
+ return "Generate random data as per the provided schema";
}
@Override
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/QuickstartRunner.java b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/QuickstartRunner.java
index 11c860e..549ac52 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/QuickstartRunner.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/QuickstartRunner.java
@@ -153,18 +153,11 @@ public class QuickstartRunner {
.setInstances(number).setRole(TenantRole.BROKER).setExecute(true).execute();
}
- public void addSchema()
- throws Exception {
- for (QuickstartTableRequest request : _tableRequests) {
- new AddSchemaCommand().setControllerPort(String.valueOf(_controllerPorts.get(0)))
- .setSchemaFilePath(request.getSchemaFile().getAbsolutePath()).setExecute(true).execute();
- }
- }
-
public void addTable()
throws Exception {
for (QuickstartTableRequest request : _tableRequests) {
- new AddTableCommand().setFilePath(request.getTableRequestFile().getAbsolutePath())
+ new AddTableCommand().setSchemaFile(request.getSchemaFile().getAbsolutePath())
+ .setTableConfigFile(request.getTableRequestFile().getAbsolutePath())
.setControllerPort(String.valueOf(_controllerPorts.get(0))).setExecute(true).execute();
}
}
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/query/comparison/ClusterStarter.java b/pinot-tools/src/main/java/org/apache/pinot/tools/query/comparison/ClusterStarter.java
index bc7eb7a..b2840ab 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/query/comparison/ClusterStarter.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/query/comparison/ClusterStarter.java
@@ -192,7 +192,8 @@ public class ClusterStarter {
throws Exception {
if (_tableConfigFile != null) {
AddTableCommand addTableCommand =
- new AddTableCommand().setControllerPort(_controllerPort).setFilePath(_tableConfigFile).setExecute(true);
+ new AddTableCommand().setControllerPort(_controllerPort)
+ .setSchemaFile(_schemaFileName).setTableConfigFile(_tableConfigFile).setExecute(true);
addTableCommand.execute();
return;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org