You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by xi...@apache.org on 2022/08/15 23:18:48 UTC
[pinot] branch master updated: Refactor offline quickstarts to load all the tables by default (#9206)
This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new a4e3adb736 Refactor offline quickstarts to load all the tables by default (#9206)
a4e3adb736 is described below
commit a4e3adb7365ef64b02d3fb72505b97474e3d7244
Author: Xiang Fu <xi...@gmail.com>
AuthorDate: Mon Aug 15 16:18:43 2022 -0700
Refactor offline quickstarts to load all the tables by default (#9206)
---
.../org/apache/pinot/tools/EmptyQuickstart.java | 47 +-----
.../org/apache/pinot/tools/GenericQuickstart.java | 87 +++-------
.../org/apache/pinot/tools/JoinQuickStart.java | 75 +--------
.../apache/pinot/tools/JsonIndexQuickStart.java | 56 +------
.../pinot/tools/MultistageEngineQuickStart.java | 79 +++------
.../OfflineComplexTypeHandlingQuickStart.java | 64 +-------
.../org/apache/pinot/tools/QuickStartBase.java | 182 ++++++++++++++++++---
.../java/org/apache/pinot/tools/Quickstart.java | 119 +++-----------
.../pinot/tools/TimestampIndexQuickstart.java | 75 ++-------
.../tools/admin/command/QuickStartCommand.java | 33 ++--
...hubComplexTypeEvents_offline_table_config.json} | 2 +-
.../githubComplexTypeEvents_schema.json} | 2 +-
.../ingestionJobSpec.yaml} | 10 +-
.../rawdata/githubComplexTypeEvents_data.json} | 0
.../batch/githubEvents/ingestionJobSpec.yaml | 2 +-
.../githubEvents_data.json | 0
16 files changed, 280 insertions(+), 553 deletions(-)
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/EmptyQuickstart.java b/pinot-tools/src/main/java/org/apache/pinot/tools/EmptyQuickstart.java
index 24c3fcc35e..6064a4f1a2 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/EmptyQuickstart.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/EmptyQuickstart.java
@@ -18,61 +18,20 @@
*/
package org.apache.pinot.tools;
-import java.io.File;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
-import org.apache.pinot.spi.auth.AuthProvider;
import org.apache.pinot.tools.admin.PinotAdministrator;
-import org.apache.pinot.tools.admin.command.QuickstartRunner;
-public class EmptyQuickstart extends QuickStartBase {
+public class EmptyQuickstart extends Quickstart {
@Override
public List<String> types() {
return Arrays.asList("EMPTY", "DEFAULT");
}
- public AuthProvider getAuthProvider() {
- return null;
- }
-
- public void execute()
- throws Exception {
- File quickstartTmpDir = new File(_dataDir.getAbsolutePath());
- File dataDir = new File(quickstartTmpDir, "rawdata");
- if (!dataDir.mkdirs()) {
- printStatus(Quickstart.Color.YELLOW, "***** Bootstrapping data from existing directory *****");
- } else {
- printStatus(Quickstart.Color.YELLOW, "***** Creating new data directory for fresh installation *****");
- }
-
- QuickstartRunner runner =
- new QuickstartRunner(new ArrayList<>(), 1, 1, 1, 1,
- dataDir, true, getAuthProvider(), getConfigOverrides(), _zkExternalAddress, false);
-
- if (_zkExternalAddress != null) {
- printStatus(Quickstart.Color.CYAN, "***** Starting controller, broker and server *****");
- } else {
- printStatus(Quickstart.Color.CYAN, "***** Starting Zookeeper, controller, broker and server *****");
- }
-
- runner.startAll();
-
- Runtime.getRuntime().addShutdownHook(new Thread(() -> {
- try {
- printStatus(Quickstart.Color.GREEN, "***** Shutting down empty quick start *****");
- runner.stop();
- } catch (Exception e) {
- e.printStackTrace();
- }
- }));
-
- waitForBootstrapToComplete(runner);
-
- printStatus(Quickstart.Color.YELLOW, "***** Empty quickstart setup complete *****");
- printStatus(Quickstart.Color.GREEN,
- "You can always go to http://localhost:9000 to play around in the query console");
+ public String[] getDefaultBatchTableDirectories() {
+ return new String[]{};
}
public static void main(String[] args)
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/GenericQuickstart.java b/pinot-tools/src/main/java/org/apache/pinot/tools/GenericQuickstart.java
index b0e8ac3069..62fc47d6b3 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/GenericQuickstart.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/GenericQuickstart.java
@@ -18,24 +18,16 @@
*/
package org.apache.pinot.tools;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-import java.io.File;
-import java.net.URL;
import java.util.Arrays;
import java.util.List;
-import org.apache.commons.io.FileUtils;
import org.apache.pinot.common.utils.ZkStarter;
import org.apache.pinot.spi.stream.StreamDataProvider;
import org.apache.pinot.spi.stream.StreamDataServerStartable;
-import org.apache.pinot.tools.Quickstart.Color;
import org.apache.pinot.tools.admin.command.QuickstartRunner;
import org.apache.pinot.tools.utils.KafkaStarterUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static org.apache.pinot.tools.Quickstart.prettyPrintResponse;
-
/**
* Sets up a demo Pinot cluster with 1 zookeeper, 1 controller, 1 broker and 1 server
@@ -49,29 +41,12 @@ import static org.apache.pinot.tools.Quickstart.prettyPrintResponse;
* ingestion_job_spec.json
* </code>
*/
-public class GenericQuickstart extends QuickStartBase {
+public class GenericQuickstart extends Quickstart {
private static final Logger LOGGER = LoggerFactory.getLogger(GenericQuickstart.class);
- private final File _schemaFile;
- private final File _tableConfigFile;
- private final File _tableDirectory;
- private final String _tableName;
private StreamDataServerStartable _kafkaStarter;
private ZkStarter.ZookeeperInstance _zookeeperInstance;
public GenericQuickstart() {
- this(GenericQuickstart.class.getClassLoader().getResource("examples/batch/starbucksStores").getPath(),
- "starbucksStores");
- }
-
- public GenericQuickstart(String tableDirectoryPath, String tableName) {
- _tableDirectory = new File(tableDirectoryPath);
- _tableName = tableName;
-
- if (!_tableDirectory.exists()) {
- Preconditions.checkState(_tableDirectory.mkdirs());
- }
- _schemaFile = new File(_tableDirectory, "schema.json");
- _tableConfigFile = new File(_tableDirectory, "table_config.json");
}
private void startKafka() {
@@ -91,41 +66,9 @@ public class GenericQuickstart extends QuickStartBase {
return Arrays.asList("GENERIC");
}
- public void execute()
+ @Override
+ public void runSampleQueries(QuickstartRunner runner)
throws Exception {
-
- File tempDir = new File(FileUtils.getTempDirectory(), String.valueOf(System.currentTimeMillis()));
- Preconditions.checkState(tempDir.mkdirs());
- QuickstartTableRequest request = new QuickstartTableRequest(_tableDirectory.getAbsolutePath());
- final QuickstartRunner runner =
- new QuickstartRunner(Lists.newArrayList(request), 1, 1, 1, 1, tempDir, getConfigOverrides());
-
- printStatus(Color.CYAN, "***** Starting Kafka *****");
- startKafka();
-
- printStatus(Color.CYAN, "***** Starting zookeeper, controller, server and broker *****");
- runner.startAll();
-
- printStatus(Color.CYAN, "***** Adding table *****");
- runner.bootstrapTable();
-
- printStatus(Color.CYAN, "***** Waiting for 10 seconds for a few events to get populated *****");
- Thread.sleep(10000);
-
- Runtime.getRuntime().addShutdownHook(new Thread(() -> {
- try {
- printStatus(Color.GREEN, "***** Shutting down QuickStart cluster *****");
- runner.stop();
- _kafkaStarter.stop();
- ZkStarter.stopLocalZkServer(_zookeeperInstance);
- FileUtils.deleteDirectory(_tableDirectory);
- } catch (Exception e) {
- LOGGER.error("Caught exception in shutting down QuickStart cluster", e);
- }
- }));
-
- printStatus(Color.YELLOW, "***** GenericQuickStart demo quickstart setup complete *****");
-
String q1 = "select count(*) from starbucksStores limit 0";
printStatus(Color.YELLOW, "Total number of documents in the table");
printStatus(Color.CYAN, "Query : " + q1);
@@ -145,17 +88,27 @@ public class GenericQuickstart extends QuickStartBase {
printStatus(Color.CYAN, "Query : " + q3);
printStatus(Color.YELLOW, prettyPrintResponse(runner.runQuery(q3)));
printStatus(Color.GREEN, "***************************************************");
+ }
+
+ public void execute()
+ throws Exception {
+ printStatus(Color.CYAN, "***** Starting Kafka *****");
+ startKafka();
- printStatus(Color.GREEN, "You can always go to http://localhost:9000 to play around in the query console");
+ Runtime.getRuntime().addShutdownHook(new Thread(() -> {
+ try {
+ printStatus(Color.GREEN, "***** Shutting down QuickStart cluster *****");
+ _kafkaStarter.stop();
+ ZkStarter.stopLocalZkServer(_zookeeperInstance);
+ } catch (Exception e) {
+ LOGGER.error("Caught exception in shutting down QuickStart cluster", e);
+ }
+ }));
+ super.execute();
}
public static void main(String[] args)
throws Exception {
- ClassLoader classLoader = GenericQuickstart.class.getClassLoader();
- URL resource = classLoader.getResource("examples/batch/starbucksStores");
- String tableDirectoryPath = resource.getPath();
-
- GenericQuickstart quickstart = new GenericQuickstart(tableDirectoryPath, "starbucksStores");
- quickstart.execute();
+ new GenericQuickstart().execute();
}
}
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/JoinQuickStart.java b/pinot-tools/src/main/java/org/apache/pinot/tools/JoinQuickStart.java
index c6d9c3bdd0..6a22dde415 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/JoinQuickStart.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/JoinQuickStart.java
@@ -18,87 +18,24 @@
*/
package org.apache.pinot.tools;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-import java.io.File;
-import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
-import org.apache.commons.io.FileUtils;
import org.apache.pinot.tools.admin.PinotAdministrator;
import org.apache.pinot.tools.admin.command.QuickstartRunner;
-import static org.apache.pinot.tools.Quickstart.prettyPrintResponse;
-
-public class JoinQuickStart extends QuickStartBase {
+public class JoinQuickStart extends Quickstart {
@Override
public List<String> types() {
return Collections.singletonList("JOIN");
}
- public void execute()
+ @Override
+ public void runSampleQueries(QuickstartRunner runner)
throws Exception {
- File quickstartTmpDir = new File(_dataDir, String.valueOf(System.currentTimeMillis()));
-
- // Baseball stat table
- File baseBallStatsBaseDir = new File(quickstartTmpDir, "baseballStats");
- File schemaFile = new File(baseBallStatsBaseDir, "baseballStats_schema.json");
- File tableConfigFile = new File(baseBallStatsBaseDir, "baseballStats_offline_table_config.json");
- File ingestionJobSpecFile = new File(baseBallStatsBaseDir, "ingestionJobSpec.yaml");
- ClassLoader classLoader = Quickstart.class.getClassLoader();
- URL resource = classLoader.getResource("examples/batch/baseballStats/baseballStats_schema.json");
- Preconditions.checkNotNull(resource);
- FileUtils.copyURLToFile(resource, schemaFile);
- resource = classLoader.getResource("examples/batch/baseballStats/baseballStats_offline_table_config.json");
- Preconditions.checkNotNull(resource);
- FileUtils.copyURLToFile(resource, tableConfigFile);
- resource = classLoader.getResource("examples/batch/baseballStats/ingestionJobSpec.yaml");
- Preconditions.checkNotNull(resource);
- FileUtils.copyURLToFile(resource, ingestionJobSpecFile);
- QuickstartTableRequest request = new QuickstartTableRequest(baseBallStatsBaseDir.getAbsolutePath());
-
- // Baseball teams dim table
- File dimBaseballTeamsBaseDir = new File(quickstartTmpDir, "dimBaseballTeams");
- schemaFile = new File(dimBaseballTeamsBaseDir, "dimBaseballTeams_schema.json");
- tableConfigFile = new File(dimBaseballTeamsBaseDir, "dimBaseballTeams_offline_table_config.json");
- ingestionJobSpecFile = new File(dimBaseballTeamsBaseDir, "ingestionJobSpec.yaml");
- classLoader = Quickstart.class.getClassLoader();
- resource = classLoader.getResource("examples/batch/dimBaseballTeams/dimBaseballTeams_schema.json");
- Preconditions.checkNotNull(resource);
- FileUtils.copyURLToFile(resource, schemaFile);
- resource = classLoader.getResource("examples/batch/dimBaseballTeams/dimBaseballTeams_offline_table_config.json");
- Preconditions.checkNotNull(resource);
- FileUtils.copyURLToFile(resource, tableConfigFile);
- resource = classLoader.getResource("examples/batch/dimBaseballTeams/ingestionJobSpec.yaml");
- Preconditions.checkNotNull(resource);
- FileUtils.copyURLToFile(resource, ingestionJobSpecFile);
- QuickstartTableRequest dimTableRequest = new QuickstartTableRequest(dimBaseballTeamsBaseDir.getAbsolutePath());
-
- File tempDir = new File(quickstartTmpDir, "tmp");
- FileUtils.forceMkdir(tempDir);
- QuickstartRunner runner =
- new QuickstartRunner(Lists.newArrayList(request, dimTableRequest), 1, 1, 3, 1, tempDir, getConfigOverrides());
-
- printStatus(Quickstart.Color.CYAN, "***** Starting Zookeeper, controller, broker and server *****");
- runner.startAll();
- Runtime.getRuntime().addShutdownHook(new Thread(() -> {
- try {
- printStatus(Quickstart.Color.GREEN, "***** Shutting down offline quick start *****");
- runner.stop();
- FileUtils.deleteDirectory(quickstartTmpDir);
- } catch (Exception e) {
- e.printStackTrace();
- }
- }));
- printStatus(Quickstart.Color.CYAN, "***** Bootstrap baseballStats table *****");
- runner.bootstrapTable();
-
- waitForBootstrapToComplete(null);
-
printStatus(Quickstart.Color.YELLOW, "***** Offline quickstart setup complete *****");
String q1 = "select count(*) from baseballStats limit 1";
@@ -120,9 +57,11 @@ public class JoinQuickStart extends QuickStartBase {
printStatus(Quickstart.Color.CYAN, "Query : " + q3);
printStatus(Quickstart.Color.YELLOW, prettyPrintResponse(runner.runQuery(q3)));
printStatus(Quickstart.Color.GREEN, "***************************************************");
+ }
- printStatus(Quickstart.Color.GREEN,
- "You can always go to http://localhost:9000 to play around in the query console");
+ @Override
+ protected int getNumQuickstartRunnerServers() {
+ return 3;
}
public static void main(String[] args)
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/JsonIndexQuickStart.java b/pinot-tools/src/main/java/org/apache/pinot/tools/JsonIndexQuickStart.java
index 035bee90e4..debc15835f 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/JsonIndexQuickStart.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/JsonIndexQuickStart.java
@@ -18,81 +18,31 @@
*/
package org.apache.pinot.tools;
-import com.google.common.base.Preconditions;
-import java.io.File;
-import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Collections;
import java.util.List;
-import org.apache.commons.io.FileUtils;
-import org.apache.pinot.tools.Quickstart.Color;
import org.apache.pinot.tools.admin.PinotAdministrator;
import org.apache.pinot.tools.admin.command.QuickstartRunner;
-import static org.apache.pinot.tools.Quickstart.prettyPrintResponse;
-
-public class JsonIndexQuickStart extends QuickStartBase {
+public class JsonIndexQuickStart extends Quickstart {
@Override
public List<String> types() {
return Arrays.asList("OFFLINE_JSON_INDEX", "OFFLINE-JSON-INDEX", "BATCH_JSON_INDEX", "BATCH-JSON-INDEX");
}
- public void execute()
+ @Override
+ public void runSampleQueries(QuickstartRunner runner)
throws Exception {
- File quickstartTmpDir = new File(_dataDir, String.valueOf(System.currentTimeMillis()));
- File baseDir = new File(quickstartTmpDir, "githubEvents");
- File dataDir = new File(quickstartTmpDir, "rawdata");
- Preconditions.checkState(dataDir.mkdirs());
-
- File schemaFile = new File(baseDir, "githubEvents_schema.json");
- File tableConfigFile = new File(baseDir, "githubEvents_offline_table_config.json");
- File ingestionJobSpecFile = new File(baseDir, "ingestionJobSpec.yaml");
-
- ClassLoader classLoader = JsonIndexQuickStart.class.getClassLoader();
- URL resource = classLoader.getResource("examples/batch/githubEvents/githubEvents_offline_table_config.json");
- Preconditions.checkNotNull(resource);
- FileUtils.copyURLToFile(resource, tableConfigFile);
- resource = classLoader.getResource("examples/batch/githubEvents/githubEvents_schema.json");
- Preconditions.checkNotNull(resource);
- FileUtils.copyURLToFile(resource, schemaFile);
- resource = classLoader.getResource("examples/batch/githubEvents/ingestionJobSpec.yaml");
- Preconditions.checkNotNull(resource);
- FileUtils.copyURLToFile(resource, ingestionJobSpecFile);
-
- QuickstartTableRequest request = new QuickstartTableRequest(baseDir.getAbsolutePath());
- final QuickstartRunner runner =
- new QuickstartRunner(Collections.singletonList(request), 1, 1, 1, 1, dataDir, getConfigOverrides());
-
- printStatus(Color.CYAN, "***** Starting Zookeeper, controller, broker and server *****");
- runner.startAll();
- Runtime.getRuntime().addShutdownHook(new Thread(() -> {
- try {
- printStatus(Color.GREEN, "***** Shutting down offline quick start *****");
- runner.stop();
- FileUtils.deleteDirectory(quickstartTmpDir);
- } catch (Exception e) {
- e.printStackTrace();
- }
- }));
- printStatus(Color.CYAN, "***** Bootstrap githubEvents table *****");
- runner.bootstrapTable();
-
- waitForBootstrapToComplete(null);
-
printStatus(Color.YELLOW, "***** Offline json-index quickstart setup complete *****");
-
String q1 =
"select json_extract_scalar(repo, '$.name', 'STRING'), count(*) from githubEvents where json_match(actor, "
+ "'\"$.login\"=''LombiqBot''') group by 1 order by 2 desc limit 10";
printStatus(Color.YELLOW, "Most contributed repos by 'LombiqBot'");
printStatus(Color.CYAN, "Query : " + q1);
printStatus(Color.YELLOW, prettyPrintResponse(runner.runQuery(q1)));
-
printStatus(Color.GREEN, "***************************************************");
- printStatus(Color.GREEN, "You can always go to http://localhost:9000 to play around in the query console");
}
public static void main(String[] args)
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/MultistageEngineQuickStart.java b/pinot-tools/src/main/java/org/apache/pinot/tools/MultistageEngineQuickStart.java
index 1bbb87bca5..2ff4928b03 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/MultistageEngineQuickStart.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/MultistageEngineQuickStart.java
@@ -18,25 +18,18 @@
*/
package org.apache.pinot.tools;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-import java.io.File;
-import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import org.apache.commons.io.FileUtils;
import org.apache.pinot.spi.utils.CommonConstants;
import org.apache.pinot.tools.admin.PinotAdministrator;
import org.apache.pinot.tools.admin.command.QuickstartRunner;
-import static org.apache.pinot.tools.Quickstart.prettyPrintResponse;
-
-public class MultistageEngineQuickStart extends QuickStartBase {
+public class MultistageEngineQuickStart extends Quickstart {
@Override
public List<String> types() {
@@ -44,59 +37,12 @@ public class MultistageEngineQuickStart extends QuickStartBase {
}
@Override
- public Map<String, Object> getConfigOverrides() {
- Map<String, Object> overrides = new HashMap<>(super.getConfigOverrides());
- overrides.put("pinot.multistage.engine.enabled", "true");
- overrides.put("pinot.server.instance.currentDataTableVersion", 4);
- return overrides;
- }
-
- public void execute()
+ public void runSampleQueries(QuickstartRunner runner)
throws Exception {
- File quickstartTmpDir = new File(_dataDir, String.valueOf(System.currentTimeMillis()));
-
- // Baseball stat table
- File baseBallStatsBaseDir = new File(quickstartTmpDir, "baseballStats");
- File schemaFile = new File(baseBallStatsBaseDir, "baseballStats_schema.json");
- File tableConfigFile = new File(baseBallStatsBaseDir, "baseballStats_offline_table_config.json");
- File ingestionJobSpecFile = new File(baseBallStatsBaseDir, "ingestionJobSpec.yaml");
- ClassLoader classLoader = Quickstart.class.getClassLoader();
- URL resource = classLoader.getResource("examples/batch/baseballStats/baseballStats_schema.json");
- Preconditions.checkNotNull(resource);
- FileUtils.copyURLToFile(resource, schemaFile);
- resource = classLoader.getResource("examples/batch/baseballStats/baseballStats_offline_table_config.json");
- Preconditions.checkNotNull(resource);
- FileUtils.copyURLToFile(resource, tableConfigFile);
- resource = classLoader.getResource("examples/batch/baseballStats/ingestionJobSpec.yaml");
- Preconditions.checkNotNull(resource);
- FileUtils.copyURLToFile(resource, ingestionJobSpecFile);
- QuickstartTableRequest request = new QuickstartTableRequest(baseBallStatsBaseDir.getAbsolutePath());
-
- File tempDir = new File(quickstartTmpDir, "tmp");
- FileUtils.forceMkdir(tempDir);
- QuickstartRunner runner =
- new QuickstartRunner(Lists.newArrayList(request), 1, 1, 1, 1, tempDir, getConfigOverrides());
-
- printStatus(Quickstart.Color.CYAN, "***** Starting Zookeeper, controller, broker and server *****");
- runner.startAll();
- Runtime.getRuntime().addShutdownHook(new Thread(() -> {
- try {
- printStatus(Quickstart.Color.GREEN, "***** Shutting down offline quick start *****");
- runner.stop();
- FileUtils.deleteDirectory(quickstartTmpDir);
- } catch (Exception e) {
- e.printStackTrace();
- }
- }));
- printStatus(Quickstart.Color.CYAN, "***** Bootstrap baseballStats table *****");
- runner.bootstrapTable();
-
- waitForBootstrapToComplete(null);
+ printStatus(Quickstart.Color.YELLOW, "***** Multi-stage engine quickstart setup complete *****");
Map<String, String> queryOptions = Collections.singletonMap("queryOptions",
CommonConstants.Broker.Request.QueryOptionKey.USE_MULTISTAGE_ENGINE + "=true");
-
- printStatus(Quickstart.Color.YELLOW, "***** Multi-stage engine quickstart setup complete *****");
String q1 = "SELECT count(*) FROM baseballStats_OFFLINE";
printStatus(Quickstart.Color.YELLOW, "Total number of documents in the table");
printStatus(Quickstart.Color.CYAN, "Query : " + q1);
@@ -112,11 +58,26 @@ public class MultistageEngineQuickStart extends QuickStartBase {
printStatus(Quickstart.Color.YELLOW, prettyPrintResponse(runner.runQuery(q2, queryOptions)));
printStatus(Quickstart.Color.GREEN, "***************************************************");
+ String q3 = "SELECT a.playerName, a.teamID, b.teamName \n"
+ + "FROM baseballStats_OFFLINE AS a\n"
+ + "JOIN dimBaseballTeams_OFFLINE AS b\n"
+ + "ON a.teamID = b.teamID";
+ printStatus(Quickstart.Color.YELLOW, "Baseball Stats with joined team names");
+ printStatus(Quickstart.Color.CYAN, "Query : " + q3);
+ printStatus(Quickstart.Color.YELLOW, prettyPrintResponse(runner.runQuery(q3, queryOptions)));
+ printStatus(Quickstart.Color.GREEN, "***************************************************");
+
printStatus(Quickstart.Color.GREEN, "***************************************************");
printStatus(Quickstart.Color.YELLOW, "Example query run completed.");
printStatus(Quickstart.Color.GREEN, "***************************************************");
- printStatus(Quickstart.Color.GREEN,
- "You can always go to http://localhost:9000 to play around in the query console");
+ }
+
+ @Override
+ public Map<String, Object> getConfigOverrides() {
+ Map<String, Object> overrides = new HashMap<>(super.getConfigOverrides());
+ overrides.put("pinot.multistage.engine.enabled", "true");
+ overrides.put("pinot.server.instance.currentDataTableVersion", 4);
+ return overrides;
}
public static void main(String[] args)
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/OfflineComplexTypeHandlingQuickStart.java b/pinot-tools/src/main/java/org/apache/pinot/tools/OfflineComplexTypeHandlingQuickStart.java
index c5cd73c819..168f496067 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/OfflineComplexTypeHandlingQuickStart.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/OfflineComplexTypeHandlingQuickStart.java
@@ -18,81 +18,29 @@
*/
package org.apache.pinot.tools;
-import com.google.common.base.Preconditions;
-import java.io.File;
-import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Collections;
import java.util.List;
-import org.apache.commons.io.FileUtils;
-import org.apache.pinot.tools.Quickstart.Color;
import org.apache.pinot.tools.admin.PinotAdministrator;
import org.apache.pinot.tools.admin.command.QuickstartRunner;
-import static org.apache.pinot.tools.Quickstart.prettyPrintResponse;
-
-public class OfflineComplexTypeHandlingQuickStart extends QuickStartBase {
+public class OfflineComplexTypeHandlingQuickStart extends Quickstart {
@Override
public List<String> types() {
- return Arrays.asList("OFFLINE_COMPLEX_TYPE", "OFFLINE-COMPLEX-TYPE", "BATCH_COMPLEX_TYPE", "BATCH-COMPLEX-TYPE");
+ return Arrays.asList("OFFLINE_COMPLEX_TYPE", "OFFLINE-COMPLEX-TYPE", "BATCH_COMPLEX_TYPE", "BATCH-COMPLEX-TYPE");
}
- public void execute()
+ @Override
+ public void runSampleQueries(QuickstartRunner runner)
throws Exception {
- File quickstartTmpDir = new File(_dataDir, String.valueOf(System.currentTimeMillis()));
- File baseDir = new File(quickstartTmpDir, "githubEvents");
- File dataDir = new File(quickstartTmpDir, "rawdata");
- Preconditions.checkState(dataDir.mkdirs());
-
- File schemaFile = new File(baseDir, "githubEvents_schema.json");
- File tableConfigFile = new File(baseDir, "githubEvents_offline_table_config.json");
- File ingestionJobSpecFile = new File(baseDir, "ingestionJobSpec.yaml");
-
- ClassLoader classLoader = OfflineComplexTypeHandlingQuickStart.class.getClassLoader();
- URL resource = classLoader
- .getResource("examples/batch/githubEvents/githubEvents_offline_complexTypeHandling_table_config.json");
- Preconditions.checkNotNull(resource);
- FileUtils.copyURLToFile(resource, tableConfigFile);
- // TODO: add all columns of the flattened fields after the schema inference
- resource =
- classLoader.getResource("examples/batch/githubEvents/githubEvents_offline_complexTypeHandling_schema.json");
- Preconditions.checkNotNull(resource);
- FileUtils.copyURLToFile(resource, schemaFile);
- resource = classLoader.getResource("examples/batch/githubEvents/ingestionJobComplexTypeHandlingSpec.yaml");
- Preconditions.checkNotNull(resource);
- FileUtils.copyURLToFile(resource, ingestionJobSpecFile);
-
- QuickstartTableRequest request = new QuickstartTableRequest(baseDir.getAbsolutePath());
- final QuickstartRunner runner =
- new QuickstartRunner(Collections.singletonList(request), 1, 1, 1, 1, dataDir, getConfigOverrides());
-
- printStatus(Color.CYAN, "***** Starting Zookeeper, controller, broker and server *****");
- runner.startAll();
- Runtime.getRuntime().addShutdownHook(new Thread(() -> {
- try {
- printStatus(Color.GREEN, "***** Shutting down offline quick start *****");
- runner.stop();
- FileUtils.deleteDirectory(quickstartTmpDir);
- } catch (Exception e) {
- e.printStackTrace();
- }
- }));
- printStatus(Color.CYAN, "***** Bootstrap githubEvents table *****");
- runner.bootstrapTable();
-
- waitForBootstrapToComplete(null);
-
printStatus(Color.YELLOW, "***** Offline complex-type-handling quickstart setup complete *****");
-
String q1 =
- "select id, \"payload.commits.author.name\", \"payload.commits.author.email\" from githubEvents limit 10";
+ "select id, \"payload.commits.author.name\", \"payload.commits.author.email\" from githubComplexTypeEvents "
+ + "limit 10";
printStatus(Color.CYAN, "Query : " + q1);
printStatus(Color.YELLOW, prettyPrintResponse(runner.runQuery(q1)));
-
printStatus(Color.GREEN, "***************************************************");
- printStatus(Color.GREEN, "You can always go to http://localhost:9000 to play around in the query console");
}
public static void main(String[] args)
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/QuickStartBase.java b/pinot-tools/src/main/java/org/apache/pinot/tools/QuickStartBase.java
index d58e38bfc8..225aeb40fd 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/QuickStartBase.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/QuickStartBase.java
@@ -18,21 +18,53 @@
*/
package org.apache.pinot.tools;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
import java.io.File;
+import java.io.IOException;
+import java.net.URL;
import java.nio.file.Paths;
+import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.pinot.tools.admin.command.QuickstartRunner;
+import org.apache.pinot.tools.utils.JarUtils;
import org.apache.pinot.tools.utils.PinotConfigUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+/**
+ * Assuming that database name is DBNAME, bootstrap path must have the file structure specified below to properly
+ * load the table:
+ * DBNAME
+ * ├── ingestionJobSpec.yaml
+ * ├── rawdata
+ * │ └── DBNAME_data.csv
+ * ├── DBNAME_offline_table_config.json
+ * └── DBNAME_schema.json
+ *
+ */
public abstract class QuickStartBase {
+ private static final Logger LOGGER = LoggerFactory.getLogger(QuickStartBase.class);
+ private static final String TAB = "\t\t";
+ private static final String NEW_LINE = "\n";
+
+ protected static final String[] DEFAULT_OFFLINE_TABLE_DIRECTORIES = new String[]{
+ "examples/batch/airlineStats",
+ "examples/minions/batch/baseballStats",
+ "examples/batch/dimBaseballTeams",
+ "examples/batch/starbucksStores",
+ "examples/batch/githubEvents",
+ "examples/batch/githubComplexTypeEvents"
+ };
+
protected File _dataDir = FileUtils.getTempDirectory();
- protected String _bootstrapDataDir;
+ protected String[] _bootstrapDataDirs;
protected String _zkExternalAddress;
protected String _configFilePath;
@@ -41,35 +73,19 @@ public abstract class QuickStartBase {
return this;
}
- public QuickStartBase setBootstrapDataDir(String bootstrapDataDir) {
- _bootstrapDataDir = bootstrapDataDir;
+ public QuickStartBase setBootstrapDataDirs(String[] bootstrapDataDirs) {
+ _bootstrapDataDirs = bootstrapDataDirs;
return this;
}
- /**
- * Assuming that database name is DBNAME, bootstrap path must have the file structure specified below to properly
- * load the table:
- * DBNAME
- * ├── ingestionJobSpec.yaml
- * ├── rawdata
- * │ └── DBNAME_data.csv
- * ├── DBNAME_offline_table_config.json
- * └── DBNAME_schema.json
- *
- * @return bootstrap path if specified by command line argument -bootstrapTableDir; otherwise, default.
- */
- public String getBootstrapDataDir(String bootstrapDataDir) {
- return _bootstrapDataDir != null ? _bootstrapDataDir : bootstrapDataDir;
- }
-
/** @return Table name if specified by command line argument -bootstrapTableDir; otherwise, default. */
public String getTableName(String bootstrapDataDir) {
- return Paths.get(getBootstrapDataDir(bootstrapDataDir)).getFileName().toString();
+ return Paths.get(bootstrapDataDir).getFileName().toString();
}
/** @return true if bootstrapTableDir is not specified by command line argument -bootstrapTableDir, else false.*/
public boolean useDefaultBootstrapTableDir() {
- return _bootstrapDataDir == null;
+ return _bootstrapDataDirs == null;
}
public QuickStartBase setZkExternalAddress(String zkExternalAddress) {
@@ -84,6 +100,10 @@ public abstract class QuickStartBase {
public abstract List<String> types();
+ public void runSampleQueries(QuickstartRunner runner)
+ throws Exception {
+ }
+
protected void waitForBootstrapToComplete(QuickstartRunner runner)
throws Exception {
QuickStartBase.printStatus(Quickstart.Color.CYAN,
@@ -98,6 +118,99 @@ public abstract class QuickStartBase {
public abstract void execute()
throws Exception;
+ protected List<QuickstartTableRequest> bootstrapOfflineTableDirectories(File quickstartTmpDir)
+ throws IOException {
+ List<QuickstartTableRequest> quickstartTableRequests = new ArrayList<>();
+ for (String directory : getDefaultBatchTableDirectories()) {
+ String tableName = getTableName(directory);
+ File baseDir = new File(quickstartTmpDir, tableName);
+ File dataDir = new File(baseDir, "rawdata");
+ Preconditions.checkState(dataDir.mkdirs());
+ if (useDefaultBootstrapTableDir()) {
+ copyResourceTableToTmpDirectory(directory, tableName, baseDir, dataDir, false);
+ } else {
+ copyFilesystemTableToTmpDirectory(directory, tableName, baseDir);
+ }
+ quickstartTableRequests.add(new QuickstartTableRequest(baseDir.getAbsolutePath()));
+ }
+ return quickstartTableRequests;
+ }
+
+ private static void copyResourceTableToTmpDirectory(String sourcePath, String tableName, File baseDir, File dataDir,
+ boolean isStreamTable)
+ throws IOException {
+ ClassLoader classLoader = Quickstart.class.getClassLoader();
+ // Copy schema
+ URL resource = classLoader.getResource(sourcePath + File.separator + tableName + "_schema.json");
+ Preconditions.checkNotNull(resource, "Missing schema json file for table - " + tableName);
+ File schemaFile = new File(baseDir, tableName + "_schema.json");
+ FileUtils.copyURLToFile(resource, schemaFile);
+
+ // Copy table config
+ String tableConfigFileSuffix = isStreamTable ? "_realtime_table_config.json" : "_offline_table_config.json";
+ File tableConfigFile = new File(baseDir, tableName + tableConfigFileSuffix);
+ String sourceTableConfig = sourcePath + File.separator + tableName + tableConfigFileSuffix;
+ resource = classLoader.getResource(sourceTableConfig);
+ Preconditions.checkNotNull(resource, "Missing table config file for table - " + tableName);
+ FileUtils.copyURLToFile(resource, tableConfigFile);
+
+ // Copy raw data
+ String sourceRawDataPath = sourcePath + File.separator + "rawdata";
+ resource = classLoader.getResource(sourceRawDataPath);
+ if (resource != null) {
+ File rawDataDir = new File(resource.getFile());
+ if (rawDataDir.isDirectory()) {
+ // Copy the directory from `pinot-tools/src/main/resources/examples` directory. This code path is used for
+ // running Quickstart inside IDE, `ClassLoader.getResource()` should source it at build directory,
+ // e.g. `/pinot-tools/target/classes/examples/batch/airlineStats/rawdata`
+ FileUtils.copyDirectory(rawDataDir, dataDir);
+ } else {
+ // Copy the directory recursively from a jar file. This code path is used for running Quickstart using
+ // pinot-admin script. The `ClassLoader.getResource()` should found the resources in the jar file then
+ // decompress it, e.g. `lib/pinot-all-jar-with-dependencies.jar!/examples/batch/airlineStats/rawdata`
+ String[] jarPathSplits = resource.toString().split("!/", 2);
+ JarUtils.copyResourcesToDirectory(jarPathSplits[0], jarPathSplits[1], dataDir.getAbsolutePath());
+ }
+ } else {
+ LOGGER.warn("Not found rawdata directory for table {} from {}", tableName, sourceRawDataPath);
+ }
+
+ if (!isStreamTable) {
+ // Copy ingestion job spec file
+ resource = classLoader.getResource(sourcePath + File.separator + "ingestionJobSpec.yaml");
+ if (resource != null) {
+ File ingestionJobSpecFile = new File(baseDir, "ingestionJobSpec.yaml");
+ FileUtils.copyURLToFile(resource, ingestionJobSpecFile);
+ }
+ }
+ }
+
+ private static void copyFilesystemTableToTmpDirectory(String sourcePath, String tableName, File baseDir)
+ throws IOException {
+ File fileDb = new File(sourcePath);
+
+ if (!fileDb.exists() || !fileDb.isDirectory()) {
+ throw new RuntimeException("Directory " + fileDb.getAbsolutePath() + " not found.");
+ }
+
+ File schemaFile = new File(fileDb, tableName + "_schema.json");
+ if (!schemaFile.exists()) {
+ throw new RuntimeException("Schema file " + schemaFile.getAbsolutePath() + " not found.");
+ }
+
+ File tableFile = new File(fileDb, tableName + "_offline_table_config.json");
+ if (!tableFile.exists()) {
+ throw new RuntimeException("Table table " + tableFile.getAbsolutePath() + " not found.");
+ }
+
+ File data = new File(fileDb, "rawdata" + File.separator + tableName + "_data.csv");
+ if (!data.exists()) {
+ throw new RuntimeException(("Data file " + data.getAbsolutePath() + " not found. "));
+ }
+
+ FileUtils.copyDirectory(fileDb, baseDir);
+ }
+
protected Map<String, Object> getConfigOverrides() {
try {
return StringUtils.isEmpty(_configFilePath) ? ImmutableMap.of()
@@ -106,4 +219,31 @@ public abstract class QuickStartBase {
throw new RuntimeException(e);
}
}
+
+ protected String[] getDefaultBatchTableDirectories() {
+ return DEFAULT_OFFLINE_TABLE_DIRECTORIES;
+ }
+
+ public static String prettyPrintResponse(JsonNode response) {
+ StringBuilder responseBuilder = new StringBuilder();
+
+ // Sql Results
+ if (response.has("resultTable")) {
+ JsonNode columns = response.get("resultTable").get("dataSchema").get("columnNames");
+ int numColumns = columns.size();
+ for (int i = 0; i < numColumns; i++) {
+ responseBuilder.append(columns.get(i).asText()).append(TAB);
+ }
+ responseBuilder.append(NEW_LINE);
+ JsonNode rows = response.get("resultTable").get("rows");
+ for (int i = 0; i < rows.size(); i++) {
+ JsonNode row = rows.get(i);
+ for (int j = 0; j < numColumns; j++) {
+ responseBuilder.append(row.get(j).asText()).append(TAB);
+ }
+ responseBuilder.append(NEW_LINE);
+ }
+ }
+ return responseBuilder.toString();
+ }
}
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/Quickstart.java b/pinot-tools/src/main/java/org/apache/pinot/tools/Quickstart.java
index d55ab66427..9dfa855a79 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/Quickstart.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/Quickstart.java
@@ -18,31 +18,27 @@
*/
package org.apache.pinot.tools;
-import com.fasterxml.jackson.databind.JsonNode;
import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
import java.io.File;
-import java.io.IOException;
-import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.io.FileUtils;
import org.apache.pinot.spi.auth.AuthProvider;
import org.apache.pinot.tools.admin.PinotAdministrator;
import org.apache.pinot.tools.admin.command.QuickstartRunner;
+/**
+ * The basic Batch/Offline Quickstart.
+ */
public class Quickstart extends QuickStartBase {
@Override
public List<String> types() {
return Arrays.asList("OFFLINE", "BATCH");
}
- private static final String TAB = "\t\t";
- private static final String NEW_LINE = "\n";
- private static final String DEFAULT_BOOTSTRAP_DIRECTORY = "examples/minions/batch/baseballStats";
-
public enum Color {
RESET("\u001B[0m"), GREEN("\u001B[32m"), YELLOW("\u001B[33m"), CYAN("\u001B[36m");
@@ -61,49 +57,18 @@ public class Quickstart extends QuickStartBase {
return null;
}
- public static String prettyPrintResponse(JsonNode response) {
- StringBuilder responseBuilder = new StringBuilder();
-
- // Sql Results
- if (response.has("resultTable")) {
- JsonNode columns = response.get("resultTable").get("dataSchema").get("columnNames");
- int numColumns = columns.size();
- for (int i = 0; i < numColumns; i++) {
- responseBuilder.append(columns.get(i).asText()).append(TAB);
- }
- responseBuilder.append(NEW_LINE);
- JsonNode rows = response.get("resultTable").get("rows");
- for (int i = 0; i < rows.size(); i++) {
- JsonNode row = rows.get(i);
- for (int j = 0; j < numColumns; j++) {
- responseBuilder.append(row.get(j).asText()).append(TAB);
- }
- responseBuilder.append(NEW_LINE);
- }
- }
- return responseBuilder.toString();
- }
-
public void execute()
throws Exception {
- String tableName = getTableName(DEFAULT_BOOTSTRAP_DIRECTORY);
File quickstartTmpDir = new File(_dataDir, String.valueOf(System.currentTimeMillis()));
- File baseDir = new File(quickstartTmpDir, tableName);
- File dataDir = new File(baseDir, "rawdata");
- Preconditions.checkState(dataDir.mkdirs());
-
- if (useDefaultBootstrapTableDir()) {
- copyResourceTableToTmpDirectory(getBootstrapDataDir(DEFAULT_BOOTSTRAP_DIRECTORY), tableName, baseDir, dataDir);
- } else {
- copyFilesystemTableToTmpDirectory(getBootstrapDataDir(DEFAULT_BOOTSTRAP_DIRECTORY), tableName, baseDir);
- }
+ File quickstartRunnerDir = new File(quickstartTmpDir, "quickstart");
+ Preconditions.checkState(quickstartRunnerDir.mkdirs());
+ List<QuickstartTableRequest> quickstartTableRequests = bootstrapOfflineTableDirectories(quickstartTmpDir);
- QuickstartTableRequest request = new QuickstartTableRequest(baseDir.getAbsolutePath());
QuickstartRunner runner =
- new QuickstartRunner(Lists.newArrayList(request), 1, 1, 1, 1, dataDir, true, getAuthProvider(),
- getConfigOverrides(), null, true);
+ new QuickstartRunner(quickstartTableRequests, 1, 1, getNumQuickstartRunnerServers(), 1, quickstartRunnerDir,
+ true, getAuthProvider(), getConfigOverrides(), null, true);
- printStatus(Color.CYAN, "***** Starting Zookeeper, controller, broker and server *****");
+ printStatus(Color.CYAN, "***** Starting Zookeeper, controller, broker, server and minion *****");
runner.startAll();
Runtime.getRuntime().addShutdownHook(new Thread(() -> {
try {
@@ -114,14 +79,16 @@ public class Quickstart extends QuickStartBase {
e.printStackTrace();
}
}));
- printStatus(Color.CYAN, "***** Bootstrap " + tableName + " table *****");
- runner.bootstrapTable();
- waitForBootstrapToComplete(runner);
+ if (!CollectionUtils.isEmpty(quickstartTableRequests)) {
+ printStatus(Color.CYAN, "***** Bootstrap tables *****");
+ runner.bootstrapTable();
+ waitForBootstrapToComplete(runner);
+ }
printStatus(Color.YELLOW, "***** Offline quickstart setup complete *****");
- if (useDefaultBootstrapTableDir()) {
+ if (useDefaultBootstrapTableDir() && !CollectionUtils.isEmpty(quickstartTableRequests)) {
// Quickstart is using the default baseballStats sample table, so run sample queries.
runSampleQueries(runner);
}
@@ -129,58 +96,12 @@ public class Quickstart extends QuickStartBase {
printStatus(Color.GREEN, "You can always go to http://localhost:9000 to play around in the query console");
}
- private static void copyResourceTableToTmpDirectory(String sourcePath, String tableName, File baseDir, File dataDir)
- throws IOException {
-
- File schemaFile = new File(baseDir, tableName + "_schema.json");
- File tableConfigFile = new File(baseDir, tableName + "_offline_table_config.json");
- File ingestionJobSpecFile = new File(baseDir, "ingestionJobSpec.yaml");
- File dataFile = new File(dataDir, tableName + "_data.csv");
-
- ClassLoader classLoader = Quickstart.class.getClassLoader();
- URL resource = classLoader.getResource(sourcePath + File.separator + tableName + "_schema.json");
- com.google.common.base.Preconditions.checkNotNull(resource);
- FileUtils.copyURLToFile(resource, schemaFile);
- resource =
- classLoader.getResource(sourcePath + File.separator + "rawdata" + File.separator + tableName + "_data.csv");
- com.google.common.base.Preconditions.checkNotNull(resource);
- FileUtils.copyURLToFile(resource, dataFile);
- resource = classLoader.getResource(sourcePath + File.separator + "ingestionJobSpec.yaml");
- if (resource != null) {
- FileUtils.copyURLToFile(resource, ingestionJobSpecFile);
- }
- resource = classLoader.getResource(sourcePath + File.separator + tableName + "_offline_table_config.json");
- com.google.common.base.Preconditions.checkNotNull(resource);
- FileUtils.copyURLToFile(resource, tableConfigFile);
- }
-
- private static void copyFilesystemTableToTmpDirectory(String sourcePath, String tableName, File baseDir)
- throws IOException {
- File fileDb = new File(sourcePath);
-
- if (!fileDb.exists() || !fileDb.isDirectory()) {
- throw new RuntimeException("Directory " + fileDb.getAbsolutePath() + " not found.");
- }
-
- File schemaFile = new File(fileDb, tableName + "_schema.json");
- if (!schemaFile.exists()) {
- throw new RuntimeException("Schema file " + schemaFile.getAbsolutePath() + " not found.");
- }
-
- File tableFile = new File(fileDb, tableName + "_offline_table_config.json");
- if (!tableFile.exists()) {
- throw new RuntimeException("Table table " + tableFile.getAbsolutePath() + " not found.");
- }
-
- File data = new File(fileDb, "rawdata" + File.separator + tableName + "_data.csv");
- if (!data.exists()) {
- throw new RuntimeException(("Data file " + data.getAbsolutePath() + " not found. "));
- }
-
- FileUtils.copyDirectory(fileDb, baseDir);
+ protected int getNumQuickstartRunnerServers() {
+ return 1;
}
- private static void runSampleQueries(QuickstartRunner runner)
+ @Override
+ public void runSampleQueries(QuickstartRunner runner)
throws Exception {
String q1 = "select count(*) from baseballStats limit 1";
printStatus(Color.YELLOW, "Total number of documents in the table");
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/TimestampIndexQuickstart.java b/pinot-tools/src/main/java/org/apache/pinot/tools/TimestampIndexQuickstart.java
index a8be24defc..e8af029757 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/TimestampIndexQuickstart.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/TimestampIndexQuickstart.java
@@ -18,84 +18,24 @@
*/
package org.apache.pinot.tools;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-import java.io.File;
-import java.io.IOException;
-import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
-import org.apache.commons.io.FileUtils;
-import org.apache.pinot.tools.Quickstart.Color;
import org.apache.pinot.tools.admin.PinotAdministrator;
import org.apache.pinot.tools.admin.command.QuickstartRunner;
-import static org.apache.pinot.tools.Quickstart.prettyPrintResponse;
-
-public class TimestampIndexQuickstart extends QuickStartBase {
+public class TimestampIndexQuickstart extends Quickstart {
@Override
public List<String> types() {
return Collections.singletonList("TIMESTAMP");
}
- private File _schemaFile;
- private File _ingestionJobSpecFile;
-
- public static void main(String[] args)
- throws Exception {
- List<String> arguments = new ArrayList<>();
- arguments.addAll(Arrays.asList("QuickStart", "-type", "TIMESTAMP"));
- arguments.addAll(Arrays.asList(args));
- PinotAdministrator.main(arguments.toArray(new String[arguments.size()]));
- }
-
- private QuickstartTableRequest prepareTableRequest(File baseDir)
- throws IOException {
- _schemaFile = new File(baseDir, "airlineStats_schema.json");
- _ingestionJobSpecFile = new File(baseDir, "ingestionJobSpec.yaml");
- File tableConfigFile = new File(baseDir, "airlineStats_offline_table_config.json");
-
- ClassLoader classLoader = Quickstart.class.getClassLoader();
- URL resource = classLoader.getResource("examples/batch/airlineStats/airlineStats_schema.json");
- Preconditions.checkNotNull(resource);
- FileUtils.copyURLToFile(resource, _schemaFile);
- resource = classLoader.getResource("examples/batch/airlineStats/ingestionJobSpec.yaml");
- Preconditions.checkNotNull(resource);
- FileUtils.copyURLToFile(resource, _ingestionJobSpecFile);
- resource = classLoader.getResource("examples/batch/airlineStats/airlineStats_offline_table_config.json");
- Preconditions.checkNotNull(resource);
- FileUtils.copyURLToFile(resource, tableConfigFile);
- return new QuickstartTableRequest(baseDir.getAbsolutePath());
- }
-
- public void execute()
+ @Override
+ public void runSampleQueries(QuickstartRunner runner)
throws Exception {
- File quickstartTmpDir = new File(_dataDir, String.valueOf(System.currentTimeMillis()));
- File baseDir = new File(quickstartTmpDir, "airlineStats");
- File dataDir = new File(baseDir, "data");
- Preconditions.checkState(dataDir.mkdirs());
- QuickstartTableRequest bootstrapTableRequest = prepareTableRequest(baseDir);
- final QuickstartRunner runner =
- new QuickstartRunner(Lists.newArrayList(bootstrapTableRequest), 1, 1, 1, 1, dataDir, getConfigOverrides());
- printStatus(Color.YELLOW, "***** Starting Zookeeper, 1 servers, 1 brokers and 1 controller *****");
- runner.startAll();
- Runtime.getRuntime().addShutdownHook(new Thread(() -> {
- try {
- printStatus(Color.GREEN, "***** Shutting down timestamp quick start *****");
- runner.stop();
- FileUtils.deleteDirectory(quickstartTmpDir);
- } catch (Exception e) {
- e.printStackTrace();
- }
- }));
- printStatus(Color.YELLOW, "***** Bootstrap airlineStats offline table *****");
- runner.bootstrapTable();
printStatus(Color.YELLOW, "***** Pinot Timestamp with timestamp table setup is complete *****");
-
-
String q1 = "select ts, $ts$DAY, $ts$WEEK, $ts$MONTH from airlineStats limit 1";
printStatus(Color.YELLOW, "Pick one row with timestamp and different granularity using generated column name ");
printStatus(Color.CYAN, "Query : " + q1);
@@ -115,6 +55,13 @@ public class TimestampIndexQuickstart extends QuickStartBase {
printStatus(Color.CYAN, "Query : " + q3);
printStatus(Color.YELLOW, prettyPrintResponse(runner.runQuery(q3)));
printStatus(Color.GREEN, "***************************************************");
- printStatus(Color.GREEN, "You can always go to http://localhost:9000 to play around in the query console");
+ }
+
+ public static void main(String[] args)
+ throws Exception {
+ List<String> arguments = new ArrayList<>();
+ arguments.addAll(Arrays.asList("QuickStart", "-type", "TIMESTAMP"));
+ arguments.addAll(Arrays.asList(args));
+ PinotAdministrator.main(arguments.toArray(new String[arguments.size()]));
}
}
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/QuickStartCommand.java b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/QuickStartCommand.java
index 96061ed4b7..1578814985 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/QuickStartCommand.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/QuickStartCommand.java
@@ -39,9 +39,9 @@ public class QuickStartCommand extends AbstractBaseAdminCommand implements Comma
description = "Type of quickstart, supported: STREAM/BATCH/HYBRID")
private String _type;
- @CommandLine.Option(names = {"-bootstrapTableDir"}, required = false,
- description = "Directory containing table schema, config, and data.")
- private String _bootstrapTableDir;
+ @CommandLine.Option(names = {"-bootstrapTableDir"}, required = false, arity = "1..*",
+ description = "A list of Directories, each directory containing table schema, config, and data.")
+ private String[] _bootstrapTableDirs;
@CommandLine.Option(names = {"-tmpDir", "-quickstartDir", "-dataDir"}, required = false,
description = "Temp Directory to host quickstart data")
@@ -87,11 +87,19 @@ public class QuickStartCommand extends AbstractBaseAdminCommand implements Comma
}
public String getBootstrapDataDir() {
- return _bootstrapTableDir;
+ return (_bootstrapTableDirs != null && _bootstrapTableDirs.length > 0) ? _bootstrapTableDirs[0] : null;
+ }
+
+ public String[] getBootstrapDataDirs() {
+ return _bootstrapTableDirs;
}
public void setBootstrapTableDir(String bootstrapTableDir) {
- _bootstrapTableDir = bootstrapTableDir;
+ _bootstrapTableDirs = new String[]{bootstrapTableDir};
+ }
+
+ public void setBootstrapTableDirs(String[] bootstrapTableDirs) {
+ _bootstrapTableDirs = bootstrapTableDirs;
}
public String getZkExternalAddress() {
@@ -125,7 +133,7 @@ public class QuickStartCommand extends AbstractBaseAdminCommand implements Comma
}
public QuickStartBase selectQuickStart(String type)
- throws NoSuchMethodException, InvocationTargetException, InstantiationException, IllegalAccessException {
+ throws NoSuchMethodException, InvocationTargetException, InstantiationException, IllegalAccessException {
Set<Class<? extends QuickStartBase>> quickStarts = allQuickStarts();
for (Class<? extends QuickStartBase> quickStart : quickStarts) {
QuickStartBase quickStartBase = quickStart.getDeclaredConstructor().newInstance();
@@ -134,18 +142,19 @@ public class QuickStartCommand extends AbstractBaseAdminCommand implements Comma
}
}
throw new UnsupportedOperationException("Unsupported QuickStart type: " + type + ". "
- + "Valid types are: " + errroMessageFor(quickStarts));
+ + "Valid types are: " + errroMessageFor(quickStarts));
}
@Override
- public boolean execute() throws Exception {
+ public boolean execute()
+ throws Exception {
PluginManager.get().init();
if (_type == null) {
Set<Class<? extends QuickStartBase>> quickStarts = allQuickStarts();
throw new UnsupportedOperationException("No QuickStart type provided. "
- + "Valid types are: " + errroMessageFor(quickStarts));
+ + "Valid types are: " + errroMessageFor(quickStarts));
}
QuickStartBase quickstart = selectQuickStart(_type);
@@ -154,8 +163,8 @@ public class QuickStartCommand extends AbstractBaseAdminCommand implements Comma
quickstart.setDataDir(_tmpDir);
}
- if (_bootstrapTableDir != null) {
- quickstart.setBootstrapDataDir(_bootstrapTableDir);
+ if (_bootstrapTableDirs != null) {
+ quickstart.setBootstrapDataDirs(_bootstrapTableDirs);
}
if (_zkExternalAddress != null) {
@@ -171,7 +180,7 @@ public class QuickStartCommand extends AbstractBaseAdminCommand implements Comma
}
private static List<String> errroMessageFor(Set<Class<? extends QuickStartBase>> quickStarts)
- throws InstantiationException, IllegalAccessException, InvocationTargetException, NoSuchMethodException {
+ throws InstantiationException, IllegalAccessException, InvocationTargetException, NoSuchMethodException {
List<String> validTypes = new ArrayList<>();
for (Class<? extends QuickStartBase> quickStart : quickStarts) {
validTypes.addAll(quickStart.getDeclaredConstructor().newInstance().types());
diff --git a/pinot-tools/src/main/resources/examples/batch/githubEvents/githubEvents_offline_complexTypeHandling_table_config.json b/pinot-tools/src/main/resources/examples/batch/githubComplexTypeEvents/githubComplexTypeEvents_offline_table_config.json
similarity index 93%
rename from pinot-tools/src/main/resources/examples/batch/githubEvents/githubEvents_offline_complexTypeHandling_table_config.json
rename to pinot-tools/src/main/resources/examples/batch/githubComplexTypeEvents/githubComplexTypeEvents_offline_table_config.json
index 154f9dfcdd..d434773aab 100644
--- a/pinot-tools/src/main/resources/examples/batch/githubEvents/githubEvents_offline_complexTypeHandling_table_config.json
+++ b/pinot-tools/src/main/resources/examples/batch/githubComplexTypeEvents/githubComplexTypeEvents_offline_table_config.json
@@ -1,5 +1,5 @@
{
- "tableName": "githubEvents",
+ "tableName": "githubComplexTypeEvents",
"tableType": "OFFLINE",
"tenants": {
},
diff --git a/pinot-tools/src/main/resources/examples/batch/githubEvents/githubEvents_offline_complexTypeHandling_schema.json b/pinot-tools/src/main/resources/examples/batch/githubComplexTypeEvents/githubComplexTypeEvents_schema.json
similarity index 97%
rename from pinot-tools/src/main/resources/examples/batch/githubEvents/githubEvents_offline_complexTypeHandling_schema.json
rename to pinot-tools/src/main/resources/examples/batch/githubComplexTypeEvents/githubComplexTypeEvents_schema.json
index 6b44c9ec65..1c79af79cf 100644
--- a/pinot-tools/src/main/resources/examples/batch/githubEvents/githubEvents_offline_complexTypeHandling_schema.json
+++ b/pinot-tools/src/main/resources/examples/batch/githubComplexTypeEvents/githubComplexTypeEvents_schema.json
@@ -71,5 +71,5 @@
"granularity": "1:SECONDS"
}
],
- "schemaName": "githubEvents"
+ "schemaName": "githubComplexTypeEvents"
}
diff --git a/pinot-tools/src/main/resources/examples/batch/githubEvents/ingestionJobComplexTypeHandlingSpec.yaml b/pinot-tools/src/main/resources/examples/batch/githubComplexTypeEvents/ingestionJobSpec.yaml
similarity index 94%
rename from pinot-tools/src/main/resources/examples/batch/githubEvents/ingestionJobComplexTypeHandlingSpec.yaml
rename to pinot-tools/src/main/resources/examples/batch/githubComplexTypeEvents/ingestionJobSpec.yaml
index 04b67c4bfa..c2c8e13cd3 100644
--- a/pinot-tools/src/main/resources/examples/batch/githubEvents/ingestionJobComplexTypeHandlingSpec.yaml
+++ b/pinot-tools/src/main/resources/examples/batch/githubComplexTypeEvents/ingestionJobSpec.yaml
@@ -41,7 +41,7 @@ executionFrameworkSpec:
jobType: SegmentCreationAndTarPush
# inputDirURI: Root directory of input data, expected to have scheme configured in PinotFS.
-inputDirURI: 'examples/batch/githubEvents/rawdata_complexTypeHandling'
+inputDirURI: 'examples/batch/githubComplexTypeEvents/rawdata'
# includeFileNamePattern: include file name pattern, supported glob pattern.
# Sample usage:
@@ -56,7 +56,7 @@ includeFileNamePattern: 'glob:**/*.json'
# _excludeFileNamePattern: ''
# outputDirURI: Root directory of output segments, expected to have scheme configured in PinotFS.
-outputDirURI: 'examples/batch/githubEvents/segments'
+outputDirURI: 'examples/batch/githubComplexTypeEvents/segments'
# overwriteOutput: Overwrite output segments if existed.
overwriteOutput: true
@@ -106,13 +106,13 @@ recordReaderSpec:
tableSpec:
# tableName: Table name
- tableName: 'githubEvents'
+ tableName: 'githubComplexTypeEvents'
# schemaURI: defines where to read the table schema, supports PinotFS or HTTP.
# E.g.
# hdfs://path/to/table_schema.json
# http://localhost:9000/tables/myTable/schema
- schemaURI: 'http://localhost:9000/tables/githubEvents/schema'
+ schemaURI: 'http://localhost:9000/tables/githubComplexTypeEvents/schema'
# tableConfigURI: defines where to reade the table config.
# Supports using PinotFS or HTTP.
@@ -121,7 +121,7 @@ tableSpec:
# http://localhost:9000/tables/myTable
# Note that the API to read Pinot table config directly from pinot controller contains a JSON wrapper.
# The real table config is the object under the field 'OFFLINE'.
- tableConfigURI: 'http://localhost:9000/tables/githubEvents'
+ tableConfigURI: 'http://localhost:9000/tables/githubComplexTypeEvents'
# pinotClusterSpecs: defines the Pinot Cluster Access Point.
pinotClusterSpecs:
diff --git a/pinot-tools/src/main/resources/examples/batch/githubEvents/rawdata_complexTypeHandling/githubEvents_data.json b/pinot-tools/src/main/resources/examples/batch/githubComplexTypeEvents/rawdata/githubComplexTypeEvents_data.json
similarity index 100%
rename from pinot-tools/src/main/resources/examples/batch/githubEvents/rawdata_complexTypeHandling/githubEvents_data.json
rename to pinot-tools/src/main/resources/examples/batch/githubComplexTypeEvents/rawdata/githubComplexTypeEvents_data.json
diff --git a/pinot-tools/src/main/resources/examples/batch/githubEvents/ingestionJobSpec.yaml b/pinot-tools/src/main/resources/examples/batch/githubEvents/ingestionJobSpec.yaml
index f8216f1bb6..a51bde638f 100644
--- a/pinot-tools/src/main/resources/examples/batch/githubEvents/ingestionJobSpec.yaml
+++ b/pinot-tools/src/main/resources/examples/batch/githubEvents/ingestionJobSpec.yaml
@@ -41,7 +41,7 @@ executionFrameworkSpec:
jobType: SegmentCreationAndTarPush
# inputDirURI: Root directory of input data, expected to have scheme configured in PinotFS.
-inputDirURI: 'examples/batch/githubEvents/rawdata_json_index'
+inputDirURI: 'examples/batch/githubEvents/rawdata'
# includeFileNamePattern: include file name pattern, supported glob pattern.
# Sample usage:
diff --git a/pinot-tools/src/main/resources/examples/batch/githubEvents/rawdata_json_index/githubEvents_data.json b/pinot-tools/src/main/resources/examples/batch/githubEvents/rawdata/githubEvents_data.json
similarity index 100%
rename from pinot-tools/src/main/resources/examples/batch/githubEvents/rawdata_json_index/githubEvents_data.json
rename to pinot-tools/src/main/resources/examples/batch/githubEvents/rawdata/githubEvents_data.json
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org