You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ne...@apache.org on 2020/05/18 19:54:56 UTC
[incubator-pinot] branch master updated: Clean slate on every
quickstart run (#5395)
This is an automated email from the ASF dual-hosted git repository.
nehapawar pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 86aea53 Clean slate on every quickstart run (#5395)
86aea53 is described below
commit 86aea534ca51ffbf5cdc7e32599fba16570ac85f
Author: Neha Pawar <ne...@gmail.com>
AuthorDate: Mon May 18 12:54:43 2020 -0700
Clean slate on every quickstart run (#5395)
---
.../org/apache/pinot/tools/HybridQuickstart.java | 44 +++++++++-------------
.../java/org/apache/pinot/tools/Quickstart.java | 26 ++++++-------
.../apache/pinot/tools/QuickstartTableRequest.java | 12 +-----
.../org/apache/pinot/tools/RealtimeQuickStart.java | 20 +++++-----
.../tools/admin/command/QuickstartRunner.java | 13 +++++--
5 files changed, 49 insertions(+), 66 deletions(-)
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/HybridQuickstart.java b/pinot-tools/src/main/java/org/apache/pinot/tools/HybridQuickstart.java
index ac87795..3c55441 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/HybridQuickstart.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/HybridQuickstart.java
@@ -42,8 +42,6 @@ import static org.apache.pinot.tools.Quickstart.printStatus;
public class HybridQuickstart {
- private File _offlineQuickStartDataDir;
- private File _realtimeQuickStartDataDir;
private StreamDataServerStartable _kafkaStarter;
private ZkStarter.ZookeeperInstance _zookeeperInstance;
private File _schemaFile;
@@ -59,17 +57,12 @@ public class HybridQuickstart {
new HybridQuickstart().execute();
}
- private QuickstartTableRequest prepareOfflineTableRequest()
+ private QuickstartTableRequest prepareOfflineTableRequest(File configDir)
throws IOException {
- _offlineQuickStartDataDir = new File("quickStartData" + System.currentTimeMillis());
- if (!_offlineQuickStartDataDir.exists()) {
- Preconditions.checkState(_offlineQuickStartDataDir.mkdirs());
- }
-
- _schemaFile = new File(_offlineQuickStartDataDir, "airlineStats_schema.json");
- _ingestionJobSpecFile = new File(_offlineQuickStartDataDir, "ingestionJobSpec.yaml");
- File tableConfigFile = new File(_offlineQuickStartDataDir, "airlineStats_offline_table_config.json");
+ _schemaFile = new File(configDir, "airlineStats_schema.json");
+ _ingestionJobSpecFile = new File(configDir, "ingestionJobSpec.yaml");
+ File tableConfigFile = new File(configDir, "airlineStats_offline_table_config.json");
ClassLoader classLoader = Quickstart.class.getClassLoader();
URL resource = classLoader.getResource("examples/batch/airlineStats/airlineStats_schema.json");
@@ -82,20 +75,15 @@ public class HybridQuickstart {
Preconditions.checkNotNull(resource);
FileUtils.copyURLToFile(resource, tableConfigFile);
- return new QuickstartTableRequest("airlineStats", _schemaFile, tableConfigFile, _ingestionJobSpecFile, _offlineQuickStartDataDir,
+ return new QuickstartTableRequest("airlineStats", _schemaFile, tableConfigFile, _ingestionJobSpecFile,
FileFormat.AVRO);
}
- private QuickstartTableRequest prepareRealtimeTableRequest()
+ private QuickstartTableRequest prepareRealtimeTableRequest(File configDir)
throws IOException {
- _realtimeQuickStartDataDir = new File("quickStartData" + System.currentTimeMillis());
-
- if (!_realtimeQuickStartDataDir.exists()) {
- Preconditions.checkState(_realtimeQuickStartDataDir.mkdirs());
- }
- _dataFile = new File(_realtimeQuickStartDataDir, "airlineStats_data.avro");
- _realtimeTableConfigFile = new File(_realtimeQuickStartDataDir, "airlineStats_realtime_table_config.json");
+ _dataFile = new File(configDir, "airlineStats_data.avro");
+ _realtimeTableConfigFile = new File(configDir, "airlineStats_realtime_table_config.json");
URL resource = Quickstart.class.getClassLoader().getResource(
"examples/stream/airlineStats/airlineStats_realtime_table_config.json");
@@ -122,13 +110,16 @@ public class HybridQuickstart {
public void execute()
throws Exception {
- QuickstartTableRequest offlineRequest = prepareOfflineTableRequest();
- QuickstartTableRequest realtimeTableRequest = prepareRealtimeTableRequest();
- File tempDir = new File(FileUtils.getTempDirectory(), String.valueOf(System.currentTimeMillis()));
- Preconditions.checkState(tempDir.mkdirs());
+ File quickstartTmpDir = new File(FileUtils.getTempDirectory(), String.valueOf(System.currentTimeMillis()));
+ File configDir = new File(quickstartTmpDir, "configs");
+ File dataDir = new File(quickstartTmpDir, "data");
+ Preconditions.checkState(configDir.mkdirs());
+ Preconditions.checkState(dataDir.mkdirs());
+ QuickstartTableRequest offlineRequest = prepareOfflineTableRequest(configDir);
+ QuickstartTableRequest realtimeTableRequest = prepareRealtimeTableRequest(configDir);
final QuickstartRunner runner =
- new QuickstartRunner(Lists.newArrayList(offlineRequest, realtimeTableRequest), 1, 1, 1, tempDir);
+ new QuickstartRunner(Lists.newArrayList(offlineRequest, realtimeTableRequest), 1, 1, 1, dataDir);
printStatus(Color.YELLOW, "***** Starting Kafka *****");
startKafka();
printStatus(Color.YELLOW, "***** Starting Zookeeper, 1 servers, 1 brokers and 1 controller *****");
@@ -202,8 +193,7 @@ public class HybridQuickstart {
runner.stop();
_kafkaStarter.stop();
ZkStarter.stopLocalZkServer(_zookeeperInstance);
- FileUtils.deleteDirectory(_offlineQuickStartDataDir);
- FileUtils.deleteDirectory(_realtimeQuickStartDataDir);
+ FileUtils.deleteDirectory(quickstartTmpDir);
} catch (Exception e) {
e.printStackTrace();
}
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/Quickstart.java b/pinot-tools/src/main/java/org/apache/pinot/tools/Quickstart.java
index 259231c..3c0b0d1 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/Quickstart.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/Quickstart.java
@@ -115,16 +115,16 @@ public class Quickstart {
public void execute()
throws Exception {
- final File quickStartDataDir = new File("quickStartData" + System.currentTimeMillis());
+ File quickstartTmpDir = new File(FileUtils.getTempDirectory(), String.valueOf(System.currentTimeMillis()));
+ File configDir = new File(quickstartTmpDir, "configs");
+ File dataDir = new File(quickstartTmpDir, "data");
+ Preconditions.checkState(configDir.mkdirs());
+ Preconditions.checkState(dataDir.mkdirs());
- if (!quickStartDataDir.exists()) {
- Preconditions.checkState(quickStartDataDir.mkdirs());
- }
-
- File schemaFile = new File(quickStartDataDir, "baseballStats_schema.json");
- File dataFile = new File(quickStartDataDir, "baseballStats_data.csv");
- File tableConfigFile = new File(quickStartDataDir, "baseballStats_offline_table_config.json");
- File ingestionJobSpecFile = new File(quickStartDataDir, "ingestionJobSpec.yaml");
+ File schemaFile = new File(configDir, "baseballStats_schema.json");
+ File dataFile = new File(configDir, "baseballStats_data.csv");
+ File tableConfigFile = new File(configDir, "baseballStats_offline_table_config.json");
+ File ingestionJobSpecFile = new File(configDir, "ingestionJobSpec.yaml");
ClassLoader classLoader = Quickstart.class.getClassLoader();
URL resource = classLoader.getResource("examples/batch/baseballStats/baseballStats_schema.json");
@@ -140,11 +140,9 @@ public class Quickstart {
com.google.common.base.Preconditions.checkNotNull(resource);
FileUtils.copyURLToFile(resource, tableConfigFile);
- File tempDir = new File(FileUtils.getTempDirectory(), String.valueOf(System.currentTimeMillis()));
- Preconditions.checkState(tempDir.mkdirs());
QuickstartTableRequest request =
- new QuickstartTableRequest("baseballStats", schemaFile, tableConfigFile, ingestionJobSpecFile, quickStartDataDir, FileFormat.CSV);
- final QuickstartRunner runner = new QuickstartRunner(Lists.newArrayList(request), 1, 1, 1, tempDir);
+ new QuickstartTableRequest("baseballStats", schemaFile, tableConfigFile, ingestionJobSpecFile, FileFormat.CSV);
+ final QuickstartRunner runner = new QuickstartRunner(Lists.newArrayList(request), 1, 1, 1, dataDir);
printStatus(Color.CYAN, "***** Starting Zookeeper, controller, broker and server *****");
runner.startAll();
@@ -161,7 +159,7 @@ public class Quickstart {
try {
printStatus(Color.GREEN, "***** Shutting down offline quick start *****");
runner.stop();
- FileUtils.deleteDirectory(quickStartDataDir);
+ FileUtils.deleteDirectory(quickstartTmpDir);
} catch (Exception e) {
e.printStackTrace();
}
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/QuickstartTableRequest.java b/pinot-tools/src/main/java/org/apache/pinot/tools/QuickstartTableRequest.java
index 55b27cb..117ccf1 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/QuickstartTableRequest.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/QuickstartTableRequest.java
@@ -28,16 +28,14 @@ public class QuickstartTableRequest {
File schemaFile;
File tableRequestFile;
File ingestionJobFile;
- File dataDir;
TableType tableType;
String tableName;
FileFormat segmentFileFormat = FileFormat.CSV;
- public QuickstartTableRequest(String tableName, File schemaFile, File tableRequest, File ingestionJobFile, File dataDir,
+ public QuickstartTableRequest(String tableName, File schemaFile, File tableRequest, File ingestionJobFile,
FileFormat segmentFileFormat) {
this.tableName = tableName;
this.schemaFile = schemaFile;
- this.dataDir = dataDir;
this.tableRequestFile = tableRequest;
tableType = TableType.OFFLINE;
this.segmentFileFormat = segmentFileFormat;
@@ -83,14 +81,6 @@ public class QuickstartTableRequest {
this.ingestionJobFile = ingestionJobFile;
}
- public File getDataDir() {
- return dataDir;
- }
-
- public void setDataDir(File dataDir) {
- this.dataDir = dataDir;
- }
-
public TableType getTableType() {
return tableType;
}
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/RealtimeQuickStart.java b/pinot-tools/src/main/java/org/apache/pinot/tools/RealtimeQuickStart.java
index dc08ba8..275abc6 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/RealtimeQuickStart.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/RealtimeQuickStart.java
@@ -47,14 +47,14 @@ public class RealtimeQuickStart {
public void execute()
throws Exception {
- final File quickStartDataDir = new File("quickStartData" + System.currentTimeMillis());
+ File quickstartTmpDir = new File(FileUtils.getTempDirectory(), String.valueOf(System.currentTimeMillis()));
+ File configDir = new File(quickstartTmpDir, "configs");
+ File dataDir = new File(quickstartTmpDir, "data");
+ Preconditions.checkState(configDir.mkdirs());
+ Preconditions.checkState(dataDir.mkdirs());
- if (!quickStartDataDir.exists()) {
- Preconditions.checkState(quickStartDataDir.mkdirs());
- }
-
- File schemaFile = new File(quickStartDataDir, "meetupRsvp_schema.json");
- File tableConfigFile = new File(quickStartDataDir, "meetupRsvp_realtime_table_config.json");
+ File schemaFile = new File(configDir, "meetupRsvp_schema.json");
+ File tableConfigFile = new File(configDir, "meetupRsvp_realtime_table_config.json");
ClassLoader classLoader = Quickstart.class.getClassLoader();
URL resource = classLoader.getResource("examples/stream/meetupRsvp/meetupRsvp_schema.json");
@@ -64,10 +64,8 @@ public class RealtimeQuickStart {
com.google.common.base.Preconditions.checkNotNull(resource);
FileUtils.copyURLToFile(resource, tableConfigFile);
- File tempDir = new File(FileUtils.getTempDirectory(), String.valueOf(System.currentTimeMillis()));
- Preconditions.checkState(tempDir.mkdirs());
QuickstartTableRequest request = new QuickstartTableRequest("meetupRsvp", schemaFile, tableConfigFile);
- final QuickstartRunner runner = new QuickstartRunner(Lists.newArrayList(request), 1, 1, 1, tempDir);
+ final QuickstartRunner runner = new QuickstartRunner(Lists.newArrayList(request), 1, 1, 1, dataDir);
printStatus(Color.CYAN, "***** Starting Kafka *****");
final ZkStarter.ZookeeperInstance zookeeperInstance = ZkStarter.startLocalZkServer();
@@ -98,7 +96,7 @@ public class RealtimeQuickStart {
runner.stop();
_kafkaStarter.stop();
ZkStarter.stopLocalZkServer(zookeeperInstance);
- FileUtils.deleteDirectory(quickStartDataDir);
+ FileUtils.deleteDirectory(quickstartTmpDir);
} catch (Exception e) {
e.printStackTrace();
}
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/QuickstartRunner.java b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/QuickstartRunner.java
index 32b49f3..ba36d79 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/QuickstartRunner.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/QuickstartRunner.java
@@ -51,6 +51,11 @@ public class QuickstartRunner {
private static final int DEFAULT_BROKER_PORT = 8000;
private static final int DEFAULT_CONTROLLER_PORT = 9000;
+ private static final String DEFAULT_ZK_DIR = "PinotZkDir";
+ private static final String DEFAULT_CONTROLLER_DIR = "PinotControllerDir";
+ private static final String DEFAULT_SERVER_DATA_DIR = "PinotServerDataDir";
+ private static final String DEFAULT_SERVER_SEGMENT_DIR = "PinotServerSegmentDir";
+
private final List<QuickstartTableRequest> _tableRequests;
private final int _numServers;
private final int _numBrokers;
@@ -85,6 +90,7 @@ public class QuickstartRunner {
throws IOException {
StartZookeeperCommand zkStarter = new StartZookeeperCommand();
zkStarter.setPort(ZK_PORT);
+ zkStarter.setDataDir(new File(_tempDir, DEFAULT_ZK_DIR).getAbsolutePath());
zkStarter.execute();
}
@@ -93,7 +99,8 @@ public class QuickstartRunner {
for (int i = 0; i < _numControllers; i++) {
StartControllerCommand controllerStarter = new StartControllerCommand();
controllerStarter.setControllerPort(String.valueOf(DEFAULT_CONTROLLER_PORT + i)).setZkAddress(ZK_ADDRESS)
- .setClusterName(CLUSTER_NAME).setTenantIsolation(_enableTenantIsolation);
+ .setClusterName(CLUSTER_NAME).setTenantIsolation(_enableTenantIsolation)
+ .setDataDir(new File(_tempDir, DEFAULT_CONTROLLER_DIR + i).getAbsolutePath());
controllerStarter.execute();
_controllerPorts.add(DEFAULT_CONTROLLER_PORT + i);
}
@@ -115,8 +122,8 @@ public class QuickstartRunner {
StartServerCommand serverStarter = new StartServerCommand();
serverStarter.setPort(DEFAULT_SERVER_NETTY_PORT + i).setAdminPort(DEFAULT_SERVER_ADMIN_API_PORT + i)
.setZkAddress(ZK_ADDRESS).setClusterName(CLUSTER_NAME)
- .setDataDir(new File(_tempDir, "PinotServerData" + i).getAbsolutePath())
- .setSegmentDir(new File(_tempDir, "PinotServerSegment" + i).getAbsolutePath());
+ .setDataDir(new File(_tempDir, DEFAULT_SERVER_DATA_DIR + i).getAbsolutePath())
+ .setSegmentDir(new File(_tempDir, DEFAULT_SERVER_SEGMENT_DIR + i).getAbsolutePath());
serverStarter.execute();
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org