You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by xi...@apache.org on 2020/12/23 04:18:10 UTC
[incubator-pinot] branch adding_default_pinot_fs_to_quickstart
created (now 1a42937)
This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a change to branch adding_default_pinot_fs_to_quickstart
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git.
at 1a42937 Adding s3 as default pinot fs for quickstart
This branch includes the following new commits:
new 1a42937 Adding s3 as default pinot fs for quickstart
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org
[incubator-pinot] 01/01: Adding s3 as default pinot fs for
quickstart
Posted by xi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch adding_default_pinot_fs_to_quickstart
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit 1a42937d779bbd9319c269a269cf0fedc84bfea9
Author: Xiang Fu <fx...@gmail.com>
AuthorDate: Tue Dec 22 20:17:41 2020 -0800
Adding s3 as default pinot fs for quickstart
---
.../org/apache/pinot/tools/BootstrapTableTool.java | 16 +++++++------
.../tools/admin/command/QuickstartRunner.java | 27 +++++++++++++++++++++-
2 files changed, 35 insertions(+), 8 deletions(-)
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/BootstrapTableTool.java b/pinot-tools/src/main/java/org/apache/pinot/tools/BootstrapTableTool.java
index 0723b06..b7eb335 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/BootstrapTableTool.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/BootstrapTableTool.java
@@ -115,13 +115,15 @@ public class BootstrapTableTool {
String inputDirURI = spec.getInputDirURI();
if (!new File(inputDirURI).exists()) {
URL resolvedInputDirURI = BootstrapTableTool.class.getClassLoader().getResource(inputDirURI);
- if (resolvedInputDirURI.getProtocol().equals("jar")) {
- String[] splits = resolvedInputDirURI.getFile().split("!");
- String inputDir = new File(setupTableTmpDir, "inputData").toString();
- JarUtils.copyResourcesToDirectory(splits[0], splits[1].substring(1), inputDir);
- spec.setInputDirURI(inputDir);
- } else {
- spec.setInputDirURI(resolvedInputDirURI.toString());
+ if (resolvedInputDirURI != null) {
+ if (resolvedInputDirURI.getProtocol().equals("jar")) {
+ String[] splits = resolvedInputDirURI.getFile().split("!");
+ String inputDir = new File(setupTableTmpDir, "inputData").toString();
+ JarUtils.copyResourcesToDirectory(splits[0], splits[1].substring(1), inputDir);
+ spec.setInputDirURI(inputDir);
+ } else {
+ spec.setInputDirURI(resolvedInputDirURI.toString());
+ }
}
}
IngestionJobLauncher.runIngestionJob(spec);
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/QuickstartRunner.java b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/QuickstartRunner.java
index e2e3e38..1f4f838 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/QuickstartRunner.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/QuickstartRunner.java
@@ -19,6 +19,7 @@
package org.apache.pinot.tools.admin.command;
import com.fasterxml.jackson.databind.JsonNode;
+import com.google.common.collect.ImmutableMap;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
@@ -28,21 +29,27 @@ import java.net.InetAddress;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
import java.util.Random;
import org.apache.commons.io.FileUtils;
import org.apache.pinot.common.utils.CommonConstants;
import org.apache.pinot.spi.config.table.TableType;
import org.apache.pinot.spi.config.tenant.TenantRole;
+import org.apache.pinot.spi.env.PinotConfiguration;
+import org.apache.pinot.spi.filesystem.PinotFSFactory;
import org.apache.pinot.spi.ingestion.batch.IngestionJobLauncher;
import org.apache.pinot.spi.ingestion.batch.spec.SegmentGenerationJobSpec;
import org.apache.pinot.spi.utils.JsonUtils;
import org.apache.pinot.tools.QuickstartTableRequest;
import org.apache.pinot.tools.BootstrapTableTool;
import org.apache.pinot.tools.utils.JarUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.yaml.snakeyaml.Yaml;
public class QuickstartRunner {
+ private static final Logger LOGGER = LoggerFactory.getLogger(QuickstartRunner.class.getName());
private static final Random RANDOM = new Random();
private static final String CLUSTER_NAME = "QuickStartCluster";
@@ -138,6 +145,7 @@ public class QuickstartRunner {
public void startAll()
throws Exception {
+ registerDefaultPinotFS();
startZookeeper();
startControllers();
startBrokers();
@@ -222,4 +230,21 @@ public class QuickstartRunner {
return JsonUtils.stringToJsonNode(new PostQueryCommand().setBrokerPort(String.valueOf(brokerPort))
.setQueryType(CommonConstants.Broker.Request.SQL).setQuery(query).run());
}
-}
+
+ public static void registerDefaultPinotFS() {
+ registerPinotFS("s3", "org.apache.pinot.plugin.filesystem.S3PinotFS", ImmutableMap.of("region", System.getProperty("AWS_REGION", "us-west-2")));
+ }
+
+ public static void registerPinotFS(String scheme, String fsClassName, Map<String, Object> configs) {
+ if (PinotFSFactory.isSchemeSupported(scheme)) {
+ LOGGER.info("PinotFS for scheme: {} is already registered.", scheme);
+ return;
+ }
+ try {
+ PinotFSFactory.register(scheme, fsClassName, new PinotConfiguration(configs));
+ LOGGER.info("Registered PinotFS for scheme: {}", scheme);
+ } catch (Exception e) {
+ LOGGER.info("Unable to init PinotFS for scheme: {}, class name: {}, configs: {}, Error: {}", scheme, fsClassName, configs, e);
+ }
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org