You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by xi...@apache.org on 2020/12/28 08:15:56 UTC
[incubator-pinot] branch master updated: Adding S3PinotFS as one of
the default PinotFS for Quickstart (#6379)
This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new e268a79 Adding S3PinotFS as one of the default PinotFS for Quickstart (#6379)
e268a79 is described below
commit e268a795a684f18e1876e4530c81ef00baec878c
Author: Xiang Fu <fx...@gmail.com>
AuthorDate: Mon Dec 28 00:15:26 2020 -0800
Adding S3PinotFS as one of the default PinotFS for Quickstart (#6379)
* Adding s3 as default pinot fs for quickstart
* Update BootstrapTableTool.java
---
.../org/apache/pinot/tools/BootstrapTableTool.java | 16 +++++++------
.../tools/admin/command/QuickstartRunner.java | 27 +++++++++++++++++++++-
2 files changed, 35 insertions(+), 8 deletions(-)
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/BootstrapTableTool.java b/pinot-tools/src/main/java/org/apache/pinot/tools/BootstrapTableTool.java
index 0723b06..3de51b7 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/BootstrapTableTool.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/BootstrapTableTool.java
@@ -115,13 +115,15 @@ public class BootstrapTableTool {
String inputDirURI = spec.getInputDirURI();
if (!new File(inputDirURI).exists()) {
URL resolvedInputDirURI = BootstrapTableTool.class.getClassLoader().getResource(inputDirURI);
- if (resolvedInputDirURI.getProtocol().equals("jar")) {
- String[] splits = resolvedInputDirURI.getFile().split("!");
- String inputDir = new File(setupTableTmpDir, "inputData").toString();
- JarUtils.copyResourcesToDirectory(splits[0], splits[1].substring(1), inputDir);
- spec.setInputDirURI(inputDir);
- } else {
- spec.setInputDirURI(resolvedInputDirURI.toString());
+ if (resolvedInputDirURI != null) {
+ if ("jar".equals(resolvedInputDirURI.getProtocol())) {
+ String[] splits = resolvedInputDirURI.getFile().split("!");
+ String inputDir = new File(setupTableTmpDir, "inputData").toString();
+ JarUtils.copyResourcesToDirectory(splits[0], splits[1].substring(1), inputDir);
+ spec.setInputDirURI(inputDir);
+ } else {
+ spec.setInputDirURI(resolvedInputDirURI.toString());
+ }
}
}
IngestionJobLauncher.runIngestionJob(spec);
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/QuickstartRunner.java b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/QuickstartRunner.java
index e2e3e38..1f4f838 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/QuickstartRunner.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/QuickstartRunner.java
@@ -19,6 +19,7 @@
package org.apache.pinot.tools.admin.command;
import com.fasterxml.jackson.databind.JsonNode;
+import com.google.common.collect.ImmutableMap;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
@@ -28,21 +29,27 @@ import java.net.InetAddress;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
import java.util.Random;
import org.apache.commons.io.FileUtils;
import org.apache.pinot.common.utils.CommonConstants;
import org.apache.pinot.spi.config.table.TableType;
import org.apache.pinot.spi.config.tenant.TenantRole;
+import org.apache.pinot.spi.env.PinotConfiguration;
+import org.apache.pinot.spi.filesystem.PinotFSFactory;
import org.apache.pinot.spi.ingestion.batch.IngestionJobLauncher;
import org.apache.pinot.spi.ingestion.batch.spec.SegmentGenerationJobSpec;
import org.apache.pinot.spi.utils.JsonUtils;
import org.apache.pinot.tools.QuickstartTableRequest;
import org.apache.pinot.tools.BootstrapTableTool;
import org.apache.pinot.tools.utils.JarUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.yaml.snakeyaml.Yaml;
public class QuickstartRunner {
+ private static final Logger LOGGER = LoggerFactory.getLogger(QuickstartRunner.class.getName());
private static final Random RANDOM = new Random();
private static final String CLUSTER_NAME = "QuickStartCluster";
@@ -138,6 +145,7 @@ public class QuickstartRunner {
public void startAll()
throws Exception {
+ registerDefaultPinotFS();
startZookeeper();
startControllers();
startBrokers();
@@ -222,4 +230,21 @@ public class QuickstartRunner {
return JsonUtils.stringToJsonNode(new PostQueryCommand().setBrokerPort(String.valueOf(brokerPort))
.setQueryType(CommonConstants.Broker.Request.SQL).setQuery(query).run());
}
-}
+
+ public static void registerDefaultPinotFS() {
+ registerPinotFS("s3", "org.apache.pinot.plugin.filesystem.S3PinotFS", ImmutableMap.of("region", System.getProperty("AWS_REGION", "us-west-2")));
+ }
+
+ public static void registerPinotFS(String scheme, String fsClassName, Map<String, Object> configs) {
+ if (PinotFSFactory.isSchemeSupported(scheme)) {
+ LOGGER.info("PinotFS for scheme: {} is already registered.", scheme);
+ return;
+ }
+ try {
+ PinotFSFactory.register(scheme, fsClassName, new PinotConfiguration(configs));
+ LOGGER.info("Registered PinotFS for scheme: {}", scheme);
+ } catch (Exception e) {
+ LOGGER.info("Unable to init PinotFS for scheme: {}, class name: {}, configs: {}, Error: {}", scheme, fsClassName, configs, e);
+ }
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org