You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ja...@apache.org on 2022/07/19 03:17:02 UTC
[pinot] branch master updated: Support variable substitution from environment in ingestion spec (#9065)
This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new eeeb3e0a37 Support variable substitution from environment in ingestion spec (#9065)
eeeb3e0a37 is described below
commit eeeb3e0a3710f367676f4538f39459b7d4f3bcf5
Author: Ravishankar <ra...@gmail.com>
AuthorDate: Tue Jul 19 08:46:57 2022 +0530
Support variable substitution from environment in ingestion spec (#9065)
---
.../spi/ingestion/batch/IngestionJobLauncher.java | 9 +++-
.../ingestion/batch/IngestionJobLauncherTest.java | 61 ++++++++++++++++------
.../command/LaunchDataIngestionJobCommand.java | 2 +-
3 files changed, 53 insertions(+), 19 deletions(-)
diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/ingestion/batch/IngestionJobLauncher.java b/pinot-spi/src/main/java/org/apache/pinot/spi/ingestion/batch/IngestionJobLauncher.java
index 8590aaaa59..a2b80b0af7 100644
--- a/pinot-spi/src/main/java/org/apache/pinot/spi/ingestion/batch/IngestionJobLauncher.java
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/ingestion/batch/IngestionJobLauncher.java
@@ -50,7 +50,7 @@ public class IngestionJobLauncher {
public static final String YAML = "yaml";
public static SegmentGenerationJobSpec getSegmentGenerationJobSpec(String jobSpecFilePath, String propertyFilePath,
- Map<String, Object> context) {
+ Map<String, Object> context, Map<String, String> environmentValues) {
Properties properties = new Properties();
if (propertyFilePath != null) {
try {
@@ -61,6 +61,13 @@ public class IngestionJobLauncher {
}
}
Map<String, Object> propertiesMap = (Map) properties;
+ if (environmentValues != null) {
+ for (String propertyName: propertiesMap.keySet()) {
+ if (environmentValues.get(propertyName) != null) {
+ propertiesMap.put(propertyName, environmentValues.get(propertyName));
+ }
+ }
+ }
if (context != null) {
propertiesMap.putAll(context);
}
diff --git a/pinot-spi/src/test/java/org/apache/pinot/spi/ingestion/batch/IngestionJobLauncherTest.java b/pinot-spi/src/test/java/org/apache/pinot/spi/ingestion/batch/IngestionJobLauncherTest.java
index 747bcdfb7b..cc1fafad79 100644
--- a/pinot-spi/src/test/java/org/apache/pinot/spi/ingestion/batch/IngestionJobLauncherTest.java
+++ b/pinot-spi/src/test/java/org/apache/pinot/spi/ingestion/batch/IngestionJobLauncherTest.java
@@ -18,66 +18,93 @@
*/
package org.apache.pinot.spi.ingestion.batch;
-import java.io.IOException;
import java.util.Arrays;
+import java.util.HashMap;
import java.util.Map;
import org.apache.pinot.spi.ingestion.batch.spec.SegmentGenerationJobSpec;
import org.apache.pinot.spi.utils.GroovyTemplateUtils;
import org.testng.Assert;
+import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
public class IngestionJobLauncherTest {
+ private Map<String, String> _defaultEnvironmentValues;
+
+ @BeforeMethod
+ public void setup() {
+ _defaultEnvironmentValues = new HashMap<String, String>() {{
+ put("year", "2022");
+ put("month", "08");
+ }};
+ }
@Test
- public void testIngestionJobLauncherWithTemplate()
- throws IOException, ClassNotFoundException {
+ public void testIngestionJobLauncherWithTemplate() {
Map<String, Object> context =
GroovyTemplateUtils.getTemplateContext(Arrays.asList("year=2020", "month=05", "day=06"));
SegmentGenerationJobSpec spec = IngestionJobLauncher.getSegmentGenerationJobSpec(
- GroovyTemplateUtils.class.getClassLoader().getResource("ingestion_job_spec_template.yaml").getFile(), null,
- context);
+ GroovyTemplateUtils.class.getClassLoader().getResource("ingestion_job_spec_template.yaml").getFile(),
+ null, context, _defaultEnvironmentValues);
Assert.assertEquals(spec.getInputDirURI(), "file:///path/to/input/2020/05/06");
Assert.assertEquals(spec.getOutputDirURI(), "file:///path/to/output/2020/05/06");
}
@Test
- public void testIngestionJobLauncherWithUnicodeCharForMultivalueFieldDelimiter()
- throws IOException, ClassNotFoundException {
+ public void testIngestionJobLauncherWithUnicodeCharForMultivalueFieldDelimiter() {
SegmentGenerationJobSpec spec = IngestionJobLauncher.getSegmentGenerationJobSpec(
GroovyTemplateUtils.class.getClassLoader().getResource("ingestion_job_spec_unicode.yaml").getFile(), null,
- null);
+ null, null);
Assert.assertEquals("\ufff0", spec.getRecordReaderSpec().getConfigs().get("multiValueDelimiter"));
}
@Test
- public void testIngestionJobLauncherWithTemplateAndPropertyFile()
- throws IOException, ClassNotFoundException {
+ public void testIngestionJobLauncherWithTemplateAndPropertyFile() {
SegmentGenerationJobSpec spec = IngestionJobLauncher.getSegmentGenerationJobSpec(
GroovyTemplateUtils.class.getClassLoader().getResource("ingestion_job_spec_template.yaml").getFile(),
- GroovyTemplateUtils.class.getClassLoader().getResource("job.config").getFile(), null);
+ GroovyTemplateUtils.class.getClassLoader().getResource("job.config").getFile(), null, null);
Assert.assertEquals(spec.getInputDirURI(), "file:///path/to/input/2019/06/07");
Assert.assertEquals(spec.getOutputDirURI(), "file:///path/to/output/2019/06/07");
}
@Test
- public void testIngestionJobLauncherWithTemplateAndPropertyFileAndValueOverride()
- throws IOException, ClassNotFoundException {
+ public void testIngestionJobLauncherWithTemplateAndPropertyFileAndValueOverride() {
Map<String, Object> context = GroovyTemplateUtils.getTemplateContext(Arrays.asList("year=2020"));
SegmentGenerationJobSpec spec = IngestionJobLauncher.getSegmentGenerationJobSpec(
GroovyTemplateUtils.class.getClassLoader().getResource("ingestion_job_spec_template.yaml").getFile(),
- GroovyTemplateUtils.class.getClassLoader().getResource("job.config").getFile(), context);
+ GroovyTemplateUtils.class.getClassLoader().getResource("job.config").getFile(), context, null);
Assert.assertEquals(spec.getInputDirURI(), "file:///path/to/input/2020/06/07");
Assert.assertEquals(spec.getOutputDirURI(), "file:///path/to/output/2020/06/07");
Assert.assertEquals(spec.getSegmentCreationJobParallelism(), 100);
}
@Test
- public void testIngestionJobLauncherWithJsonTemplate()
- throws IOException, ClassNotFoundException {
+ public void testIngestionJobLauncherWithTemplateAndPropertyFileAndEnvironmentVariableOverride() {
+ SegmentGenerationJobSpec spec = IngestionJobLauncher.getSegmentGenerationJobSpec(
+ GroovyTemplateUtils.class.getClassLoader().getResource("ingestion_job_spec_template.yaml").getFile(),
+ GroovyTemplateUtils.class.getClassLoader().getResource("job.config").getFile(), null,
+ _defaultEnvironmentValues);
+ Assert.assertEquals(spec.getInputDirURI(), "file:///path/to/input/2022/08/07");
+ Assert.assertEquals(spec.getOutputDirURI(), "file:///path/to/output/2022/08/07");
+ Assert.assertEquals(spec.getSegmentCreationJobParallelism(), 100);
+ }
+
+ @Test
+ public void testIngestionJobLauncherWithTemplateAndPropertyFileAndValueAndEnvironmentVariableOverride() {
+ Map<String, Object> context = GroovyTemplateUtils.getTemplateContext(Arrays.asList("year=2020"));
+ SegmentGenerationJobSpec spec = IngestionJobLauncher.getSegmentGenerationJobSpec(
+ GroovyTemplateUtils.class.getClassLoader().getResource("ingestion_job_spec_template.yaml").getFile(),
+ GroovyTemplateUtils.class.getClassLoader().getResource("job.config").getFile(), context,
+ _defaultEnvironmentValues);
+ Assert.assertEquals(spec.getInputDirURI(), "file:///path/to/input/2020/08/07");
+ Assert.assertEquals(spec.getOutputDirURI(), "file:///path/to/output/2020/08/07");
+ Assert.assertEquals(spec.getSegmentCreationJobParallelism(), 100);
+ }
+ @Test
+ public void testIngestionJobLauncherWithJsonTemplate() {
SegmentGenerationJobSpec spec = IngestionJobLauncher.getSegmentGenerationJobSpec(
GroovyTemplateUtils.class.getClassLoader().getResource("ingestion_job_json_spec_template.json").getFile(),
- GroovyTemplateUtils.class.getClassLoader().getResource("job_json.config").getFile(), null);
+ GroovyTemplateUtils.class.getClassLoader().getResource("job_json.config").getFile(), null, null);
Assert.assertEquals(spec.getInputDirURI(), "file:///path/to/input/2020/07/22");
Assert.assertEquals(spec.getOutputDirURI(), "file:///path/to/output/2020/07/22");
Assert.assertEquals(spec.getSegmentCreationJobParallelism(), 0);
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/LaunchDataIngestionJobCommand.java b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/LaunchDataIngestionJobCommand.java
index c21f312104..bb58a07ca8 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/LaunchDataIngestionJobCommand.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/LaunchDataIngestionJobCommand.java
@@ -109,7 +109,7 @@ public class LaunchDataIngestionJobCommand extends AbstractBaseAdminCommand impl
SegmentGenerationJobSpec spec;
try {
spec = IngestionJobLauncher.getSegmentGenerationJobSpec(jobSpecFilePath, propertyFilePath,
- GroovyTemplateUtils.getTemplateContext(_values));
+ GroovyTemplateUtils.getTemplateContext(_values), System.getenv());
} catch (Exception e) {
LOGGER.error("Got exception to generate IngestionJobSpec for data ingestion job - ", e);
throw e;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org