You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by xi...@apache.org on 2021/01/21 00:10:01 UTC
[incubator-pinot] branch
removing_minion_segment_generator_for_table_config created (now bd3ee84)
This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a change to branch removing_minion_segment_generator_for_table_config
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git.
at bd3ee84 Update SegmentGenerationAndPushTaskGenerator to set schemaURI and tableConfigURI by default
This branch includes the following new commits:
new bd3ee84 Update SegmentGenerationAndPushTaskGenerator to set schemaURI and tableConfigURI by default
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org
[incubator-pinot] 01/01: Update
SegmentGenerationAndPushTaskGenerator to set schemaURI and tableConfigURI
by default
Posted by xi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch removing_minion_segment_generator_for_table_config
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit bd3ee846238b69cd0b478b830da9aed0218f6ce2
Author: Xiang Fu <fx...@gmail.com>
AuthorDate: Wed Jan 20 16:09:29 2021 -0800
Update SegmentGenerationAndPushTaskGenerator to set schemaURI and tableConfigURI by default
---
.../generator/SegmentGenerationAndPushTaskGenerator.java | 9 +++++----
.../minion/executor/SegmentGenerationAndPushTaskExecutor.java | 10 +++++++++-
.../pinot/spi/ingestion/batch/BatchConfigProperties.java | 4 ++--
3 files changed, 16 insertions(+), 7 deletions(-)
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/minion/generator/SegmentGenerationAndPushTaskGenerator.java b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/minion/generator/SegmentGenerationAndPushTaskGenerator.java
index 41b7740..07a221d 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/minion/generator/SegmentGenerationAndPushTaskGenerator.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/minion/generator/SegmentGenerationAndPushTaskGenerator.java
@@ -36,6 +36,7 @@ import java.util.Map;
import java.util.Set;
import org.apache.helix.task.TaskState;
import org.apache.pinot.common.metadata.segment.OfflineSegmentZKMetadata;
+import org.apache.pinot.controller.helix.ControllerRequestURLBuilder;
import org.apache.pinot.controller.helix.core.minion.ClusterInfoAccessor;
import org.apache.pinot.core.common.MinionConstants;
import org.apache.pinot.core.minion.PinotTaskConfig;
@@ -225,10 +226,10 @@ public class SegmentGenerationAndPushTaskGenerator implements PinotTaskGenerator
URI outputSegmentDirURI = getRelativeOutputPath(inputDirURI, inputFileURI, outputDirURI);
singleFileGenerationTaskConfig.put(BatchConfigProperties.OUTPUT_SEGMENT_DIR_URI, outputSegmentDirURI.toString());
}
- singleFileGenerationTaskConfig.put(BatchConfigProperties.SCHEMA,
- JsonUtils.objectToString(_clusterInfoAccessor.getTableSchema(offlineTableName)));
- singleFileGenerationTaskConfig.put(BatchConfigProperties.TABLE_CONFIGS,
- JsonUtils.objectToString(_clusterInfoAccessor.getTableConfig(offlineTableName)));
+ ControllerRequestURLBuilder controllerRequestURLBuilder =
+ ControllerRequestURLBuilder.baseUrl(_clusterInfoAccessor.getVipUrl());
+ singleFileGenerationTaskConfig.put(BatchConfigProperties.SCHEMA_URI, controllerRequestURLBuilder.forTableSchemaGet(offlineTableName));
+ singleFileGenerationTaskConfig.put(BatchConfigProperties.TABLE_CONFIGS_URI, controllerRequestURLBuilder.forTableGet(offlineTableName));
singleFileGenerationTaskConfig.put(BatchConfigProperties.SEQUENCE_ID, String.valueOf(sequenceID));
singleFileGenerationTaskConfig
.put(BatchConfigProperties.SEGMENT_NAME_GENERATOR_TYPE, BatchConfigProperties.SegmentNameGeneratorType.SIMPLE);
diff --git a/pinot-minion/src/main/java/org/apache/pinot/minion/executor/SegmentGenerationAndPushTaskExecutor.java b/pinot-minion/src/main/java/org/apache/pinot/minion/executor/SegmentGenerationAndPushTaskExecutor.java
index 535248f..ca6444d 100644
--- a/pinot-minion/src/main/java/org/apache/pinot/minion/executor/SegmentGenerationAndPushTaskExecutor.java
+++ b/pinot-minion/src/main/java/org/apache/pinot/minion/executor/SegmentGenerationAndPushTaskExecutor.java
@@ -292,7 +292,15 @@ public class SegmentGenerationAndPushTaskExecutor extends BaseTaskExecutor {
"Missing schema for segment generation job: please set `schema` or `schemaURI` in task config.");
}
taskSpec.setSchema(schema);
- JsonNode tableConfig = JsonUtils.stringToJsonNode(taskConfigs.get(BatchConfigProperties.TABLE_CONFIGS));
+ JsonNode tableConfig;
+ if (taskConfigs.containsKey(BatchConfigProperties.TABLE_CONFIGS)) {
+ tableConfig = JsonUtils.stringToJsonNode(taskConfigs.get(BatchConfigProperties.TABLE_CONFIGS));
+ } else if (taskConfigs.containsKey(BatchConfigProperties.TABLE_CONFIGS_URI)) {
+ tableConfig = SegmentGenerationUtils.getTableConfig(taskConfigs.get(BatchConfigProperties.TABLE_CONFIGS_URI)).toJsonNode();
+ } else {
+ throw new RuntimeException(
+ "Missing schema for segment generation job: please set `tableConfigs` or `tableConfigsURI` in task config.");
+ }
taskSpec.setTableConfig(tableConfig);
taskSpec.setSequenceId(Integer.parseInt(taskConfigs.get(BatchConfigProperties.SEQUENCE_ID)));
SegmentNameGeneratorSpec segmentNameGeneratorSpec = new SegmentNameGeneratorSpec();
diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/ingestion/batch/BatchConfigProperties.java b/pinot-spi/src/main/java/org/apache/pinot/spi/ingestion/batch/BatchConfigProperties.java
index be0b917..5a28884 100644
--- a/pinot-spi/src/main/java/org/apache/pinot/spi/ingestion/batch/BatchConfigProperties.java
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/ingestion/batch/BatchConfigProperties.java
@@ -22,8 +22,6 @@ package org.apache.pinot.spi.ingestion.batch;
* Defines all the keys used in the batch configs map
*/
public class BatchConfigProperties {
-
- public static final String TABLE_CONFIGS = "tableConfigs";
public static final String TABLE_NAME = "tableName";
public static final String INPUT_DIR_URI = "inputDirURI";
@@ -38,6 +36,8 @@ public class BatchConfigProperties {
public static final String RECORD_READER_CLASS = "recordReader.className";
public static final String RECORD_READER_CONFIG_CLASS = "recordReader.configClassName";
public static final String RECORD_READER_PROP_PREFIX = "recordReader.prop";
+ public static final String TABLE_CONFIGS = "tableConfigs";
+ public static final String TABLE_CONFIGS_URI = "tableConfigsURI";
public static final String SCHEMA = "schema";
public static final String SCHEMA_URI = "schemaURI";
public static final String SEQUENCE_ID = "sequenceId";
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org