You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by je...@apache.org on 2019/07/23 20:30:00 UTC
[incubator-pinot] branch invalid-partition-configs created (now
f1995fb)
This is an automated email from the ASF dual-hosted git repository.
jenniferdai pushed a change to branch invalid-partition-configs
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git.
at f1995fb Ensuring partition config is valid during preprocess
This branch includes the following new commits:
new f1995fb Ensuring partition config is valid during preprocess
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org
[incubator-pinot] 01/01: Ensuring partition config is valid during
preprocess
Posted by je...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jenniferdai pushed a commit to branch invalid-partition-configs
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit f1995fbebb803f6abc5e746d6385e06dca70b479
Author: Jennifer Dai <jd...@linkedin.com>
AuthorDate: Tue Jul 23 13:29:49 2019 -0700
Ensuring partition config is valid during preprocess
---
.../java/org/apache/pinot/hadoop/job/SegmentPreprocessingJob.java | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/SegmentPreprocessingJob.java b/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/SegmentPreprocessingJob.java
index a3f3901..a2f8d4b 100644
--- a/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/SegmentPreprocessingJob.java
+++ b/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/SegmentPreprocessingJob.java
@@ -41,6 +41,7 @@ import org.apache.pinot.common.config.IndexingConfig;
import org.apache.pinot.common.config.SegmentPartitionConfig;
import org.apache.pinot.common.config.TableConfig;
import org.apache.pinot.common.config.TableCustomConfig;
+import org.apache.pinot.core.data.partition.PartitionFunctionFactory;
import org.apache.pinot.hadoop.utils.PushLocation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -286,6 +287,11 @@ public class SegmentPreprocessingJob extends BaseSegmentJob {
.format("Partition column: %s is not found from the schema of input files.", _partitionColumn));
Preconditions.checkArgument(_numberOfPartitions > 0, String.format("Number of partitions should be positive. Current value: %s", _numberOfPartitions));
Preconditions.checkArgument(_partitionFunction != null, "Partition function should not be null!");
+ try {
+ PartitionFunctionFactory.PartitionFunctionType.fromString(_partitionFunction);
+ } catch (IllegalArgumentException e) {
+ _logger.info("Partition function needs to be one of Modulo, Murmur, ByteArray, HashCode");
+ }
}
if (_enableSorting) {
Preconditions.checkArgument(_sortedColumn != null, "Sorted column should not be null!");
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org