[incubator-pinot] branch master updated: Ensuring partition config is valid during Hadoop preprocess (#4463)

[je...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

jenniferdai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git


The following commit(s) were added to refs/heads/master by this push:
     new a1fff55  Ensuring partition config is valid during Hadoop preprocess (#4463)
a1fff55 is described below

commit a1fff551d143e99719e66cdf73273602fc993a2f
Author: Jennifer Dai <je...@users.noreply.github.com>
AuthorDate: Tue Jul 23 14:57:37 2019 -0700

    Ensuring partition config is valid during Hadoop preprocess (#4463)
    
    * Check to make sure the partition algorithm is valid.
---
 .../java/org/apache/pinot/hadoop/job/SegmentPreprocessingJob.java  | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/SegmentPreprocessingJob.java b/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/SegmentPreprocessingJob.java
index a3f3901..65fbd73 100644
--- a/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/SegmentPreprocessingJob.java
+++ b/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/SegmentPreprocessingJob.java
@@ -41,6 +41,7 @@ import org.apache.pinot.common.config.IndexingConfig;
 import org.apache.pinot.common.config.SegmentPartitionConfig;
 import org.apache.pinot.common.config.TableConfig;
 import org.apache.pinot.common.config.TableCustomConfig;
+import org.apache.pinot.core.data.partition.PartitionFunctionFactory;
 import org.apache.pinot.hadoop.utils.PushLocation;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -286,6 +287,12 @@ public class SegmentPreprocessingJob extends BaseSegmentJob {
           .format("Partition column: %s is not found from the schema of input files.", _partitionColumn));
       Preconditions.checkArgument(_numberOfPartitions > 0, String.format("Number of partitions should be positive. Current value: %s", _numberOfPartitions));
       Preconditions.checkArgument(_partitionFunction != null, "Partition function should not be null!");
+      try {
+        PartitionFunctionFactory.PartitionFunctionType.fromString(_partitionFunction);
+      } catch (IllegalArgumentException e) {
+        _logger.error("Partition function needs to be one of Modulo, Murmur, ByteArray, HashCode, it is currently {}", _partitionColumn);
+        throw new IllegalArgumentException(e);
+      }
     }
     if (_enableSorting) {
       Preconditions.checkArgument(_sortedColumn != null, "Sorted column should not be null!");


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org