[GitHub] [incubator-pinot] siddharthteotia commented on a change in pull request #4368: 3891: Check for validity of segment start/end time during segment generation

[GitBox <gi...@apache.org>]

siddharthteotia commented on a change in pull request #4368: 3891: Check for validity of segment start/end time during segment generation
URL: https://github.com/apache/incubator-pinot/pull/4368#discussion_r302770571
 
 

 ##########
 File path: pinot-core/src/main/java/org/apache/pinot/core/segment/creator/impl/SegmentColumnarIndexCreator.java
 ##########
 @@ -391,6 +399,102 @@ void writeMetadata()
     properties.save();
   }
 
+  /**
+   * Check for the validity of segment start and end time
+   * @param startTime segment start time
+   * @param endTime segment end time
+   * @param segmentName segment name
+   */
+  private void checkTime(final SegmentGeneratorConfig config, final Object startTime,
+      final Object endTime, final String segmentName) {
+    if (!config.isCheckTimeColumnValidityDuringGeneration()) {
+      return;
+    }
+
+    if (startTime == null || endTime == null) {
+      throw new RuntimeException("Expecting non-null start/end time for segment: " + segmentName);
+    }
+
+    long start;
+    long end;
+
+    if (startTime instanceof Long && endTime instanceof Long) {
+      start = (long)startTime;
+      end = (long)endTime;
+    } else if (startTime instanceof String && endTime instanceof String) {
+      start = Long.parseLong((String)startTime);
+      end = Long.parseLong((String)endTime);
+    } else if (startTime instanceof Integer && endTime instanceof Integer) {
+      start = ((Integer) startTime).longValue();
+      end = ((Integer)endTime).longValue();
+    } else  {
+      throw new RuntimeException("Unable to interpret type of time column");
+    }
+
+    // note that handling of SimpleDateFormat (TimeColumnType.SIMPLE)
+    // is done by the caller of this function that converts the simple format
+    // into millis since epoch before calling this function for validation.
+    // For TimeColumnType.EPOCH, the time field spec could still have unit
+    // as any of the following and we need to convert to millis for doing the
+    // min-max comparison against TimeUtils.getValidMinTimeMillis() and
+    // TimeUtils.getValidMaxTimeMillis()
+    if (config.getTimeColumnType() == SegmentGeneratorConfig.TimeColumnType.EPOCH) {
+      switch (config.getSegmentTimeUnit()) {
+        case DAYS:
+          start = TimeUnit.DAYS.toMillis(start);
+          end = TimeUnit.DAYS.toMillis(end);
+          break;
+        case HOURS:
+          start = TimeUnit.HOURS.toMillis(start);
+          end = TimeUnit.HOURS.toMillis(end);
+          break;
+        case MINUTES:
+          start = TimeUnit.MINUTES.toMillis(start);
+          end = TimeUnit.MINUTES.toMillis(end);
+          break;
+        case SECONDS:
+          start = TimeUnit.SECONDS.toMillis(start);
+          end = TimeUnit.SECONDS.toMillis(end);
+          break;
+        case MICROSECONDS:
+          start = TimeUnit.MICROSECONDS.toMillis(start);
+          end = TimeUnit.MICROSECONDS.toMillis(end);
+          break;
+        case NANOSECONDS:
+          start = TimeUnit.NANOSECONDS.toMillis(start);
+          end = TimeUnit.NANOSECONDS.toMillis(end);
+          break;
+        default:
+          if (config.getSegmentTimeUnit() != TimeUnit.MILLISECONDS) {
+            final StringBuilder sb = new StringBuilder();
+            sb.append("Unexpected time unit: ").append(config.getSegmentTimeUnit())
+                .append(" for time column: ").append(config.getTimeColumnName());
+            throw new RuntimeException(sb.toString());
+          }
+      }
+    }
+
+    if (!TimeUtils.timeValueInValidRange(start) || !TimeUtils.timeValueInValidRange(end)) {
+      final Date minDate = new Date(TimeUtils.getValidMinTimeMillis());
+      final Date maxDate = new Date(TimeUtils.getValidMaxTimeMillis());
+      LOGGER.error(
 
 Review comment:
   Done

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


With regards,
Apache Git Services

---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org