[GitHub] [incubator-pinot] siddharthteotia commented on a change in pull request #4910: Support ORDER BY and BYTES in data anonymizer tool

[GitBox <gi...@apache.org>]

siddharthteotia commented on a change in pull request #4910: Support ORDER BY and BYTES in data anonymizer tool
URL: https://github.com/apache/incubator-pinot/pull/4910#discussion_r357421601
 
 

 ##########
 File path: pinot-tools/src/main/java/org/apache/pinot/tools/PinotDataAndQueryAnonymizer.java
 ##########
 @@ -530,68 +544,107 @@ private void generateDerivedValuesForGlobalDictionary(OrigAndDerivedValueHolder
     int cardinality = valueHolder._index;
     switch (valueHolder._dataType) {
       case INT:
-        valueHolder.setDerivedValues(generateDerivedIntValues(cardinality));
+        valueHolder.setDerivedValues(generateDerivedIntValuesForGD(cardinality));
         break;
       case LONG:
-        valueHolder.setDerivedValues(generateDerivedLongValues(cardinality));
+        valueHolder.setDerivedValues(generateDerivedLongValuesForGD(cardinality));
         break;
       case FLOAT:
-        valueHolder.setDerivedValues(generateDerivedFloatValues(cardinality));
+        valueHolder.setDerivedValues(generateDerivedFloatValuesForGD(cardinality));
         break;
       case DOUBLE:
-        valueHolder.setDerivedValues(generateDerivedDoubleValues(cardinality));
+        valueHolder.setDerivedValues(generateDerivedDoubleValuesForGD(cardinality));
         break;
       case STRING:
-        valueHolder.setDerivedValues(generateDerivedStringValues(valueHolder));
+        valueHolder.setDerivedValues(generateDerivedStringValuesForGD(valueHolder));
+        break;
+      case BYTES:
+        valueHolder.setDerivedValues(generateDerivedByteValuesForGD(valueHolder));
         break;
       default:
         throw new UnsupportedOperationException("global dictionary currently does not support: " + valueHolder._dataType.name());
     }
   }
 
-  private Integer[] generateDerivedIntValues(int cardinality) {
+  private Integer[] generateDerivedIntValuesForGD(int cardinality) {
     Integer[] values = new Integer[cardinality];
     for (int i = 0; i < cardinality; i++) {
       values[i] = INT_BASE_VALUE + i;
     }
     return values;
   }
 
-  private Long[] generateDerivedLongValues(int cardinality) {
+  private Long[] generateDerivedLongValuesForGD(int cardinality) {
     Long[] values = new Long[cardinality];
     for (int i = 0; i < cardinality; i++) {
       values[i] = LONG_BASE_VALUE + (long)i;
     }
     return values;
   }
 
-  private Float[] generateDerivedFloatValues(int cardinality) {
+  private Float[] generateDerivedFloatValuesForGD(int cardinality) {
     Float[] values = new Float[cardinality];
     for (int i = 0; i < cardinality; i++) {
       values[i] = FLOAT_BASE_VALUE + (float)i;
     }
     return values;
   }
 
-  private Double[] generateDerivedDoubleValues(int cardinality) {
+  private Double[] generateDerivedDoubleValuesForGD(int cardinality) {
     Double[] values = new Double[cardinality];
     for (int i = 0; i < cardinality; i++) {
       values[i] = DOUBLE_BASE_VALUE + (double)i;
     }
     return values;
   }
 
-  private String[] generateDerivedStringValues(OrigAndDerivedValueHolder valueHolder) {
+  private String[] generateDerivedStringValuesForGD(OrigAndDerivedValueHolder valueHolder) {
     int cardinality = valueHolder._index;
     String[] values = new String[cardinality];
     for (int i = 0; i < cardinality; i++) {
-      int origValLength = ((String)valueHolder._origValues[i]).length();
+      String val = (String)valueHolder._origValues[i];
+      int origValLength;
+      if (val == null || val.equals("") || val.equals(" ") || val.equals("null")) {
 
 Review comment:
   done

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


With regards,
Apache Git Services

---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org