[GitHub] amansinha100 commented on a change in pull request #729: Drill 1328: Support table statistics for Parquet

[GitBox <gi...@apache.org>]

amansinha100 commented on a change in pull request #729: Drill 1328: Support table statistics for Parquet
URL: https://github.com/apache/drill/pull/729#discussion_r250438442
 
 

 ##########
 File path: exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/StatisticsAggrFunctions.java
 ##########
 @@ -0,0 +1,5516 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * This class is automatically generated from AggrTypeFunctions2.tdd using FreeMarker.
+ */
+
+package org.apache.drill.exec.expr.fn.impl;
+
+import io.netty.buffer.DrillBuf;
+import javax.inject.Inject;
+import org.apache.drill.exec.expr.DrillAggFunc;
+import org.apache.drill.exec.expr.DrillSimpleFunc;
+import org.apache.drill.exec.expr.annotations.FunctionTemplate;
+import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope;
+import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling;
+import org.apache.drill.exec.expr.annotations.Output;
+import org.apache.drill.exec.expr.annotations.Param;
+import org.apache.drill.exec.expr.annotations.Workspace;
+import org.apache.drill.exec.expr.holders.BigIntHolder;
+import org.apache.drill.exec.expr.holders.BitHolder;
+import org.apache.drill.exec.expr.holders.DateHolder;
+import org.apache.drill.exec.expr.holders.Decimal18Holder;
+import org.apache.drill.exec.expr.holders.Decimal28DenseHolder;
+import org.apache.drill.exec.expr.holders.Decimal28SparseHolder;
+import org.apache.drill.exec.expr.holders.Decimal38DenseHolder;
+import org.apache.drill.exec.expr.holders.Decimal38SparseHolder;
+import org.apache.drill.exec.expr.holders.Decimal9Holder;
+import org.apache.drill.exec.expr.holders.Float4Holder;
+import org.apache.drill.exec.expr.holders.Float8Holder;
+import org.apache.drill.exec.expr.holders.IntHolder;
+import org.apache.drill.exec.expr.holders.IntervalHolder;
+import org.apache.drill.exec.expr.holders.NullableBigIntHolder;
+import org.apache.drill.exec.expr.holders.NullableBitHolder;
+import org.apache.drill.exec.expr.holders.NullableDateHolder;
+import org.apache.drill.exec.expr.holders.NullableDecimal18Holder;
+import org.apache.drill.exec.expr.holders.NullableDecimal28DenseHolder;
+import org.apache.drill.exec.expr.holders.NullableDecimal28SparseHolder;
+import org.apache.drill.exec.expr.holders.NullableDecimal38DenseHolder;
+import org.apache.drill.exec.expr.holders.NullableDecimal38SparseHolder;
+import org.apache.drill.exec.expr.holders.NullableDecimal9Holder;
+import org.apache.drill.exec.expr.holders.NullableFloat4Holder;
+import org.apache.drill.exec.expr.holders.NullableFloat8Holder;
+import org.apache.drill.exec.expr.holders.NullableIntHolder;
+import org.apache.drill.exec.expr.holders.NullableIntervalHolder;
+import org.apache.drill.exec.expr.holders.NullableTimeHolder;
+import org.apache.drill.exec.expr.holders.NullableTimeStampHolder;
+import org.apache.drill.exec.expr.holders.NullableVar16CharHolder;
+import org.apache.drill.exec.expr.holders.NullableVarBinaryHolder;
+import org.apache.drill.exec.expr.holders.NullableVarCharHolder;
+import org.apache.drill.exec.expr.holders.ObjectHolder;
+import org.apache.drill.exec.expr.holders.TimeHolder;
+import org.apache.drill.exec.expr.holders.TimeStampHolder;
+import org.apache.drill.exec.expr.holders.Var16CharHolder;
+import org.apache.drill.exec.expr.holders.VarBinaryHolder;
+import org.apache.drill.exec.expr.holders.VarCharHolder;
+import org.apache.drill.exec.server.options.OptionManager;
+import org.apache.drill.exec.vector.complex.reader.FieldReader;
+
+@SuppressWarnings("unused")
+public class StatisticsAggrFunctions {
+  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(StatisticsAggrFunctions.class);
+
+  /* IMPORTANT NOTE: Please make sure to create a new function for each datatype. See the examples below.
+  * This will result in more performant generated code. Use switch-case/if-else statements judiciously
+  * as it MAY cause the generated code to slow down considerably.
+  * */
+  @FunctionTemplate(name = "rowcount", scope = FunctionTemplate.FunctionScope.POINT_AGGREGATE)
+  public static class RowCount implements DrillAggFunc {
+    @Param FieldReader in;
+    @Workspace BigIntHolder count;
+    @Output NullableBigIntHolder out;
+
+    @Override
+    public void setup() {
+      count = new BigIntHolder();
+    }
+
+    @Override
+    public void add() {
+      count.value++;
+    }
+
+    @Override
+    public void output() {
+      out.isSet = 1;
+      out.value = count.value;
+    }
+
+    @Override
+    public void reset() {
+      count.value = 0;
+    }
+  }
+
+  @FunctionTemplate(name = "nonnullrowcount", scope = FunctionTemplate.FunctionScope.POINT_AGGREGATE)
+  public static class NonNullRowCount implements DrillAggFunc {
+    @Param FieldReader in;
+    @Workspace BigIntHolder count;
+    @Output NullableBigIntHolder out;
+
+    @Override
+    public void setup() {
+      count = new BigIntHolder();
+    }
+
+    @Override
+    public void add() {
+      if (in.isSet()) {
+        count.value++;
+      }
+    }
+
+    @Override
+    public void output() {
+      out.isSet = 1;
+      out.value = count.value;
+    }
+
+    @Override
+    public void reset() {
+      count.value = 0;
+    }
+  }
+
+  /**
+   * The log2m parameter defines the accuracy of the counter.  The larger the
+   * log2m the better the accuracy.
+   * accuracy = 1.04/sqrt(2^log2m)
+   * where
+   * log2m - the number of bits to use as the basis for the HLL instance
+   */
+  @FunctionTemplate(name = "hll", scope = FunctionTemplate.FunctionScope.POINT_AGGREGATE)
+  public static class HllFieldReader implements DrillAggFunc {
+    @Param FieldReader in;
+    @Workspace ObjectHolder work;
+    @Output NullableVarBinaryHolder out;
+    @Inject OptionManager options;
+    @Inject DrillBuf buffer;
+    @Workspace IntHolder hllAccuracy;
+
+    @Override
+    public void setup() {
+      work = new ObjectHolder();
+      hllAccuracy.value = (int) options.getLong(org.apache.drill.exec.ExecConstants.HLL_ACCURACY);
+      work.obj = new com.clearspring.analytics.stream.cardinality.HyperLogLog(hllAccuracy.value);
+    }
+
+    @Override
+    public void add() {
+      if (work.obj != null) {
+        com.clearspring.analytics.stream.cardinality.HyperLogLog hll =
+            (com.clearspring.analytics.stream.cardinality.HyperLogLog) work.obj;
+        int mode = in.getType().getMode().getNumber();
+        int type = in.getType().getMinorType().getNumber();
+
+        switch (mode) {
+          case org.apache.drill.common.types.TypeProtos.DataMode.OPTIONAL_VALUE:
+            if (!in.isSet()) {
+              hll.offer(null);
+              break;
+            }
+            // fall through //
+          case org.apache.drill.common.types.TypeProtos.DataMode.REQUIRED_VALUE:
+            switch (type) {
+              case org.apache.drill.common.types.TypeProtos.MinorType.VARCHAR_VALUE:
+                hll.offer(in.readText().toString());
+                break;
+              case org.apache.drill.common.types.TypeProtos.MinorType.BIGINT_VALUE:
+                hll.offer(in.readLong());
+                break;
+              default:
+                work.obj = null;
+            }
+            break;
+          default:
+            work.obj = null;
+        }
+      }
+    }
+
+    @Override
+    public void output() {
+      if (work.obj != null) {
+        com.clearspring.analytics.stream.cardinality.HyperLogLog hll =
+            (com.clearspring.analytics.stream.cardinality.HyperLogLog) work.obj;
+
+        try {
+          byte[] ba = hll.getBytes();
+          out.buffer = buffer.reallocIfNeeded(ba.length);
+          out.start = 0;
+          out.end = ba.length;
+          out.buffer.setBytes(0, ba);
+          out.isSet = 1;
+        } catch (java.io.IOException e) {
+          throw new org.apache.drill.common.exceptions.DrillRuntimeException("Failed to get HyperLogLog output", e);
+        }
+      } else {
+        out.isSet = 0;
+      }
+    }
+
+    @Override
+    public void reset() {
+      work.obj = new com.clearspring.analytics.stream.cardinality.HyperLogLog(hllAccuracy.value);
+    }
+  }
+
+  @FunctionTemplate(name = "ndv", scope = FunctionTemplate.FunctionScope.POINT_AGGREGATE)
 
 Review comment:
   There is the "ndv" functions and the "hll" functions but only the latter are actually going to be used for the ANALYZE command, so let's only keep that.   
   A related comment is that since we are computing the approximate count distinct, in the future we may externalize it using the SQL standard function **approx_count_distinct** that Oracle, MS SQL Server and others support.  So, we anyways should remove "ndv" since it can be misinterpreted to give actual num distinct values rather than approximate. 

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


With regards,
Apache Git Services