You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by bo...@apache.org on 2018/07/30 20:54:06 UTC
[drill] branch 1.14.0 updated (c705271 -> 2a5e983)
This is an automated email from the ASF dual-hosted git repository.
boaz pushed a change to branch 1.14.0
in repository https://gitbox.apache.org/repos/asf/drill.git.
discard c705271 [maven-release-plugin] prepare release drill-1.14.0
new 2a5e983 DRILL-6641: Fix columnValueCounts in ParquetGroupScanStatistics when ParquetGroupScan has RowGroupInfo without column statistics
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (c705271)
\
N -- N -- N refs/heads/1.14.0 (2a5e983)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
common/pom.xml | 2 +-
contrib/data/pom.xml | 2 +-
contrib/data/tpch-sample-data/pom.xml | 2 +-
contrib/format-maprdb/pom.xml | 2 +-
contrib/gis/pom.xml | 2 +-
contrib/pom.xml | 2 +-
contrib/sqlline/pom.xml | 2 +-
contrib/storage-hbase/pom.xml | 2 +-
contrib/storage-hive/core/pom.xml | 2 +-
contrib/storage-hive/hive-exec-shade/pom.xml | 2 +-
contrib/storage-hive/pom.xml | 2 +-
contrib/storage-jdbc/pom.xml | 2 +-
contrib/storage-kafka/pom.xml | 2 +-
contrib/storage-kudu/pom.xml | 2 +-
contrib/storage-mongo/pom.xml | 2 +-
contrib/storage-opentsdb/pom.xml | 2 +-
distribution/pom.xml | 2 +-
drill-yarn/pom.xml | 2 +-
exec/java-exec/pom.xml | 2 +-
.../store/parquet/ParquetGroupScanStatistics.java | 26 ++++++++++------------
exec/jdbc-all/pom.xml | 2 +-
exec/jdbc/pom.xml | 2 +-
exec/memory/base/pom.xml | 2 +-
exec/memory/pom.xml | 2 +-
exec/pom.xml | 2 +-
exec/rpc/pom.xml | 2 +-
exec/vector/pom.xml | 2 +-
logical/pom.xml | 2 +-
pom.xml | 10 ++++-----
protocol/pom.xml | 2 +-
tools/fmpp/pom.xml | 2 +-
tools/pom.xml | 2 +-
32 files changed, 47 insertions(+), 49 deletions(-)
[drill] 01/01: DRILL-6641: Fix columnValueCounts in
ParquetGroupScanStatistics when ParquetGroupScan has RowGroupInfo without
column statistics
Posted by bo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
boaz pushed a commit to branch 1.14.0
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 2a5e983a7294633fd0e3e274bcd68d1ad4080fd3
Author: Volodymyr Vysotskyi <vv...@gmail.com>
AuthorDate: Fri Jul 27 13:49:09 2018 +0300
DRILL-6641: Fix columnValueCounts in ParquetGroupScanStatistics when ParquetGroupScan has RowGroupInfo without column statistics
---
.../store/parquet/ParquetGroupScanStatistics.java | 26 ++++++++++------------
1 file changed, 12 insertions(+), 14 deletions(-)
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScanStatistics.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScanStatistics.java
index f7d5687..9381043 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScanStatistics.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScanStatistics.java
@@ -17,6 +17,7 @@
*/
package org.apache.drill.exec.store.parquet;
+import org.apache.commons.lang3.mutable.MutableLong;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.exec.physical.base.GroupScan;
@@ -44,7 +45,7 @@ public class ParquetGroupScanStatistics {
// only for partition columns : value is unique for each partition
private Map<SchemaPath, TypeProtos.MajorType> partitionColTypeMap;
// total number of non-null value for each column in parquet files
- private Map<SchemaPath, Long> columnValueCounts;
+ private Map<SchemaPath, MutableLong> columnValueCounts;
// total number of rows (obtained from parquet footer)
private long rowCount;
@@ -61,7 +62,8 @@ public class ParquetGroupScanStatistics {
}
public long getColumnValueCount(SchemaPath column) {
- return columnValueCounts.containsKey(column) ? columnValueCounts.get(column) : 0;
+ MutableLong count = columnValueCounts.get(column);
+ return count != null ? count.getValue() : 0;
}
public List<SchemaPath> getPartitionColumns() {
@@ -87,19 +89,15 @@ public class ParquetGroupScanStatistics {
long rowCount = rowGroup.getRowCount();
for (ColumnMetadata column : rowGroup.getColumns()) {
SchemaPath schemaPath = SchemaPath.getCompoundPath(column.getName());
- Long previousCount = columnValueCounts.get(schemaPath);
- if (previousCount != null) {
- if (previousCount != GroupScan.NO_COLUMN_STATS && column.isNumNullsSet()) {
- Long newCount = rowCount - column.getNulls();
- columnValueCounts.put(schemaPath, columnValueCounts.get(schemaPath) + newCount);
- }
+ MutableLong emptyCount = new MutableLong();
+ MutableLong previousCount = columnValueCounts.putIfAbsent(schemaPath, emptyCount);
+ if (previousCount == null) {
+ previousCount = emptyCount;
+ }
+ if (previousCount.longValue() != GroupScan.NO_COLUMN_STATS && column.isNumNullsSet()) {
+ previousCount.add(rowCount - column.getNulls());
} else {
- if (column.isNumNullsSet()) {
- Long newCount = rowCount - column.getNulls();
- columnValueCounts.put(schemaPath, newCount);
- } else {
- columnValueCounts.put(schemaPath, GroupScan.NO_COLUMN_STATS);
- }
+ previousCount.setValue(GroupScan.NO_COLUMN_STATS);
}
boolean partitionColumn = checkForPartitionColumn(column, first, rowCount, parquetTableMetadata);
if (partitionColumn) {