You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2018/10/05 00:01:17 UTC
[1/2] impala git commit: IMPALA-7351: Improve memory estimates for
Kudu Scan Nodes
Repository: impala
Updated Branches:
refs/heads/master 93606e604 -> 80edf3701
IMPALA-7351: Improve memory estimates for Kudu Scan Nodes
This patch adds memory estimates for kudu scan nodes based on
empirically derived estimates for the scan's memory consumption
that were added in IMPALA-7096.
Testing:
Modified resource requirements planner test.
Change-Id: If9bb52530271b0bff91311a67d222a2e9fac1229
Reviewed-on: http://gerrit.cloudera.org:8080/11440
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/3fabc2de
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/3fabc2de
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/3fabc2de
Branch: refs/heads/master
Commit: 3fabc2de4771349079bcd9dc8bdcb267f43b2a6b
Parents: 93606e6
Author: Bikramjeet Vig <bi...@cloudera.com>
Authored: Tue Aug 28 14:39:16 2018 -0700
Committer: Impala Public Jenkins <im...@cloudera.com>
Committed: Thu Oct 4 22:04:02 2018 +0000
----------------------------------------------------------------------
be/src/util/backend-gflag-util.cc | 7 ++
common/thrift/BackendGflags.thrift | 4 ++
.../org/apache/impala/planner/HdfsScanNode.java | 22 +-----
.../org/apache/impala/planner/KuduScanNode.java | 25 ++++++-
.../org/apache/impala/planner/ScanNode.java | 38 ++++++++++
.../queries/PlannerTest/kudu-selectivity.test | 52 +++++++-------
.../PlannerTest/min-max-runtime-filters.test | 24 +++----
.../PlannerTest/resource-requirements.test | 74 ++++++++++++++++++++
.../queries/PlannerTest/tpch-kudu.test | 38 +++++-----
9 files changed, 205 insertions(+), 79 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/3fabc2de/be/src/util/backend-gflag-util.cc
----------------------------------------------------------------------
diff --git a/be/src/util/backend-gflag-util.cc b/be/src/util/backend-gflag-util.cc
index 50d72bd..09975a8 100644
--- a/be/src/util/backend-gflag-util.cc
+++ b/be/src/util/backend-gflag-util.cc
@@ -63,6 +63,9 @@ DECLARE_bool(invalidate_tables_on_memory_pressure);
DECLARE_double(invalidate_tables_gc_old_gen_full_threshold);
DECLARE_double(invalidate_tables_fraction_on_memory_pressure);
DECLARE_int32(local_catalog_max_fetch_retries);
+DECLARE_int64(kudu_scanner_thread_estimated_bytes_per_column);
+DECLARE_int64(kudu_scanner_thread_max_estimated_bytes);
+
namespace impala {
Status GetThriftBackendGflags(JNIEnv* jni_env, jbyteArray* cfg_bytes) {
@@ -119,6 +122,10 @@ Status GetThriftBackendGflags(JNIEnv* jni_env, jbyteArray* cfg_bytes) {
cfg.__set_invalidate_tables_fraction_on_memory_pressure(
FLAGS_invalidate_tables_fraction_on_memory_pressure);
cfg.__set_local_catalog_max_fetch_retries(FLAGS_local_catalog_max_fetch_retries);
+ cfg.__set_kudu_scanner_thread_estimated_bytes_per_column(
+ FLAGS_kudu_scanner_thread_estimated_bytes_per_column);
+ cfg.__set_kudu_scanner_thread_max_estimated_bytes(
+ FLAGS_kudu_scanner_thread_max_estimated_bytes);
RETURN_IF_ERROR(SerializeThriftMsg(jni_env, &cfg, cfg_bytes));
return Status::OK();
}
http://git-wip-us.apache.org/repos/asf/impala/blob/3fabc2de/common/thrift/BackendGflags.thrift
----------------------------------------------------------------------
diff --git a/common/thrift/BackendGflags.thrift b/common/thrift/BackendGflags.thrift
index 10003f5..5f971c0 100644
--- a/common/thrift/BackendGflags.thrift
+++ b/common/thrift/BackendGflags.thrift
@@ -99,4 +99,8 @@ struct TBackendGflags {
36: required double invalidate_tables_fraction_on_memory_pressure
37: required i32 local_catalog_max_fetch_retries
+
+ 38: required i64 kudu_scanner_thread_estimated_bytes_per_column
+
+ 39: required i64 kudu_scanner_thread_max_estimated_bytes
}
http://git-wip-us.apache.org/repos/asf/impala/blob/3fabc2de/fe/src/main/java/org/apache/impala/planner/HdfsScanNode.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/planner/HdfsScanNode.java b/fe/src/main/java/org/apache/impala/planner/HdfsScanNode.java
index 8c10e5f..63c4307 100644
--- a/fe/src/main/java/org/apache/impala/planner/HdfsScanNode.java
+++ b/fe/src/main/java/org/apache/impala/planner/HdfsScanNode.java
@@ -140,11 +140,6 @@ public class HdfsScanNode extends ScanNode {
// threads. Corresponds to the default value of --num_threads_per_core in the backend.
private static final int MAX_THREAD_TOKENS_PER_CORE = 3;
- // Factor capturing the worst-case deviation from a uniform distribution of scan ranges
- // among nodes. The factor of 1.2 means that a particular node may have 20% more
- // scan ranges than would have been estimated assuming a uniform distribution.
- private static final double SCAN_RANGE_SKEW_FACTOR = 1.2;
-
// The minimum amount of memory we estimate a scan will use. The number is
// derived experimentally: running metadata-only Parquet count(*) scans on TPC-H
// lineitem and TPC-DS store_sales of different sizes resulted in memory consumption
@@ -1376,8 +1371,7 @@ public class HdfsScanNode extends ScanNode {
// excluding partition columns and columns that are populated from file metadata.
partitionScanRange = columnReservations.size();
} else {
- partitionScanRange = (int) Math.ceil(
- ((double) scanRangeSize / (double) numNodes_) * SCAN_RANGE_SKEW_FACTOR);
+ partitionScanRange = estimatePerHostScanRanges(scanRangeSize);
}
// From the resource management purview, we want to conservatively estimate memory
// consumption based on the partition with the highest memory requirements.
@@ -1388,18 +1382,8 @@ public class HdfsScanNode extends ScanNode {
// The non-MT scan node requires at least one scanner thread.
int requiredThreads = useMtScanNode_ ? 0 : 1;
- int maxScannerThreads;
- if (queryOptions.getMt_dop() >= 1) {
- maxScannerThreads = 1;
- } else {
- maxScannerThreads = Math.min(perHostScanRanges, RuntimeEnv.INSTANCE.getNumCores());
- // Account for the max scanner threads query option.
- if (queryOptions.isSetNum_scanner_threads() &&
- queryOptions.getNum_scanner_threads() > 0) {
- maxScannerThreads =
- Math.min(maxScannerThreads, queryOptions.getNum_scanner_threads());
- }
- }
+ int maxScannerThreads = computeMaxNumberOfScannerThreads(queryOptions,
+ perHostScanRanges);
long avgScanRangeBytes = (long) Math.ceil(totalBytes_ / (double) scanRangeSize);
// The +1 accounts for an extra I/O buffer to read past the scan range due to a
http://git-wip-us.apache.org/repos/asf/impala/blob/3fabc2de/fe/src/main/java/org/apache/impala/planner/KuduScanNode.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/planner/KuduScanNode.java b/fe/src/main/java/org/apache/impala/planner/KuduScanNode.java
index 95b31a2..56c7602 100644
--- a/fe/src/main/java/org/apache/impala/planner/KuduScanNode.java
+++ b/fe/src/main/java/org/apache/impala/planner/KuduScanNode.java
@@ -39,6 +39,7 @@ import org.apache.impala.catalog.FeKuduTable;
import org.apache.impala.catalog.KuduColumn;
import org.apache.impala.catalog.Type;
import org.apache.impala.common.ImpalaRuntimeException;
+import org.apache.impala.service.BackendConfig;
import org.apache.impala.thrift.TExplainLevel;
import org.apache.impala.thrift.TKuduScanNode;
import org.apache.impala.thrift.TNetworkAddress;
@@ -225,7 +226,7 @@ public class KuduScanNode extends ScanNode {
TScanRangeLocationList locs = new TScanRangeLocationList();
locs.setScan_range(scanRange);
- locs.locations = locations;
+ locs.setLocations(locations);
scanRangeSpecs_.addToConcrete_ranges(locs);
}
}
@@ -272,9 +273,27 @@ public class KuduScanNode extends ScanNode {
@Override
public void computeNodeResourceProfile(TQueryOptions queryOptions) {
- // TODO: add a memory estimate when we revisit memory estimates overall.
+ // The bulk of memory used by Kudu scan node is generally utilized by the
+ // RowbatchQueue plus the row batches filled in by the scanner threads and
+ // waiting to be queued into the RowbatchQueue. Due to a number of factors
+ // like variable length string columns, mem pool usage pattern,
+ // variability of the number of scanner threads being spawned and the
+ // variability of the average RowbatchQueue size, it is increasingly
+ // difficult to precisely estimate the memory usage. Therefore, we fall back
+ // to a more simpler approach of using empirically derived estimates.
+ int numOfScanRanges = scanRangeSpecs_.getConcrete_rangesSize();
+ int perHostScanRanges = estimatePerHostScanRanges(numOfScanRanges);
+ int maxScannerThreads = computeMaxNumberOfScannerThreads(queryOptions,
+ perHostScanRanges);
+ int num_cols = desc_.getSlots().size();
+ long estimated_bytes_per_column_per_thread = BackendConfig.INSTANCE.getBackendCfg().
+ kudu_scanner_thread_estimated_bytes_per_column;
+ long max_estimated_bytes_per_thread = BackendConfig.INSTANCE.getBackendCfg().
+ kudu_scanner_thread_max_estimated_bytes;
+ long mem_estimate_per_thread = Math.min(num_cols *
+ estimated_bytes_per_column_per_thread, max_estimated_bytes_per_thread);
nodeResourceProfile_ = new ResourceProfileBuilder()
- .setMemEstimateBytes(0)
+ .setMemEstimateBytes(mem_estimate_per_thread * maxScannerThreads)
.setThreadReservation(useMtScanNode_ ? 0 : 1).build();
}
http://git-wip-us.apache.org/repos/asf/impala/blob/3fabc2de/fe/src/main/java/org/apache/impala/planner/ScanNode.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/planner/ScanNode.java b/fe/src/main/java/org/apache/impala/planner/ScanNode.java
index f2daef5..76e920b 100644
--- a/fe/src/main/java/org/apache/impala/planner/ScanNode.java
+++ b/fe/src/main/java/org/apache/impala/planner/ScanNode.java
@@ -25,7 +25,9 @@ import org.apache.impala.catalog.FeTable;
import org.apache.impala.catalog.HdfsFileFormat;
import org.apache.impala.catalog.Type;
import org.apache.impala.common.NotImplementedException;
+import org.apache.impala.common.RuntimeEnv;
import org.apache.impala.thrift.TNetworkAddress;
+import org.apache.impala.thrift.TQueryOptions;
import org.apache.impala.thrift.TScanRangeSpec;
import org.apache.impala.thrift.TTableStats;
@@ -38,6 +40,13 @@ import com.google.common.collect.Lists;
* Representation of the common elements of all scan nodes.
*/
abstract public class ScanNode extends PlanNode {
+
+ // Factor capturing the worst-case deviation from a uniform distribution of scan ranges
+ // among nodes. The factor of 1.2 means that a particular node may have 20% more
+ // scan ranges than would have been estimated assuming a uniform distribution.
+ // Used for HDFS and Kudu Scan node estimations.
+ protected static final double SCAN_RANGE_SKEW_FACTOR = 1.2;
+
protected final TupleDescriptor desc_;
// Total number of rows this node is expected to process
@@ -217,6 +226,35 @@ abstract public class ScanNode extends PlanNode {
}
/**
+ * Helper function that returns the estimated number of scan ranges that would
+ * be assigned to each host based on the total number of scan ranges.
+ */
+ protected int estimatePerHostScanRanges(long totalNumOfScanRanges) {
+ return (int) Math.ceil(((double) totalNumOfScanRanges / (double) numNodes_) *
+ SCAN_RANGE_SKEW_FACTOR);
+ }
+
+ /**
+ * Helper function that returns the max number of scanner threads that can be
+ * spawned by a scan node.
+ */
+ protected int computeMaxNumberOfScannerThreads(TQueryOptions queryOptions,
+ int perHostScanRanges) {
+ // The non-MT scan node requires at least one scanner thread.
+ if (queryOptions.getMt_dop() >= 1) {
+ return 1;
+ }
+ int maxScannerThreads = Math.min(perHostScanRanges,
+ RuntimeEnv.INSTANCE.getNumCores());
+ // Account for the max scanner threads query option.
+ if (queryOptions.isSetNum_scanner_threads() &&
+ queryOptions.getNum_scanner_threads() > 0) {
+ maxScannerThreads = Math.min(maxScannerThreads,
+ queryOptions.getNum_scanner_threads());
+ }
+ return maxScannerThreads;
+ }
+ /**
* Returns true if this node has conjuncts to be evaluated by Impala against the scan
* tuple.
*/
http://git-wip-us.apache.org/repos/asf/impala/blob/3fabc2de/testdata/workloads/functional-planner/queries/PlannerTest/kudu-selectivity.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/kudu-selectivity.test b/testdata/workloads/functional-planner/queries/PlannerTest/kudu-selectivity.test
index e8c91b0..8a59968 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/kudu-selectivity.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/kudu-selectivity.test
@@ -1,13 +1,13 @@
select * from functional_kudu.zipcode_incomes where id = '8600000US00601'
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-Per-Host Resources: mem-estimate=0B mem-reservation=0B thread-reservation=2
+Per-Host Resources: mem-estimate=1.88MB mem-reservation=0B thread-reservation=2
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B thread-reservation=0
|
00:SCAN KUDU [functional_kudu.zipcode_incomes]
kudu predicates: id = '8600000US00601'
- mem-estimate=0B mem-reservation=0B thread-reservation=1
+ mem-estimate=1.88MB mem-reservation=0B thread-reservation=1
tuple-ids=0 row-size=124B cardinality=1
in pipelines: 00(GETNEXT)
---- DISTRIBUTEDPLAN
@@ -22,12 +22,12 @@ Per-Host Resources: mem-estimate=0B mem-reservation=0B thread-reservation=1
in pipelines: 00(GETNEXT)
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=0B mem-reservation=0B thread-reservation=2
+Per-Host Resources: mem-estimate=1.88MB mem-reservation=0B thread-reservation=2
DATASTREAM SINK [FRAGMENT=F01, EXCHANGE=01, UNPARTITIONED]
| mem-estimate=0B mem-reservation=0B thread-reservation=0
00:SCAN KUDU [functional_kudu.zipcode_incomes]
kudu predicates: id = '8600000US00601'
- mem-estimate=0B mem-reservation=0B thread-reservation=1
+ mem-estimate=1.88MB mem-reservation=0B thread-reservation=1
tuple-ids=0 row-size=124B cardinality=1
in pipelines: 00(GETNEXT)
====
@@ -35,14 +35,14 @@ Per-Host Resources: mem-estimate=0B mem-reservation=0B thread-reservation=2
select * from functional_kudu.zipcode_incomes where id != '1' and zip = '2'
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-Per-Host Resources: mem-estimate=0B mem-reservation=0B thread-reservation=2
+Per-Host Resources: mem-estimate=3.75MB mem-reservation=0B thread-reservation=2
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B thread-reservation=0
|
00:SCAN KUDU [functional_kudu.zipcode_incomes]
predicates: id != '1'
kudu predicates: zip = '2'
- mem-estimate=0B mem-reservation=0B thread-reservation=1
+ mem-estimate=3.75MB mem-reservation=0B thread-reservation=1
tuple-ids=0 row-size=124B cardinality=1
in pipelines: 00(GETNEXT)
---- DISTRIBUTEDPLAN
@@ -57,26 +57,26 @@ Per-Host Resources: mem-estimate=0B mem-reservation=0B thread-reservation=1
in pipelines: 00(GETNEXT)
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=0B mem-reservation=0B thread-reservation=2
+Per-Host Resources: mem-estimate=3.75MB mem-reservation=0B thread-reservation=2
DATASTREAM SINK [FRAGMENT=F01, EXCHANGE=01, UNPARTITIONED]
| mem-estimate=0B mem-reservation=0B thread-reservation=0
00:SCAN KUDU [functional_kudu.zipcode_incomes]
predicates: id != '1'
kudu predicates: zip = '2'
- mem-estimate=0B mem-reservation=0B thread-reservation=1
+ mem-estimate=3.75MB mem-reservation=0B thread-reservation=1
tuple-ids=0 row-size=124B cardinality=1
in pipelines: 00(GETNEXT)
====
select * from functional_kudu.zipcode_incomes where id > '1' and zip > '2'
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-Per-Host Resources: mem-estimate=0B mem-reservation=0B thread-reservation=2
+Per-Host Resources: mem-estimate=3.75MB mem-reservation=0B thread-reservation=2
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B thread-reservation=0
|
00:SCAN KUDU [functional_kudu.zipcode_incomes]
kudu predicates: zip > '2', id > '1'
- mem-estimate=0B mem-reservation=0B thread-reservation=1
+ mem-estimate=3.75MB mem-reservation=0B thread-reservation=1
tuple-ids=0 row-size=124B cardinality=3317
in pipelines: 00(GETNEXT)
---- DISTRIBUTEDPLAN
@@ -91,25 +91,25 @@ Per-Host Resources: mem-estimate=0B mem-reservation=0B thread-reservation=1
in pipelines: 00(GETNEXT)
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=0B mem-reservation=0B thread-reservation=2
+Per-Host Resources: mem-estimate=3.75MB mem-reservation=0B thread-reservation=2
DATASTREAM SINK [FRAGMENT=F01, EXCHANGE=01, UNPARTITIONED]
| mem-estimate=0B mem-reservation=0B thread-reservation=0
00:SCAN KUDU [functional_kudu.zipcode_incomes]
kudu predicates: zip > '2', id > '1'
- mem-estimate=0B mem-reservation=0B thread-reservation=1
+ mem-estimate=3.75MB mem-reservation=0B thread-reservation=1
tuple-ids=0 row-size=124B cardinality=3317
in pipelines: 00(GETNEXT)
====
select * from functional_kudu.zipcode_incomes where id = '1' or id = '2' or zip = '3'
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-Per-Host Resources: mem-estimate=0B mem-reservation=0B thread-reservation=2
+Per-Host Resources: mem-estimate=3.75MB mem-reservation=0B thread-reservation=2
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B thread-reservation=0
|
00:SCAN KUDU [functional_kudu.zipcode_incomes]
predicates: id IN ('1', '2') OR zip = '3'
- mem-estimate=0B mem-reservation=0B thread-reservation=1
+ mem-estimate=3.75MB mem-reservation=0B thread-reservation=1
tuple-ids=0 row-size=124B cardinality=3
in pipelines: 00(GETNEXT)
---- DISTRIBUTEDPLAN
@@ -124,12 +124,12 @@ Per-Host Resources: mem-estimate=0B mem-reservation=0B thread-reservation=1
in pipelines: 00(GETNEXT)
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=0B mem-reservation=0B thread-reservation=2
+Per-Host Resources: mem-estimate=3.75MB mem-reservation=0B thread-reservation=2
DATASTREAM SINK [FRAGMENT=F01, EXCHANGE=01, UNPARTITIONED]
| mem-estimate=0B mem-reservation=0B thread-reservation=0
00:SCAN KUDU [functional_kudu.zipcode_incomes]
predicates: id IN ('1', '2') OR zip = '3'
- mem-estimate=0B mem-reservation=0B thread-reservation=1
+ mem-estimate=3.75MB mem-reservation=0B thread-reservation=1
tuple-ids=0 row-size=124B cardinality=3
in pipelines: 00(GETNEXT)
====
@@ -159,14 +159,14 @@ string_col not in ("bar") and
id in (int_col)
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-Per-Host Resources: mem-estimate=0B mem-reservation=0B thread-reservation=2
+Per-Host Resources: mem-estimate=9.75MB mem-reservation=0B thread-reservation=2
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B thread-reservation=0
|
00:SCAN KUDU [functional_kudu.alltypes]
predicates: id IN (int_col), bigint_col IN (9999999999999999999), double_col IN (CAST('inf' AS DOUBLE)), float_col IN (CAST('NaN' AS FLOAT)), int_col IN (9999999999), smallint_col IN (99999, 2), tinyint_col IN (1, 999), bool_col IN (1), string_col NOT IN ('bar')
kudu predicates: double_col IN (0.0), float_col IN (0.0), bigint_col IN (1, 2), int_col IN (1, 2), smallint_col IN (0, 2), string_col IN ('foo', 'foo '), tinyint_col IN (1, 2), bool_col IN (TRUE)
- mem-estimate=0B mem-reservation=0B thread-reservation=1
+ mem-estimate=9.75MB mem-reservation=0B thread-reservation=1
tuple-ids=0 row-size=97B cardinality=5
in pipelines: 00(GETNEXT)
====
@@ -174,13 +174,13 @@ Per-Host Resources: mem-estimate=0B mem-reservation=0B thread-reservation=2
select * from functional_kudu.decimal_tbl where d1 in (1234, 12345);
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-Per-Host Resources: mem-estimate=0B mem-reservation=0B thread-reservation=2
+Per-Host Resources: mem-estimate=4.50MB mem-reservation=0B thread-reservation=2
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B thread-reservation=0
|
00:SCAN KUDU [functional_kudu.decimal_tbl]
kudu predicates: d1 IN (1234, 12345)
- mem-estimate=0B mem-reservation=0B thread-reservation=1
+ mem-estimate=4.50MB mem-reservation=0B thread-reservation=1
tuple-ids=0 row-size=56B cardinality=2
in pipelines: 00(GETNEXT)
====
@@ -192,14 +192,14 @@ timestamp_col > (nanoseconds_add(cast('1987-05-19 00:00:00' as timestamp), 10))
timestamp_col < (seconds_add(cast('9999-12-31 24:59:59' as timestamp), 10))
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-Per-Host Resources: mem-estimate=0B mem-reservation=0B thread-reservation=2
+Per-Host Resources: mem-estimate=9.75MB mem-reservation=0B thread-reservation=2
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B thread-reservation=0
|
00:SCAN KUDU [functional_kudu.alltypes]
predicates: CAST(date_string_col AS TINYINT) IS NULL, timestamp_col < NULL
kudu predicates: smallint_col IS NULL, tinyint_col IS NOT NULL, timestamp_col > TIMESTAMP '1987-05-19 00:00:00.000000010'
- mem-estimate=0B mem-reservation=0B thread-reservation=1
+ mem-estimate=9.75MB mem-reservation=0B thread-reservation=1
tuple-ids=0 row-size=97B cardinality=730
in pipelines: 00(GETNEXT)
====
@@ -208,13 +208,13 @@ timestamp_col in (cast('2010-03-01 00:00:00' as timestamp),
cast('2010-03-01 00:01:00' as timestamp))
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-Per-Host Resources: mem-estimate=0B mem-reservation=0B thread-reservation=2
+Per-Host Resources: mem-estimate=9.75MB mem-reservation=0B thread-reservation=2
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B thread-reservation=0
|
00:SCAN KUDU [functional_kudu.alltypes]
kudu predicates: timestamp_col IN (TIMESTAMP '2010-03-01 00:00:00', TIMESTAMP '2010-03-01 00:01:00')
- mem-estimate=0B mem-reservation=0B thread-reservation=1
+ mem-estimate=9.75MB mem-reservation=0B thread-reservation=1
tuple-ids=0 row-size=97B cardinality=1
in pipelines: 00(GETNEXT)
====
@@ -224,13 +224,13 @@ timestamp_col in (cast('2010-03-01 00:00:00' as timestamp),
cast('2010-03-01 00:01:00' as timestamp))
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-Per-Host Resources: mem-estimate=0B mem-reservation=0B thread-reservation=2
+Per-Host Resources: mem-estimate=9.75MB mem-reservation=0B thread-reservation=2
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B thread-reservation=0
|
00:SCAN KUDU [functional_kudu.alltypes]
predicates: timestamp_col IN (TIMESTAMP '2010-03-01 00:00:00', NULL, TIMESTAMP '2010-03-01 00:01:00')
- mem-estimate=0B mem-reservation=0B thread-reservation=1
+ mem-estimate=9.75MB mem-reservation=0B thread-reservation=1
tuple-ids=0 row-size=97B cardinality=3
in pipelines: 00(GETNEXT)
====
http://git-wip-us.apache.org/repos/asf/impala/blob/3fabc2de/testdata/workloads/functional-planner/queries/PlannerTest/min-max-runtime-filters.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/min-max-runtime-filters.test b/testdata/workloads/functional-planner/queries/PlannerTest/min-max-runtime-filters.test
index ae8493e..0ddccdc 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/min-max-runtime-filters.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/min-max-runtime-filters.test
@@ -3,7 +3,7 @@ select count(*) from functional_kudu.alltypes a, functional_kudu.alltypestiny b
where a.int_col = b.tinyint_col + 1 and a.string_col = b.string_col
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=11.94MB mem-reservation=1.94MB thread-reservation=3
+| Per-Host Resources: mem-estimate=13.44MB mem-reservation=1.94MB thread-reservation=3
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B thread-reservation=0
|
@@ -22,13 +22,13 @@ PLAN-ROOT SINK
| in pipelines: 00(GETNEXT), 01(OPEN)
|
|--01:SCAN KUDU [functional_kudu.alltypestiny b]
-| mem-estimate=0B mem-reservation=0B thread-reservation=1
+| mem-estimate=1.50MB mem-reservation=0B thread-reservation=1
| tuple-ids=1 row-size=18B cardinality=8
| in pipelines: 01(GETNEXT)
|
00:SCAN KUDU [functional_kudu.alltypes a]
runtime filters: RF002[min_max] -> a.string_col, RF003[min_max] -> a.int_col
- mem-estimate=0B mem-reservation=0B thread-reservation=1
+ mem-estimate=1.50MB mem-reservation=0B thread-reservation=1
tuple-ids=0 row-size=21B cardinality=7300
in pipelines: 00(GETNEXT)
====
@@ -40,7 +40,7 @@ where a.int_col + 1 = b.int_col
and a.tinyint_col is not distinct from b.tinyint_col
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=11.94MB mem-reservation=1.94MB thread-reservation=3
+| Per-Host Resources: mem-estimate=14.19MB mem-reservation=1.94MB thread-reservation=3
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B thread-reservation=0
|
@@ -59,12 +59,12 @@ PLAN-ROOT SINK
| in pipelines: 00(GETNEXT), 01(OPEN)
|
|--01:SCAN KUDU [functional_kudu.alltypestiny b]
-| mem-estimate=0B mem-reservation=0B thread-reservation=1
+| mem-estimate=2.25MB mem-reservation=0B thread-reservation=1
| tuple-ids=1 row-size=22B cardinality=8
| in pipelines: 01(GETNEXT)
|
00:SCAN KUDU [functional_kudu.alltypes a]
- mem-estimate=0B mem-reservation=0B thread-reservation=1
+ mem-estimate=2.25MB mem-reservation=0B thread-reservation=1
tuple-ids=0 row-size=22B cardinality=7300
in pipelines: 00(GETNEXT)
====
@@ -76,7 +76,7 @@ where a.tinyint_col = b.bigint_col
and cast(a.float_col as double) = b.double_col
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=11.94MB mem-reservation=1.94MB thread-reservation=3
+| Per-Host Resources: mem-estimate=14.94MB mem-reservation=1.94MB thread-reservation=3
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B thread-reservation=0
|
@@ -95,13 +95,13 @@ PLAN-ROOT SINK
| in pipelines: 00(GETNEXT), 01(OPEN)
|
|--01:SCAN KUDU [functional_kudu.alltypestiny b]
-| mem-estimate=0B mem-reservation=0B thread-reservation=1
+| mem-estimate=3.00MB mem-reservation=0B thread-reservation=1
| tuple-ids=1 row-size=34B cardinality=8
| in pipelines: 01(GETNEXT)
|
00:SCAN KUDU [functional_kudu.alltypes a]
runtime filters: RF007[min_max] -> a.tinyint_col
- mem-estimate=0B mem-reservation=0B thread-reservation=1
+ mem-estimate=3.00MB mem-reservation=0B thread-reservation=1
tuple-ids=0 row-size=26B cardinality=7300
in pipelines: 00(GETNEXT)
====
@@ -130,7 +130,7 @@ PLAN-ROOT SINK
| in pipelines: 00(GETNEXT), 02(OPEN)
|
|--02:SCAN KUDU [functional_kudu.alltypes c]
-| mem-estimate=0B mem-reservation=0B thread-reservation=1
+| mem-estimate=768.00KB mem-reservation=0B thread-reservation=1
| tuple-ids=2 row-size=4B cardinality=7300
| in pipelines: 02(GETNEXT)
|
@@ -143,7 +143,7 @@ PLAN-ROOT SINK
| in pipelines: 00(GETNEXT), 01(OPEN)
|
|--01:SCAN HDFS [functional_parquet.alltypes b]
-| partitions=24/24 files=24 size=188.92KB
+| partitions=24/24 files=24 size=199.69KB
| runtime filters: RF000[bloom] -> b.int_col
| stored statistics:
| table: rows=unavailable size=unavailable
@@ -156,7 +156,7 @@ PLAN-ROOT SINK
|
00:SCAN KUDU [functional_kudu.alltypes a]
runtime filters: RF001[min_max] -> a.int_col, RF003[min_max] -> a.int_col
- mem-estimate=0B mem-reservation=0B thread-reservation=1
+ mem-estimate=768.00KB mem-reservation=0B thread-reservation=1
tuple-ids=0 row-size=4B cardinality=7300
in pipelines: 00(GETNEXT)
====
http://git-wip-us.apache.org/repos/asf/impala/blob/3fabc2de/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test b/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test
index 6dbeb73..6eb42ef 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test
@@ -5396,3 +5396,77 @@ PLAN-ROOT SINK
tuple-ids=0 row-size=3B cardinality=11000
in pipelines: 00(GETNEXT)
====
+# Kudu Scan
+select * from functional_kudu.alltypes
+---- PLAN
+Max Per-Host Resource Reservation: Memory=0B Threads=2
+Per-Host Resource Estimates: Memory=10MB
+Codegen disabled by planner
+
+F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
+| Per-Host Resources: mem-estimate=4.88MB mem-reservation=0B thread-reservation=2
+PLAN-ROOT SINK
+| mem-estimate=0B mem-reservation=0B thread-reservation=0
+|
+00:SCAN KUDU [functional_kudu.alltypes]
+ mem-estimate=4.88MB mem-reservation=0B thread-reservation=1
+ tuple-ids=0 row-size=97B cardinality=7300
+ in pipelines: 00(GETNEXT)
+====
+# Kudu Scan of single column
+select int_col from functional_kudu.alltypes
+---- PLAN
+Max Per-Host Resource Reservation: Memory=0B Threads=2
+Per-Host Resource Estimates: Memory=10MB
+Codegen disabled by planner
+
+F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
+| Per-Host Resources: mem-estimate=384.00KB mem-reservation=0B thread-reservation=2
+PLAN-ROOT SINK
+| mem-estimate=0B mem-reservation=0B thread-reservation=0
+|
+00:SCAN KUDU [functional_kudu.alltypes]
+ mem-estimate=384.00KB mem-reservation=0B thread-reservation=1
+ tuple-ids=0 row-size=4B cardinality=7300
+ in pipelines: 00(GETNEXT)
+====
+# Kudu Scan count(*)
+select count(*) from functional_kudu.alltypes
+---- PLAN
+Max Per-Host Resource Reservation: Memory=0B Threads=2
+Per-Host Resource Estimates: Memory=10MB
+Codegen disabled by planner
+
+F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
+| Per-Host Resources: mem-estimate=10.00MB mem-reservation=0B thread-reservation=2
+PLAN-ROOT SINK
+| mem-estimate=0B mem-reservation=0B thread-reservation=0
+|
+01:AGGREGATE [FINALIZE]
+| output: count(*)
+| mem-estimate=10.00MB mem-reservation=0B spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=1 row-size=8B cardinality=1
+| in pipelines: 01(GETNEXT), 00(OPEN)
+|
+00:SCAN KUDU [functional_kudu.alltypes]
+ mem-estimate=0B mem-reservation=0B thread-reservation=1
+ tuple-ids=0 row-size=0B cardinality=7300
+ in pipelines: 00(GETNEXT)
+====
+# Kudu Scan
+select * from tpch_kudu.nation
+---- PLAN
+Max Per-Host Resource Reservation: Memory=0B Threads=2
+Per-Host Resource Estimates: Memory=10MB
+Codegen disabled by planner
+
+F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
+| Per-Host Resources: mem-estimate=1.50MB mem-reservation=0B thread-reservation=2
+PLAN-ROOT SINK
+| mem-estimate=0B mem-reservation=0B thread-reservation=0
+|
+00:SCAN KUDU [tpch_kudu.nation]
+ mem-estimate=1.50MB mem-reservation=0B thread-reservation=1
+ tuple-ids=0 row-size=117B cardinality=25
+ in pipelines: 00(GETNEXT)
+====
http://git-wip-us.apache.org/repos/asf/impala/blob/3fabc2de/testdata/workloads/functional-planner/queries/PlannerTest/tpch-kudu.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/tpch-kudu.test b/testdata/workloads/functional-planner/queries/PlannerTest/tpch-kudu.test
index 3de3714..2a5cbd7 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/tpch-kudu.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/tpch-kudu.test
@@ -82,7 +82,7 @@ order by
limit 100
---- PLAN
Max Per-Host Resource Reservation: Memory=18.31MB Threads=10
-Per-Host Resource Estimates: Memory=24MB
+Per-Host Resource Estimates: Memory=49MB
PLAN-ROOT SINK
|
18:TOP-N [LIMIT=100]
@@ -178,7 +178,7 @@ order by
limit 10
---- PLAN
Max Per-Host Resource Reservation: Memory=12.38MB Threads=4
-Per-Host Resource Estimates: Memory=20MB
+Per-Host Resource Estimates: Memory=26MB
PLAN-ROOT SINK
|
06:TOP-N [LIMIT=10]
@@ -231,7 +231,7 @@ order by
o_orderpriority
---- PLAN
Max Per-Host Resource Reservation: Memory=13.94MB Threads=3
-Per-Host Resource Estimates: Memory=22MB
+Per-Host Resource Estimates: Memory=42MB
PLAN-ROOT SINK
|
04:SORT
@@ -279,7 +279,7 @@ order by
revenue desc
---- PLAN
Max Per-Host Resource Reservation: Memory=15.38MB Threads=7
-Per-Host Resource Estimates: Memory=23MB
+Per-Host Resource Estimates: Memory=32MB
PLAN-ROOT SINK
|
12:SORT
@@ -340,7 +340,7 @@ where
and l_quantity < 24
---- PLAN
Max Per-Host Resource Reservation: Memory=0B Threads=2
-Per-Host Resource Estimates: Memory=10MB
+Per-Host Resource Estimates: Memory=16MB
PLAN-ROOT SINK
|
01:AGGREGATE [FINALIZE]
@@ -390,7 +390,7 @@ order by
l_year
---- PLAN
Max Per-Host Resource Reservation: Memory=76.69MB Threads=7
-Per-Host Resource Estimates: Memory=83MB
+Per-Host Resource Estimates: Memory=90MB
PLAN-ROOT SINK
|
12:SORT
@@ -478,7 +478,7 @@ order by
o_year
---- PLAN
Max Per-Host Resource Reservation: Memory=12.56MB Threads=9
-Per-Host Resource Estimates: Memory=18MB
+Per-Host Resource Estimates: Memory=33MB
PLAN-ROOT SINK
|
16:SORT
@@ -574,7 +574,7 @@ order by
o_year desc
---- PLAN
Max Per-Host Resource Reservation: Memory=79.50MB Threads=7
-Per-Host Resource Estimates: Memory=104MB
+Per-Host Resource Estimates: Memory=118MB
PLAN-ROOT SINK
|
12:SORT
@@ -656,7 +656,7 @@ order by
limit 20
---- PLAN
Max Per-Host Resource Reservation: Memory=44.44MB Threads=5
-Per-Host Resource Estimates: Memory=61MB
+Per-Host Resource Estimates: Memory=71MB
PLAN-ROOT SINK
|
08:TOP-N [LIMIT=20]
@@ -726,7 +726,7 @@ order by
value desc
---- PLAN
Max Per-Host Resource Reservation: Memory=9.69MB Threads=7
-Per-Host Resource Estimates: Memory=28MB
+Per-Host Resource Estimates: Memory=38MB
PLAN-ROOT SINK
|
13:SORT
@@ -807,7 +807,7 @@ order by
l_shipmode
---- PLAN
Max Per-Host Resource Reservation: Memory=35.94MB Threads=3
-Per-Host Resource Estimates: Memory=46MB
+Per-Host Resource Estimates: Memory=49MB
PLAN-ROOT SINK
|
04:SORT
@@ -891,7 +891,7 @@ where
and l_shipdate < '1995-10-01'
---- PLAN
Max Per-Host Resource Reservation: Memory=17.00MB Threads=3
-Per-Host Resource Estimates: Memory=27MB
+Per-Host Resource Estimates: Memory=33MB
PLAN-ROOT SINK
|
03:AGGREGATE [FINALIZE]
@@ -940,7 +940,7 @@ order by
s_suppkey
---- PLAN
Max Per-Host Resource Reservation: Memory=15.88MB Threads=4
-Per-Host Resource Estimates: Memory=32MB
+Per-Host Resource Estimates: Memory=42MB
PLAN-ROOT SINK
|
08:SORT
@@ -1056,7 +1056,7 @@ where
)
---- PLAN
Max Per-Host Resource Reservation: Memory=8.62MB Threads=4
-Per-Host Resource Estimates: Memory=22MB
+Per-Host Resource Estimates: Memory=24MB
PLAN-ROOT SINK
|
06:AGGREGATE [FINALIZE]
@@ -1121,7 +1121,7 @@ order by
limit 100
---- PLAN
Max Per-Host Resource Reservation: Memory=81.25MB Threads=5
-Per-Host Resource Estimates: Memory=155MB
+Per-Host Resource Estimates: Memory=158MB
PLAN-ROOT SINK
|
09:TOP-N [LIMIT=100]
@@ -1196,7 +1196,7 @@ where
)
---- PLAN
Max Per-Host Resource Reservation: Memory=2.88MB Threads=3
-Per-Host Resource Estimates: Memory=13MB
+Per-Host Resource Estimates: Memory=22MB
PLAN-ROOT SINK
|
03:AGGREGATE [FINALIZE]
@@ -1253,7 +1253,7 @@ order by
s_name
---- PLAN
Max Per-Host Resource Reservation: Memory=50.81MB Threads=6
-Per-Host Resource Estimates: Memory=51MB
+Per-Host Resource Estimates: Memory=60MB
PLAN-ROOT SINK
|
10:SORT
@@ -1339,7 +1339,7 @@ order by
limit 100
---- PLAN
Max Per-Host Resource Reservation: Memory=10.56MB Threads=7
-Per-Host Resource Estimates: Memory=12MB
+Per-Host Resource Estimates: Memory=71MB
PLAN-ROOT SINK
|
12:TOP-N [LIMIT=100]
@@ -1426,7 +1426,7 @@ order by
cntrycode
---- PLAN
Max Per-Host Resource Reservation: Memory=13.94MB Threads=4
-Per-Host Resource Estimates: Memory=22MB
+Per-Host Resource Estimates: Memory=31MB
PLAN-ROOT SINK
|
07:SORT
[2/2] impala git commit: IMPALA-7575: [DOCS] FMOD() is not the same
as the % operator
Posted by ta...@apache.org.
IMPALA-7575: [DOCS] FMOD() is not the same as the % operator
- Removed the text about FMOD being equivalent to %.
- Added a note that MOD will show as % in the query plan.
Change-Id: I3b02d3e3f556d93e1d651eaee12217d6b0e3f9e0
Reviewed-on: http://gerrit.cloudera.org:8080/11586
Reviewed-by: Tim Armstrong <ta...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/80edf370
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/80edf370
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/80edf370
Branch: refs/heads/master
Commit: 80edf37010884d6aa7edae818d8219918ea0809e
Parents: 3fabc2d
Author: Alex Rodoni <ar...@cloudera.com>
Authored: Thu Oct 4 15:44:49 2018 -0700
Committer: Alex Rodoni <ar...@cloudera.com>
Committed: Thu Oct 4 23:11:07 2018 +0000
----------------------------------------------------------------------
docs/topics/impala_math_functions.xml | 95 ++++++++++++++----------------
1 file changed, 45 insertions(+), 50 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/80edf370/docs/topics/impala_math_functions.xml
----------------------------------------------------------------------
diff --git a/docs/topics/impala_math_functions.xml b/docs/topics/impala_math_functions.xml
index 7961a11..b4f97c5 100644
--- a/docs/topics/impala_math_functions.xml
+++ b/docs/topics/impala_math_functions.xml
@@ -446,26 +446,23 @@ select factorial(-100);
<dd>
<indexterm audience="hidden">fmod() function</indexterm>
- <b>Purpose:</b> Returns the modulus of a floating-point number. Equivalent to the <codeph>%</codeph> arithmetic operator.
- <p>
- <b>Return type:</b> <codeph>float</codeph> or <codeph>double</codeph>, depending on type of arguments
- </p>
+ <b>Purpose:</b> Returns the modulus of a floating-point number.<p>
+ <b>Return type:</b>
+ <codeph>float</codeph> or <codeph>double</codeph>, depending on type
+ of arguments </p>
<p conref="../shared/impala_common.xml#common/added_in_111"/>
<p conref="../shared/impala_common.xml#common/usage_notes_blurb"/>
- <p>
- Because this function operates on <codeph>DOUBLE</codeph> or <codeph>FLOAT</codeph>
- values, it is subject to potential rounding errors for values that cannot be
- represented precisely. Prefer to use whole numbers, or values that you know
- can be represented precisely by the <codeph>DOUBLE</codeph> or <codeph>FLOAT</codeph>
- types.
- </p>
+ <p> Because this function operates on <codeph>DOUBLE</codeph> or
+ <codeph>FLOAT</codeph> values, it is subject to potential rounding
+ errors for values that cannot be represented precisely. Prefer to
+ use whole numbers, or values that you know can be represented
+ precisely by the <codeph>DOUBLE</codeph> or <codeph>FLOAT</codeph>
+ types. </p>
<p conref="../shared/impala_common.xml#common/example_blurb"/>
- <p>
- The following examples show equivalent operations with the <codeph>fmod()</codeph>
- function and the <codeph>%</codeph> arithmetic operator, for values not subject
- to any rounding error.
- </p>
-<codeblock>select fmod(10,3);
+ <p> The following examples show equivalent operations with the
+ <codeph>fmod()</codeph> function and the <codeph>%</codeph>
+ arithmetic operator, for values not subject to any rounding error. </p>
+ <codeblock>select fmod(10,3);
+-------------+
| fmod(10, 3) |
+-------------+
@@ -493,17 +490,16 @@ select 5.5 % 2;
| 1.5 |
+---------+
</codeblock>
- <p>
- The following examples show operations with the <codeph>fmod()</codeph>
- function for values that cannot be represented precisely by the
- <codeph>DOUBLE</codeph> or <codeph>FLOAT</codeph> types, and thus are
- subject to rounding error. <codeph>fmod(9.9,3.0)</codeph> returns a value
- slightly different than the expected 0.9 because of rounding.
- <codeph>fmod(9.9,3.3)</codeph> returns a value quite different from
- the expected value of 0 because of rounding error during intermediate
- calculations.
- </p>
-<codeblock>select fmod(9.9,3.0);
+ <p> The following examples show operations with the
+ <codeph>fmod()</codeph> function for values that cannot be
+ represented precisely by the <codeph>DOUBLE</codeph> or
+ <codeph>FLOAT</codeph> types, and thus are subject to rounding
+ error. <codeph>fmod(9.9,3.0)</codeph> returns a value slightly
+ different than the expected 0.9 because of rounding.
+ <codeph>fmod(9.9,3.3)</codeph> returns a value quite different
+ from the expected value of 0 because of rounding error during
+ intermediate calculations. </p>
+ <codeblock>select fmod(9.9,3.0);
+--------------------+
| fmod(9.9, 3.0) |
+--------------------+
@@ -826,31 +822,30 @@ select fmod(9.9,3.3);
<dd rev="2.2.0">
<indexterm audience="hidden">mod() function</indexterm>
- <b>Purpose:</b> Returns the modulus of a number. Equivalent to the <codeph>%</codeph> arithmetic operator.
- Works with any size integer type, any size floating-point type, and <codeph>DECIMAL</codeph>
- with any precision and scale.
- <p conref="../shared/impala_common.xml#common/return_type_same"/>
+ <b>Purpose:</b> Returns the modulus of a number. Equivalent to the
+ <codeph>%</codeph> arithmetic operator. Works with any size integer
+ type, any size floating-point type, and <codeph>DECIMAL</codeph> with
+ any precision and scale. <p
+ conref="../shared/impala_common.xml#common/return_type_same"/>
<p conref="../shared/impala_common.xml#common/added_in_220"/>
<p conref="../shared/impala_common.xml#common/usage_notes_blurb"/>
- <p>
- Because this function works with <codeph>DECIMAL</codeph> values, prefer it over <codeph>fmod()</codeph>
- when working with fractional values. It is not subject to the rounding errors that make
- <codeph>fmod()</codeph> problematic with floating-point numbers.
- The <codeph>%</codeph> arithmetic operator now uses the <codeph>mod()</codeph> function
- in cases where its arguments can be interpreted as <codeph>DECIMAL</codeph> values,
- increasing the accuracy of that operator.
- </p>
+ <p> Because this function works with <codeph>DECIMAL</codeph> values,
+ prefer it over <codeph>fmod()</codeph> when working with fractional
+ values. It is not subject to the rounding errors that make
+ <codeph>fmod()</codeph> problematic with floating-point
+ numbers.</p><p rev="IMPALA-6202">Query plans shows the
+ <codeph>MOD()</codeph> function as the <codeph>%</codeph>
+ operator.</p>
<p conref="../shared/impala_common.xml#common/example_blurb"/>
- <p>
- The following examples show how the <codeph>mod()</codeph> function works for
- whole numbers and fractional values, and how the <codeph>%</codeph> operator
- works the same way. In the case of <codeph>mod(9.9,3)</codeph>,
- the type conversion for the second argument results in the first argument
- being interpreted as <codeph>DOUBLE</codeph>, so to produce an accurate
- <codeph>DECIMAL</codeph> result requires casting the second argument
- or writing it as a <codeph>DECIMAL</codeph> literal, 3.0.
- </p>
-<codeblock>select mod(10,3);
+ <p> The following examples show how the <codeph>mod()</codeph>
+ function works for whole numbers and fractional values, and how the
+ <codeph>%</codeph> operator works the same way. In the case of
+ <codeph>mod(9.9,3)</codeph>, the type conversion for the second
+ argument results in the first argument being interpreted as
+ <codeph>DOUBLE</codeph>, so to produce an accurate
+ <codeph>DECIMAL</codeph> result requires casting the second
+ argument or writing it as a <codeph>DECIMAL</codeph> literal, 3.0. </p>
+ <codeblock>select mod(10,3);
+-------------+
| fmod(10, 3) |
+-------------+