You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by cs...@apache.org on 2022/08/30 10:57:02 UTC
[impala] branch master updated (a26b98612 -> 876829951)
This is an automated email from the ASF dual-hosted git repository.
csringhofer pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
from a26b98612 IMPALA-11505: Fix flaky test test_kudu_txn_abort_partition_lock
new e8ebf0be6 IMPALA-11496: Push down date literals to Kudu scanner
new 876829951 IMPALA-11483: [DOCS] Document EXPIRE SNAPSHOTS functionality
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
docs/topics/impala_iceberg.xml | 32 ++++++++++
.../org/apache/impala/planner/KuduScanNode.java | 12 +++-
.../queries/PlannerTest/kudu.test | 68 ++++++++++++++++++++++
3 files changed, 109 insertions(+), 3 deletions(-)
[impala] 02/02: IMPALA-11483: [DOCS] Document EXPIRE SNAPSHOTS functionality
Posted by cs...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
csringhofer pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 876829951dc23ac044e79c3ed3088207d812d3bb
Author: Tamas Mate <tm...@cloudera.com>
AuthorDate: Thu Aug 11 16:43:44 2022 +0200
IMPALA-11483: [DOCS] Document EXPIRE SNAPSHOTS functionality
This commit documents the Iceberg expire snapshot functionality.
Testing:
- Built the docs locally and verified the generated html.
Change-Id: Iddd927dd1b6ee3b99dd29982018a610389e64836
Reviewed-on: http://gerrit.cloudera.org:8080/18838
Reviewed-by: Tamas Mate <tm...@apache.org>
Tested-by: Tamas Mate <tm...@apache.org>
---
docs/topics/impala_iceberg.xml | 32 ++++++++++++++++++++++++++++++++
1 file changed, 32 insertions(+)
diff --git a/docs/topics/impala_iceberg.xml b/docs/topics/impala_iceberg.xml
index 78c865832..1ae133623 100644
--- a/docs/topics/impala_iceberg.xml
+++ b/docs/topics/impala_iceberg.xml
@@ -506,6 +506,38 @@ CREATE TABLE new_ice_tbl LIKE orig_ice_tbl;
</conbody>
</concept>
+ <concept id="iceberg_expire_snapshots">
+ <title>Expiring snapshots</title>
+ <conbody>
+ <p>
+ Iceberg snapshots accumulate until they are deleted by a user action. Snapshots
+ can be deleted with <codeph>ALTER TABLE ... EXECUTE expire_snapshots(...)</codeph>
+ statement, which will expire snapshots that are older than the specified
+ timestamp. For example:
+ <codeblock>
+ALTER TABLE ice_tbl EXECUTE expire_snapshots('2022-01-04 10:00:00');
+ALTER TABLE ice_tbl EXECUTE expire_snapshots(now() - interval 5 days);
+ </codeblock>
+ </p>
+ <p>
+ Expire snapshots:
+ <ul>
+ <li>does not remove old metadata files by default.</li>
+ <li>does not remove orphaned data files.</li>
+ <li>respects the minimum number of snapshots to keep:
+ <codeph>history.expire.min-snapshots-to-keep</codeph> table property.</li>
+ </ul>
+ </p>
+ <p>
+ Old metadata file clean up can be configured with
+ <codeph>write.metadata.delete-after-commit.enabled=true</codeph> and
+ <codeph>write.metadata.previous-versions-max</codeph> table properties. This
+ allows automatic metadata file removal after operations that modify metadata
+ such as expiring snapshots or inserting data.
+ </p>
+ </conbody>
+ </concept>
+
<concept id="iceberg_table_properties">
<title>Iceberg table properties</title>
<conbody>
[impala] 01/02: IMPALA-11496: Push down date literals to Kudu scanner
Posted by cs...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
csringhofer pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit e8ebf0be67828b855d4533c07ef544f85897ec62
Author: Peter Rozsa <pr...@cloudera.com>
AuthorDate: Mon Aug 22 11:19:24 2022 +0200
IMPALA-11496: Push down date literals to Kudu scanner
This change enables creating, and pushing down Kudu
predicates from DATE type. Kudu planner testcases
added for DATE, BIGINT, FLOAT, DOUBLE types to
cover all supported type's predicate creation.
Change-Id: I4f323d23df3a92361390f9da387182db3f3bde3e
Reviewed-on: http://gerrit.cloudera.org:8080/18898
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
.../org/apache/impala/planner/KuduScanNode.java | 12 +++-
.../queries/PlannerTest/kudu.test | 68 ++++++++++++++++++++++
2 files changed, 77 insertions(+), 3 deletions(-)
diff --git a/fe/src/main/java/org/apache/impala/planner/KuduScanNode.java b/fe/src/main/java/org/apache/impala/planner/KuduScanNode.java
index 6663460ae..65f8c5a05 100644
--- a/fe/src/main/java/org/apache/impala/planner/KuduScanNode.java
+++ b/fe/src/main/java/org/apache/impala/planner/KuduScanNode.java
@@ -29,8 +29,8 @@ import java.util.Set;
import org.apache.impala.analysis.Analyzer;
import org.apache.impala.analysis.BinaryPredicate;
import org.apache.impala.analysis.BoolLiteral;
+import org.apache.impala.analysis.DateLiteral;
import org.apache.impala.analysis.Expr;
-import org.apache.impala.analysis.ExprSubstitutionMap;
import org.apache.impala.analysis.InPredicate;
import org.apache.impala.analysis.IsNullPredicate;
import org.apache.impala.analysis.LiteralExpr;
@@ -567,14 +567,20 @@ public class KuduScanNode extends ScanNode {
}
break;
}
+ case DATE:
+ kuduPredicate = KuduPredicate.newComparisonPredicate(column, op,
+ ((DateLiteral)literal).getValue());
+ break;
case DECIMAL: {
kuduPredicate = KuduPredicate.newComparisonPredicate(column, op,
((NumericLiteral)literal).getValue());
break;
}
- default: break;
+ default:
+ //All supported types are covered, should not reach default case
+ Preconditions.checkState(false);
}
- if (kuduPredicate == null) return false;
+ Preconditions.checkState(kuduPredicate != null);
kuduConjuncts_.add(predicate);
kuduPredicates_.add(kuduPredicate);
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/kudu.test b/testdata/workloads/functional-planner/queries/PlannerTest/kudu.test
index 6bdcc41b9..7d5a3b3dc 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/kudu.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/kudu.test
@@ -712,3 +712,71 @@ PLAN-ROOT SINK
kudu predicates: l_orderkey = 1, l_partkey = 1, l_suppkey = 1
row-size=0B cardinality=1
====
+# DATE predicate.
+select * from functional_kudu.date_tbl where date_col = DATE "1970-01-01";
+---- PLAN
+PLAN-ROOT SINK
+|
+00:SCAN KUDU [functional_kudu.date_tbl]
+ kudu predicates: date_col = DATE '1970-01-01'
+ row-size=12B cardinality=1
+---- DISTRIBUTEDPLAN
+PLAN-ROOT SINK
+|
+01:EXCHANGE [UNPARTITIONED]
+|
+00:SCAN KUDU [functional_kudu.date_tbl]
+ kudu predicates: date_col = DATE '1970-01-01'
+ row-size=12B cardinality=1
+====
+# BIGINT predicate.
+select * from functional_kudu.alltypes where bigint_col = 1;
+---- PLAN
+PLAN-ROOT SINK
+|
+00:SCAN KUDU [functional_kudu.alltypes]
+ kudu predicates: bigint_col = 1
+ row-size=97B cardinality=730
+---- DISTRIBUTEDPLAN
+PLAN-ROOT SINK
+|
+01:EXCHANGE [UNPARTITIONED]
+|
+00:SCAN KUDU [functional_kudu.alltypes]
+ kudu predicates: bigint_col = 1
+ row-size=97B cardinality=730
+====
+# FLOAT predicate.
+select * from functional_kudu.alltypes where float_col = 1.0;
+---- PLAN
+PLAN-ROOT SINK
+|
+00:SCAN KUDU [functional_kudu.alltypes]
+ kudu predicates: float_col = 1.0
+ row-size=97B cardinality=730
+---- DISTRIBUTEDPLAN
+PLAN-ROOT SINK
+|
+01:EXCHANGE [UNPARTITIONED]
+|
+00:SCAN KUDU [functional_kudu.alltypes]
+ kudu predicates: float_col = 1.0
+ row-size=97B cardinality=730
+====
+# DOUBLE predicate.
+select * from functional_kudu.alltypes where double_col = 1.0;
+---- PLAN
+PLAN-ROOT SINK
+|
+00:SCAN KUDU [functional_kudu.alltypes]
+ kudu predicates: double_col = 1.0
+ row-size=97B cardinality=730
+---- DISTRIBUTEDPLAN
+PLAN-ROOT SINK
+|
+01:EXCHANGE [UNPARTITIONED]
+|
+00:SCAN KUDU [functional_kudu.alltypes]
+ kudu predicates: double_col = 1.0
+ row-size=97B cardinality=730
+====
\ No newline at end of file