You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@druid.apache.org by ab...@apache.org on 2024/09/30 18:50:11 UTC
(druid) branch 31.0.0 updated: Update Delta Kernel to 3.2.1 (#17179) (#17198)
This is an automated email from the ASF dual-hosted git repository.
abhishekrb pushed a commit to branch 31.0.0
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/31.0.0 by this push:
new 8531a84d6cf Update Delta Kernel to 3.2.1 (#17179) (#17198)
8531a84d6cf is described below
commit 8531a84d6cf5a7ad270086322a5e38432139f5c2
Author: Abhishek Radhakrishnan <ab...@gmail.com>
AuthorDate: Mon Sep 30 11:50:04 2024 -0700
Update Delta Kernel to 3.2.1 (#17179) (#17198)
Updated Delta Kernel from 3.2.0 to 3.2.1. This upstream version bump contains fixes to reading long columns, class loader and better retry mechanism when reading checkpoint files.
---
extensions-contrib/druid-deltalake-extensions/pom.xml | 8 +-------
.../apache/druid/delta/input/DeltaInputSource.java | 19 ++++---------------
2 files changed, 5 insertions(+), 22 deletions(-)
diff --git a/extensions-contrib/druid-deltalake-extensions/pom.xml b/extensions-contrib/druid-deltalake-extensions/pom.xml
index 71a167c315e..5d4009c5f36 100644
--- a/extensions-contrib/druid-deltalake-extensions/pom.xml
+++ b/extensions-contrib/druid-deltalake-extensions/pom.xml
@@ -35,7 +35,7 @@
<modelVersion>4.0.0</modelVersion>
<properties>
- <delta-kernel.version>3.2.0</delta-kernel.version>
+ <delta-kernel.version>3.2.1</delta-kernel.version>
</properties>
<dependencies>
@@ -49,12 +49,6 @@
<artifactId>delta-kernel-defaults</artifactId>
<version>${delta-kernel.version}</version>
</dependency>
- <dependency>
- <groupId>io.delta</groupId>
- <artifactId>delta-storage</artifactId>
- <version>${delta-kernel.version}</version>
- </dependency>
-
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client-api</artifactId>
diff --git a/extensions-contrib/druid-deltalake-extensions/src/main/java/org/apache/druid/delta/input/DeltaInputSource.java b/extensions-contrib/druid-deltalake-extensions/src/main/java/org/apache/druid/delta/input/DeltaInputSource.java
index c4c2f2668b0..4f255d020f7 100644
--- a/extensions-contrib/druid-deltalake-extensions/src/main/java/org/apache/druid/delta/input/DeltaInputSource.java
+++ b/extensions-contrib/druid-deltalake-extensions/src/main/java/org/apache/druid/delta/input/DeltaInputSource.java
@@ -42,7 +42,6 @@ import io.delta.kernel.types.StructField;
import io.delta.kernel.types.StructType;
import io.delta.kernel.utils.CloseableIterator;
import io.delta.kernel.utils.FileStatus;
-import io.delta.storage.LogStore;
import org.apache.druid.data.input.ColumnsFilter;
import org.apache.druid.data.input.InputFormat;
import org.apache.druid.data.input.InputRowSchema;
@@ -340,20 +339,10 @@ public class DeltaInputSource implements SplittableInputSource<DeltaSplit>
private Snapshot getSnapshotForTable(final Table table, final Engine engine)
{
- // Setting the LogStore class loader before calling the Delta Kernel snapshot API is required as a workaround with
- // the 3.2.0 Delta Kernel because the Kernel library cannot instantiate the LogStore class otherwise. Please see
- // https://github.com/delta-io/delta/issues/3299 for details. This workaround can be removed once the issue is fixed.
- final ClassLoader currCtxCl = Thread.currentThread().getContextClassLoader();
- try {
- Thread.currentThread().setContextClassLoader(LogStore.class.getClassLoader());
- if (snapshotVersion != null) {
- return table.getSnapshotAsOfVersion(engine, snapshotVersion);
- } else {
- return table.getLatestSnapshot(engine);
- }
- }
- finally {
- Thread.currentThread().setContextClassLoader(currCtxCl);
+ if (snapshotVersion != null) {
+ return table.getSnapshotAsOfVersion(engine, snapshotVersion);
+ } else {
+ return table.getLatestSnapshot(engine);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org