You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by dz...@apache.org on 2023/02/13 04:01:14 UTC
[drill] branch master updated (c38091fdd6 -> 9d6af6dea6)
This is an automated email from the ASF dual-hosted git repository.
dzamo pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
from c38091fdd6 DRILL-4232: Support for EXCEPT and INTERSECT set operator (#2599)
new 6562d27e70 DRILL-8398: Fix GitHub Actions to use proper JDK version
new 9d6af6dea6 DRILL-8400: Fix pruning partitions with pushed transitive predicates
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/ci.yml | 29 ++++++++++------------
.../drill/exec/store/hive/HiveStoragePlugin.java | 2 +-
.../drill/exec/TestHivePartitionPruning.java | 4 +--
contrib/storage-phoenix/pom.xml | 13 +++++++++-
.../drill/TestTpchDistributedConcurrent.java | 3 ++-
.../drill/exec/server/TestDrillbitResilience.java | 6 ++---
pom.xml | 2 +-
7 files changed, 34 insertions(+), 25 deletions(-)
[drill] 01/02: DRILL-8398: Fix GitHub Actions to use proper JDK version
Posted by dz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dzamo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 6562d27e702c738ce040d84ad76b3958a480bd7a
Author: Volodymyr Vysotskyi <vv...@gmail.com>
AuthorDate: Thu Feb 9 22:29:32 2023 +0200
DRILL-8398: Fix GitHub Actions to use proper JDK version
DRILL-8398: Disable Phoenix tests for JDK 14+
---
.github/workflows/ci.yml | 29 ++++++++++------------
.../drill/exec/TestHivePartitionPruning.java | 2 ++
contrib/storage-phoenix/pom.xml | 13 +++++++++-
.../drill/TestTpchDistributedConcurrent.java | 3 ++-
.../drill/exec/server/TestDrillbitResilience.java | 6 ++---
pom.xml | 2 +-
6 files changed, 33 insertions(+), 22 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 2f765bb488..265f87985f 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -41,15 +41,13 @@ jobs:
fail-fast: false
steps:
- name: Checkout
- uses: actions/checkout@v2
- # Caches maven cache and uses hashes of pom.xml files to find the required cache
- - name: Cache Maven Repository
- uses: actions/cache@v2
+ uses: actions/checkout@v3
+ - name: Setup java
+ uses: actions/setup-java@v3
with:
- path: ~/.m2/repository
- key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
- restore-keys: |
- ${{ runner.os }}-maven-
+ distribution: 'temurin'
+ java-version: ${{ matrix.java }}
+ cache: 'maven'
- name: Build and test
# The total GitHub Actions memory is 7000Mb. But GitHub CI requires some memory for the container to perform tests
run: |
@@ -69,17 +67,16 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout
- uses: actions/checkout@v2
- - name: Cache Maven Repository
- uses: actions/cache@v2
+ uses: actions/checkout@v3
+ - name: Setup java
+ uses: actions/setup-java@v3
with:
- path: ~/.m2/repository
- key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
- restore-keys: |
- ${{ runner.os }}-maven-
+ distribution: 'temurin'
+ java-version: '8'
+ cache: 'maven'
# Caches built protobuf library
- name: Cache protobufs
- uses: actions/cache@v2
+ uses: actions/cache@v3
with:
path: ~/protobuf
key: ${{ runner.os }}-protobuf
diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/TestHivePartitionPruning.java b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/TestHivePartitionPruning.java
index 971349ece1..62a2c136ae 100644
--- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/TestHivePartitionPruning.java
+++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/TestHivePartitionPruning.java
@@ -28,6 +28,7 @@ import org.apache.drill.exec.planner.physical.PlannerSettings;
import org.apache.drill.exec.rpc.user.QueryDataBatch;
import org.junit.AfterClass;
import org.junit.BeforeClass;
+import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -162,6 +163,7 @@ public class TestHivePartitionPruning extends HiveTestBase {
}
@Test // DRILL-6173
+ @Ignore("DRILL-8400")
public void prunePartitionsBasedOnTransitivePredicates() throws Exception {
String query = String.format("SELECT * FROM hive.partition_pruning_test t1 " +
"JOIN hive.partition_with_few_schemas t2 ON t1.`d` = t2.`d` AND t1.`e` = t2.`e` " +
diff --git a/contrib/storage-phoenix/pom.xml b/contrib/storage-phoenix/pom.xml
index cb3dc23419..d6a79617fd 100644
--- a/contrib/storage-phoenix/pom.xml
+++ b/contrib/storage-phoenix/pom.xml
@@ -27,7 +27,7 @@
</parent>
<artifactId>drill-storage-phoenix</artifactId>
<name>Drill : Contrib : Storage : Phoenix</name>
-
+
<properties>
<phoenix.version>5.1.2</phoenix.version>
<!-- Keep the 2.4.2 to reduce dependency conflict -->
@@ -395,5 +395,16 @@
</plugins>
</build>
</profile>
+ <profile>
+ <!-- Disable unit tests for JDK 14+ until Phoenix 5.2.0+ is released.
+ See PHOENIX-6723 for details.-->
+ <id>jdk14+</id>
+ <activation>
+ <jdk>[14,)</jdk>
+ </activation>
+ <properties>
+ <skipTests>true</skipTests>
+ </properties>
+ </profile>
</profiles>
</project>
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestTpchDistributedConcurrent.java b/exec/java-exec/src/test/java/org/apache/drill/TestTpchDistributedConcurrent.java
index c144c53bde..74732fec73 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestTpchDistributedConcurrent.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestTpchDistributedConcurrent.java
@@ -24,6 +24,7 @@ import java.util.Random;
import java.util.Set;
import java.util.concurrent.Semaphore;
+import org.apache.drill.categories.EasyOutOfMemory;
import org.apache.drill.categories.SlowTest;
import org.apache.drill.common.exceptions.UserException;
import org.apache.drill.exec.ExecConstants;
@@ -55,7 +56,7 @@ import static org.junit.Assert.assertNull;
* unstable from running a lot of queries concurrently -- it's not about
* any particular order of execution. We ignore the results.
*/
-@Category({SlowTest.class})
+@Category({SlowTest.class, EasyOutOfMemory.class})
public class TestTpchDistributedConcurrent extends ClusterTest {
private static final Logger logger = LoggerFactory.getLogger(TestTpchDistributedConcurrent.class);
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestDrillbitResilience.java b/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestDrillbitResilience.java
index 3f66fd67a5..a71ea313c2 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestDrillbitResilience.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestDrillbitResilience.java
@@ -29,6 +29,7 @@ import java.io.IOException;
import ch.qos.logback.classic.Level;
import org.apache.commons.math3.util.Pair;
+import org.apache.drill.categories.EasyOutOfMemory;
import org.apache.drill.categories.FlakyTest;
import org.apache.drill.exec.client.DrillClient;
import org.apache.drill.exec.physical.impl.partitionsender.PartitionSenderRootExec;
@@ -82,12 +83,12 @@ import org.apache.drill.exec.testing.ControlsInjectionUtil;
import org.apache.drill.exec.util.Pointer;
import org.apache.drill.exec.work.fragment.FragmentExecutor;
import org.apache.drill.categories.SlowTest;
+import org.junit.experimental.categories.Category;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.RepeatedTest;
import org.junit.jupiter.api.BeforeAll;
-import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInfo;
import org.junit.jupiter.api.Timeout;
@@ -105,8 +106,7 @@ import org.apache.drill.shaded.guava.com.google.common.base.Preconditions;
* <li>specify Level.DEBUG for CURRENT_LOG_LEVEL</li>
* <li>compare trace output for successful test case and failed</li>
*/
-@Tag(SlowTest.TAG)
-@Tag(FlakyTest.TAG)
+@Category({ SlowTest.class, FlakyTest.class, EasyOutOfMemory.class })
public class TestDrillbitResilience extends ClusterTest {
private static final Logger logger = org.slf4j.LoggerFactory.getLogger(TestDrillbitResilience.class);
protected static LogFixture logFixture;
diff --git a/pom.xml b/pom.xml
index a653d62ac2..235dfab97a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -110,7 +110,7 @@
<codemodel.version>2.6</codemodel.version>
<joda.version>2.10.14</joda.version>
<javax.el.version>3.0.0</javax.el.version>
- <surefire.version>3.0.0-M7</surefire.version>
+ <surefire.version>3.0.0-M8</surefire.version>
<jna.version>5.8.0</jna.version>
<commons.compress.version>1.21</commons.compress.version>
<hikari.version>4.0.3</hikari.version>
[drill] 02/02: DRILL-8400: Fix pruning partitions with pushed transitive predicates
Posted by dz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dzamo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 9d6af6dea6ed6f47f45a2bd064c01bf69d85acbf
Author: Volodymyr Vysotskyi <vv...@gmail.com>
AuthorDate: Sun Feb 12 16:41:28 2023 +0200
DRILL-8400: Fix pruning partitions with pushed transitive predicates
---
.../java/org/apache/drill/exec/store/hive/HiveStoragePlugin.java | 2 +-
.../test/java/org/apache/drill/exec/TestHivePartitionPruning.java | 6 ++----
2 files changed, 3 insertions(+), 5 deletions(-)
diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveStoragePlugin.java b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveStoragePlugin.java
index c021ebca14..1ce138c0e3 100644
--- a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveStoragePlugin.java
+++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveStoragePlugin.java
@@ -192,7 +192,7 @@ public class HiveStoragePlugin extends AbstractStoragePlugin {
@Override
public Set<StoragePluginOptimizerRule> getOptimizerRules(OptimizerRulesContext optimizerContext, PlannerPhase phase) {
switch (phase) {
- case LOGICAL:
+ case PARTITION_PRUNING:
final String defaultPartitionValue = hiveConf.get(ConfVars.DEFAULTPARTITIONNAME.varname);
ImmutableSet.Builder<StoragePluginOptimizerRule> ruleBuilder = ImmutableSet.builder();
ruleBuilder.add(HivePushPartitionFilterIntoScan.getFilterOnProject(optimizerContext, defaultPartitionValue));
diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/TestHivePartitionPruning.java b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/TestHivePartitionPruning.java
index 62a2c136ae..608aaf8d1e 100644
--- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/TestHivePartitionPruning.java
+++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/TestHivePartitionPruning.java
@@ -28,7 +28,6 @@ import org.apache.drill.exec.planner.physical.PlannerSettings;
import org.apache.drill.exec.rpc.user.QueryDataBatch;
import org.junit.AfterClass;
import org.junit.BeforeClass;
-import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -163,11 +162,10 @@ public class TestHivePartitionPruning extends HiveTestBase {
}
@Test // DRILL-6173
- @Ignore("DRILL-8400")
public void prunePartitionsBasedOnTransitivePredicates() throws Exception {
- String query = String.format("SELECT * FROM hive.partition_pruning_test t1 " +
+ String query = "SELECT * FROM hive.partition_pruning_test t1 " +
"JOIN hive.partition_with_few_schemas t2 ON t1.`d` = t2.`d` AND t1.`e` = t2.`e` " +
- "WHERE t2.`e` IS NOT NULL AND t1.`d` = 1");
+ "WHERE t2.`e` IS NOT NULL AND t1.`d` = 1";
int actualRowCount = testSql(query);
int expectedRowCount = 450;