You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by dz...@apache.org on 2022/07/11 08:29:29 UTC
[drill] branch master updated: Try and reduce the vm crash on fork
This is an automated email from the ASF dual-hosted git repository.
dzamo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
The following commit(s) were added to refs/heads/master by this push:
new b64acddd05 Try and reduce the vm crash on fork
b64acddd05 is described below
commit b64acddd0567e261ed29d8a4fe1f474d962e09d2
Author: luocooong <lu...@apache.org>
AuthorDate: Sun Jul 3 20:02:40 2022 +0800
Try and reduce the vm crash on fork
Use the drop_caches and split unit tests
Ready for the review
---
.github/workflows/ci.yml | 22 ++++++++---------
.travis.yml | 2 +-
.../apache/drill/categories/EasyOutOfMemory.java | 28 ++++++++++++++++++++++
.../org/apache/drill/categories/package-info.java | 2 ++
.../drill/exec/fn/impl/TestAggregateFunctions.java | 3 ++-
.../physical/impl/writer/TestParquetWriter.java | 5 ----
.../drill/exec/server/TestDrillbitResilience.java | 2 +-
pom.xml | 2 +-
8 files changed, 45 insertions(+), 21 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 230fdb10c6..7988390440 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -46,22 +46,20 @@ jobs:
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-maven-
- # Caches MySQL directory used for JDBC storage plugin tests
- - name: Cache MySQL
- uses: actions/cache@v2
- with:
- path: ~/.embedmysql
- key: ${{ runner.os }}-mysql
- - name: Set up JDK ${{ matrix.java }}
- uses: actions/setup-java@v2
- with:
- distribution: 'adopt'
- java-version: ${{ matrix.java }}
- name: Build and test
# The total GitHub Actions memory is 7000Mb. But GitHub CI requires some memory for the container to perform tests
run: |
MAVEN_OPTS="-XX:+UseG1GC"
- mvn install --batch-mode --no-transfer-progress -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 # -X -V for debugging
+ sudo sh -c 'echo 1 > /proc/sys/vm/drop_caches' && \
+ mvn install --batch-mode --no-transfer-progress \
+ -DexcludedGroups=org.apache.drill.categories.EasyOutOfMemory \
+ -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120
+ - name: Test Specific Categories # EasyOutOfMemory
+ run: |
+ sudo sh -c 'echo 1 > /proc/sys/vm/drop_caches' && \
+ mvn test -pl org.apache.drill.exec:drill-java-exec \
+ -Dgroups=org.apache.drill.categories.EasyOutOfMemory \
+ -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120
checkstyle_protobuf:
name: Run checkstyle and generate protobufs
diff --git a/.travis.yml b/.travis.yml
index 6461e8205d..6e576e1640 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -73,7 +73,7 @@ install:
# For protobuf phase: builds Drill project, performs license checkstyle goal and regenerates Java and C++ Protobuf files
- |
if [ $PHASE = "tests" ]; then \
- mvn install --batch-mode --no-transfer-progress -DforkCount=1 -DdirectMemoryMb=$DIRECTMEMORYMB \
+ mvn install --batch-mode --no-transfer-progress -DdirectMemoryMb=$DIRECTMEMORYMB \
-DexcludedGroups="org.apache.drill.categories.SlowTest,org.apache.drill.categories.UnlikelyTest,org.apache.drill.categories.SecurityTest"; \
elif [ $PHASE = "build_checkstyle_protobuf" ]; then \
MAVEN_OPTS="-Xms1G -Xmx1G" mvn install --no-transfer-progress -Drat.skip=false -Dlicense.skip=false --batch-mode -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -DskipTests=true -Dmaven.javadoc.skip=true -Dmaven.source.skip=true && \
diff --git a/common/src/test/java/org/apache/drill/categories/EasyOutOfMemory.java b/common/src/test/java/org/apache/drill/categories/EasyOutOfMemory.java
new file mode 100644
index 0000000000..2dd29456ab
--- /dev/null
+++ b/common/src/test/java/org/apache/drill/categories/EasyOutOfMemory.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.categories;
+
+/**
+ * Split the unit test for EasyOutOfMemory categories.<br/>
+ * Note: If you use this category to mark the test class,
+ * the CI will use a new clean JVM to test them.
+ *
+ */
+public interface EasyOutOfMemory extends FlakyTest {
+
+}
diff --git a/common/src/test/java/org/apache/drill/categories/package-info.java b/common/src/test/java/org/apache/drill/categories/package-info.java
index bb699c14ba..4f80760fa9 100644
--- a/common/src/test/java/org/apache/drill/categories/package-info.java
+++ b/common/src/test/java/org/apache/drill/categories/package-info.java
@@ -25,6 +25,8 @@
* <li>{@link org.apache.drill.categories.SlowTest}: These tests run slowly, and are disabled by default.</li>
* <li>{@link org.apache.drill.categories.UnlikelyTest}: These tests represent corner cases, specific bug fixes, or tests for pieces of code that are unlikely to be changed.
* These are disabled by default</li>
+ * <li>{@link org.apache.drill.categories.FlakyTest}: A category for tests that intermittently fail.</li>
+ * <li>{@link org.apache.drill.categories.EasyOutOfMemory}: Inherited class FlakyTest and allow the CI tool uses a new JVM to test the unit.</li>
* </ul>
* </p>
* <p>
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestAggregateFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestAggregateFunctions.java
index 182c983583..11be3bc3c6 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestAggregateFunctions.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestAggregateFunctions.java
@@ -31,6 +31,7 @@ import org.apache.drill.exec.planner.physical.PlannerSettings;
import org.apache.drill.exec.record.RecordBatchLoader;
import org.apache.drill.exec.record.VectorWrapper;
import org.apache.drill.exec.util.Text;
+import org.apache.drill.categories.EasyOutOfMemory;
import org.apache.drill.categories.OperatorTest;
import org.apache.drill.categories.PlannerTest;
import org.apache.drill.categories.SqlFunctionTest;
@@ -68,7 +69,7 @@ import static org.hamcrest.CoreMatchers.containsString;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
-@Category({SqlFunctionTest.class, OperatorTest.class, PlannerTest.class})
+@Category({ SqlFunctionTest.class, OperatorTest.class, PlannerTest.class, EasyOutOfMemory.class })
public class TestAggregateFunctions extends ClusterTest {
@Rule
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java
index 80ba94418b..28b009e869 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java
@@ -52,9 +52,6 @@ import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
-import org.junit.jupiter.api.condition.DisabledOnOs;
-import org.junit.jupiter.api.condition.EnabledIfSystemProperty;
-import org.junit.jupiter.api.condition.OS;
import java.io.File;
import java.io.FileWriter;
@@ -1003,8 +1000,6 @@ public class TestParquetWriter extends ClusterTest {
// Only attempt this test on Linux / amd64 because com.rdblue.brotli-codec
// only bundles natives for Mac and Linux on AMD64. See PARQUET-1975.
@Test
- @EnabledIfSystemProperty(named = "os.arch", matches = "(amd64|x86_64)")
- @DisabledOnOs({ OS.WINDOWS })
public void testTPCHReadWriteBrotli() throws Exception {
try {
client.alterSession(ExecConstants.PARQUET_WRITER_COMPRESSION_TYPE, "brotli");
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestDrillbitResilience.java b/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestDrillbitResilience.java
index 564112e1fc..3f66fd67a5 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestDrillbitResilience.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestDrillbitResilience.java
@@ -117,7 +117,7 @@ public class TestDrillbitResilience extends ClusterTest {
private static final int NUM_RUNS = 3;
private static final int PROBLEMATIC_TEST_NUM_RUNS = 3;
private static final int TIMEOUT = 15;
- private final static Level CURRENT_LOG_LEVEL = Level.DEBUG;
+ private final static Level CURRENT_LOG_LEVEL = Level.INFO;
/**
* Note: Counting sys.memory executes a fragment on every drillbit. This is a better check in comparison to
diff --git a/pom.xml b/pom.xml
index 78d59dc831..5564b365f2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -116,7 +116,7 @@
<codemodel.version>2.6</codemodel.version>
<joda.version>2.10.5</joda.version>
<javax.el.version>3.0.0</javax.el.version>
- <surefire.version>3.0.0-M5</surefire.version>
+ <surefire.version>3.0.0-M7</surefire.version>
<jna.version>5.8.0</jna.version>
<commons.compress.version>1.21</commons.compress.version>
<hikari.version>4.0.3</hikari.version>