You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by vo...@apache.org on 2021/02/11 16:51:11 UTC
[drill] branch master updated: DRILL-7857: Fix misc GitHub Actions
failures (#2163)
This is an automated email from the ASF dual-hosted git repository.
volodymyr pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
The following commit(s) were added to refs/heads/master by this push:
new 6bcc0c5 DRILL-7857: Fix misc GitHub Actions failures (#2163)
6bcc0c5 is described below
commit 6bcc0c5effee645a2abc280cdaa861a872be68c0
Author: Volodymyr Vysotskyi <vv...@gmail.com>
AuthorDate: Thu Feb 11 18:50:57 2021 +0200
DRILL-7857: Fix misc GitHub Actions failures (#2163)
---
contrib/storage-cassandra/pom.xml | 23 +++++--
.../exec/store/cassandra/BaseCassandraTest.java | 58 ++++--------------
.../exec/store/cassandra/TestCassandraSuit.java | 71 ++++++++++++++++++++++
.../drill/exec/store/kafka/TestKafkaSuit.java | 2 +-
.../drill/exec/server/rest/InteractiveUI.java | 5 +-
.../drill/exec/server/rest/TestRestJson.java | 13 ++--
.../drill/exec/util/ExecutorServiceUtilTest.java | 30 ++++-----
.../drill/exec/util/TestQueryMemoryAlloc.java | 6 +-
pom.xml | 4 +-
9 files changed, 135 insertions(+), 77 deletions(-)
diff --git a/contrib/storage-cassandra/pom.xml b/contrib/storage-cassandra/pom.xml
index 2cb6983..facfb5e 100644
--- a/contrib/storage-cassandra/pom.xml
+++ b/contrib/storage-cassandra/pom.xml
@@ -57,7 +57,7 @@
<dependency>
<groupId>com.scylladb</groupId>
<artifactId>scylla-driver-core</artifactId>
- <version>3.10.1-scylla-0</version>
+ <version>3.10.2-scylla-1</version>
</dependency>
<dependency>
<groupId>org.ow2.asm</groupId>
@@ -80,10 +80,16 @@
<scope>test</scope>
</dependency>
<dependency>
- <groupId>com.github.nosan</groupId>
- <artifactId>embedded-cassandra</artifactId>
- <version>4.0.0</version>
+ <groupId>org.testcontainers</groupId>
+ <artifactId>cassandra</artifactId>
+ <version>1.15.1</version>
<scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>com.datastax.cassandra</groupId>
+ <artifactId>cassandra-driver-core</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>net.hydromatic</groupId>
@@ -98,7 +104,14 @@
<plugin>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
- <forkCount combine.self="override">1</forkCount>
+ <includes>
+ <include>**/TestCassandraSuit.class</include>
+ </includes>
+ <excludes>
+ <exclude>**/CassandraComplexTypesTest.java</exclude>
+ <exclude>**/CassandraPlanTest.java</exclude>
+ <exclude>**/CassandraQueryTest.java</exclude>
+ </excludes>
</configuration>
</plugin>
</plugins>
diff --git a/contrib/storage-cassandra/src/test/java/org/apache/drill/exec/store/cassandra/BaseCassandraTest.java b/contrib/storage-cassandra/src/test/java/org/apache/drill/exec/store/cassandra/BaseCassandraTest.java
index 9413db2..225873b 100644
--- a/contrib/storage-cassandra/src/test/java/org/apache/drill/exec/store/cassandra/BaseCassandraTest.java
+++ b/contrib/storage-cassandra/src/test/java/org/apache/drill/exec/store/cassandra/BaseCassandraTest.java
@@ -17,68 +17,36 @@
*/
package org.apache.drill.exec.store.cassandra;
-import com.datastax.driver.core.Cluster;
-import com.datastax.driver.core.Session;
-import com.github.nosan.embedded.cassandra.Cassandra;
-import com.github.nosan.embedded.cassandra.CassandraBuilder;
-import com.github.nosan.embedded.cassandra.Settings;
-import com.github.nosan.embedded.cassandra.cql.CqlScript;
import org.apache.drill.test.ClusterFixture;
import org.apache.drill.test.ClusterTest;
-import org.apache.hadoop.util.ComparableVersion;
import org.junit.AfterClass;
import org.junit.BeforeClass;
-
-import static org.junit.Assume.assumeTrue;
+import org.testcontainers.containers.CassandraContainer;
public class BaseCassandraTest extends ClusterTest {
- private static Cassandra cassandra;
@BeforeClass
- public static void init() throws Exception {
- assumeTrue(
- "Skipping tests for JDK 12+ since Cassandra supports only versions up to 11 (including).",
- new ComparableVersion(System.getProperty("java.version"))
- .compareTo(new ComparableVersion("12")) < 0);
+ public static void setUpBeforeClass() throws Exception {
+ TestCassandraSuit.initCassandra();
+ initCassandraPlugin(TestCassandraSuit.cassandra);
+ }
+ private static void initCassandraPlugin(CassandraContainer<?> cassandra) throws Exception {
startCluster(ClusterFixture.builder(dirTestWatcher));
- startCassandra();
-
CassandraStorageConfig config = new CassandraStorageConfig(
- cassandra.getSettings().getAddress().getHostAddress(),
- cassandra.getSettings().getPort(),
- null,
- null);
+ cassandra.getHost(),
+ cassandra.getMappedPort(CassandraContainer.CQL_PORT),
+ cassandra.getUsername(),
+ cassandra.getPassword());
config.setEnabled(true);
cluster.defineStoragePlugin("cassandra", config);
-
- prepareData();
}
@AfterClass
- public static void cleanUp() {
- if (cassandra != null) {
- cassandra.stop();
- }
- }
-
- private static void startCassandra() {
- cassandra = new CassandraBuilder().build();
- cassandra.start();
- }
-
- private static void prepareData() {
- Settings settings = cassandra.getSettings();
-
- try (Cluster cluster = Cluster.builder()
- .addContactPoints(settings.getAddress())
- .withPort(settings.getPort())
- .withoutMetrics()
- .withoutJMXReporting()
- .build()) {
- Session session = cluster.connect();
- CqlScript.ofClassPath("queries.cql").forEachStatement(session::execute);
+ public static void tearDownCassandra() {
+ if (TestCassandraSuit.isRunningSuite()) {
+ TestCassandraSuit.tearDownCluster();
}
}
}
diff --git a/contrib/storage-cassandra/src/test/java/org/apache/drill/exec/store/cassandra/TestCassandraSuit.java b/contrib/storage-cassandra/src/test/java/org/apache/drill/exec/store/cassandra/TestCassandraSuit.java
new file mode 100644
index 0000000..60d21c7
--- /dev/null
+++ b/contrib/storage-cassandra/src/test/java/org/apache/drill/exec/store/cassandra/TestCassandraSuit.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.cassandra;
+
+import org.apache.drill.categories.SlowTest;
+import org.apache.drill.test.BaseTest;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+import org.junit.runners.Suite;
+import org.testcontainers.containers.CassandraContainer;
+
+import java.util.concurrent.atomic.AtomicInteger;
+
+@Category(SlowTest.class)
+@RunWith(Suite.class)
+@Suite.SuiteClasses({CassandraComplexTypesTest.class, CassandraPlanTest.class, CassandraQueryTest.class})
+public class TestCassandraSuit extends BaseTest {
+
+ protected static CassandraContainer<?> cassandra;
+
+ private static final AtomicInteger initCount = new AtomicInteger(0);
+
+ private static volatile boolean runningSuite = false;
+
+ @BeforeClass
+ public static void initCassandra() {
+ synchronized (TestCassandraSuit.class) {
+ if (initCount.get() == 0) {
+ startCassandra();
+ }
+ initCount.incrementAndGet();
+ runningSuite = true;
+ }
+ }
+
+ public static boolean isRunningSuite() {
+ return runningSuite;
+ }
+
+ @AfterClass
+ public static void tearDownCluster() {
+ synchronized (TestCassandraSuit.class) {
+ if (initCount.decrementAndGet() == 0 && cassandra != null) {
+ cassandra.stop();
+ }
+ }
+ }
+
+ private static void startCassandra() {
+ cassandra = new CassandraContainer<>("cassandra")
+ .withInitScript("queries.cql");
+ cassandra.start();
+ }
+}
diff --git a/contrib/storage-kafka/src/test/java/org/apache/drill/exec/store/kafka/TestKafkaSuit.java b/contrib/storage-kafka/src/test/java/org/apache/drill/exec/store/kafka/TestKafkaSuit.java
index 7556779..6886b05 100644
--- a/contrib/storage-kafka/src/test/java/org/apache/drill/exec/store/kafka/TestKafkaSuit.java
+++ b/contrib/storage-kafka/src/test/java/org/apache/drill/exec/store/kafka/TestKafkaSuit.java
@@ -61,7 +61,7 @@ public class TestKafkaSuit extends BaseTest {
private static ZkClient zkClient;
- private static AtomicInteger initCount = new AtomicInteger(0);
+ private static final AtomicInteger initCount = new AtomicInteger(0);
static final int NUM_JSON_MSG = 10;
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/InteractiveUI.java b/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/InteractiveUI.java
index 5189db8..c8f8c75 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/InteractiveUI.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/InteractiveUI.java
@@ -28,8 +28,9 @@ import org.apache.drill.test.ClusterTest;
public class InteractiveUI extends ClusterTest {
public static void main(String[] args) {
- ClusterFixtureBuilder builder = new ClusterFixtureBuilder();
- builder.configBuilder().put(ExecConstants.HTTP_ENABLE, true);
+ ClusterFixtureBuilder builder = new ClusterFixtureBuilder()
+ .configProperty(ExecConstants.HTTP_ENABLE, true)
+ .configProperty(ExecConstants.HTTP_PORT_HUNT, true);
try {
startCluster(builder);
while (true) {
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/TestRestJson.java b/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/TestRestJson.java
index b4ca496..2c55459 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/TestRestJson.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/TestRestJson.java
@@ -71,6 +71,8 @@ public class TestRestJson extends ClusterTest {
MediaType.parse("application/json");
public static final int TIMEOUT = 3000; // for debugging
+ private static int portNumber;
+
protected static File testDir;
private final OkHttpClient httpClient = new OkHttpClient.Builder()
.connectTimeout(TIMEOUT, TimeUnit.SECONDS)
@@ -83,10 +85,13 @@ public class TestRestJson extends ClusterTest {
@BeforeClass
public static void setup() throws Exception {
- ClusterFixtureBuilder builder = new ClusterFixtureBuilder(dirTestWatcher);
- builder.configBuilder().put(ExecConstants.HTTP_ENABLE, true);
+ ClusterFixtureBuilder builder = new ClusterFixtureBuilder(dirTestWatcher)
+ .configProperty(ExecConstants.HTTP_ENABLE, true)
+ .configProperty(ExecConstants.HTTP_PORT_HUNT, true);
startCluster(builder);
+ portNumber = cluster.drillbit().getWebServerPort();
+
// Set up CSV storage plugin using headers.
TextFormatConfig csvFormat = new TextFormatConfig(
null,
@@ -217,7 +222,7 @@ public class TestRestJson extends ClusterTest {
private void runQuery(QueryWrapper query, File destFile) throws IOException {
ObjectWriter writer = mapper.writerFor(QueryWrapper.class);
String json = writer.writeValueAsString(query);
- String url = "http://localhost:8047/query.json";
+ String url = String.format("http://localhost:%d/query.json", portNumber);
Request request = new Request.Builder()
.url(url)
.post(RequestBody.create(json, JSON_MEDIA_TYPE))
@@ -232,7 +237,7 @@ public class TestRestJson extends ClusterTest {
private void runQuery(QueryWrapper query) throws IOException {
ObjectWriter writer = mapper.writerFor(QueryWrapper.class);
String json = writer.writeValueAsString(query);
- String url = "http://localhost:8047/query.json";
+ String url = String.format("http://localhost:%d/query.json", portNumber);
Request request = new Request.Builder()
.url(url)
.post(RequestBody.create(json, JSON_MEDIA_TYPE))
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/util/ExecutorServiceUtilTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/util/ExecutorServiceUtilTest.java
index e2101ad..097df20 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/util/ExecutorServiceUtilTest.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/util/ExecutorServiceUtilTest.java
@@ -35,7 +35,7 @@ import org.junit.Test;
public final class ExecutorServiceUtilTest extends DrillTest {
@Test
- public void testSuccessfulExecution() throws Exception {
+ public void testSuccessfulExecution() {
final int numThreads = 2;
final int numTasks = 20;
ExecutorService service = Executors.newFixedThreadPool(numThreads);
@@ -74,7 +74,7 @@ public final class ExecutorServiceUtilTest extends DrillTest {
}
@Test
- public void testFailedExecution() throws Exception {
+ public void testFailedExecution() {
final int numThreads = 2;
final int numTasks = 20;
ExecutorService service = Executors.newFixedThreadPool(numThreads);
@@ -117,7 +117,7 @@ public final class ExecutorServiceUtilTest extends DrillTest {
}
@Test
- public void testMixedExecution() throws Exception {
+ public void testMixedExecution() {
final int numThreads = 2;
final int numTasks = 20;
ExecutorService service = Executors.newFixedThreadPool(numThreads);
@@ -133,7 +133,7 @@ public final class ExecutorServiceUtilTest extends DrillTest {
// Launch the tasks
for (int idx = 0; idx < numTasks; idx++) {
- CallableTask task = null;
+ CallableTask task;
if (idx % 2 == 0) {
task = new CallableTask(successParams);
@@ -172,10 +172,10 @@ public final class ExecutorServiceUtilTest extends DrillTest {
}
@Test
- public void testCancelExecution() throws Exception {
+ public void testCancelExecution() {
final int numThreads = 2;
ExecutorService service = Executors.newFixedThreadPool(numThreads);
- RequestContainer request = null;
+ RequestContainer request;
// Set the test parameters
TestParams params = new TestParams();
@@ -224,7 +224,7 @@ public final class ExecutorServiceUtilTest extends DrillTest {
SUCCEEDED,
FAILED,
CANCELLED
- };
+ }
private ExecutionStatus status;
@@ -252,12 +252,12 @@ public final class ExecutorServiceUtilTest extends DrillTest {
@SuppressWarnings("unused")
private static final class TaskExecutionController {
- private boolean canStart = false;
- private boolean canExit = false;
- private boolean started = false;
- private boolean exited = false;
- private int delayMillisOnExit = 0;
- private Object monitor = new Object();
+ private volatile boolean canStart = false;
+ private volatile boolean canExit = false;
+ private volatile boolean started = false;
+ private volatile boolean exited = false;
+ private volatile int delayMillisOnExit = 0;
+ private final Object monitor = new Object();
private void canStart() {
synchronized(monitor) {
@@ -340,13 +340,13 @@ public final class ExecutorServiceUtilTest extends DrillTest {
}
private static final class TestParams {
- private int waitTimeMillis = 2;
+ private final int waitTimeMillis = 2;
private boolean generateException = false;
private TaskExecutionController controller = null;
}
private static final class CallableTask implements Callable<TaskResult> {
- private volatile TaskResult result = new TaskResult();
+ private final TaskResult result = new TaskResult();
private final TestParams params;
private CallableTask(TestParams params) {
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/util/TestQueryMemoryAlloc.java b/exec/java-exec/src/test/java/org/apache/drill/exec/util/TestQueryMemoryAlloc.java
index 14a9447..28405ce 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/util/TestQueryMemoryAlloc.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/util/TestQueryMemoryAlloc.java
@@ -67,17 +67,17 @@ public class TestQueryMemoryAlloc extends DrillTest {
public void testCustomFloor() throws Exception {
OperatorFixture.Builder builder = OperatorFixture.builder(dirTestWatcher);
builder.systemOption(ExecConstants.PERCENT_MEMORY_PER_QUERY_KEY, 0.05);
- builder.systemOption(ExecConstants.MAX_QUERY_MEMORY_PER_NODE_KEY, 3 * ONE_GB);
+ builder.systemOption(ExecConstants.MAX_QUERY_MEMORY_PER_NODE_KEY, 2 * ONE_GB);
try (OperatorFixture fixture = builder.build()) {
final OptionManager optionManager = fixture.getOptionManager();
optionManager.setLocalOption(ExecConstants.PERCENT_MEMORY_PER_QUERY_KEY, 0.05);
- optionManager.setLocalOption(ExecConstants.MAX_QUERY_MEMORY_PER_NODE_KEY, 3 * ONE_GB);
+ optionManager.setLocalOption(ExecConstants.MAX_QUERY_MEMORY_PER_NODE_KEY, 2 * ONE_GB);
// Out-of-box memory, use query memory per node as floor.
long mem = MemoryAllocationUtilities.computeQueryMemory(fixture.config(), optionManager, 8 * ONE_GB);
- assertEquals(3 * ONE_GB, mem);
+ assertEquals(2 * ONE_GB, mem);
// Up to 60 GB, query memory dominates.
diff --git a/pom.xml b/pom.xml
index 6b2bd1a..e4973ff 100644
--- a/pom.xml
+++ b/pom.xml
@@ -95,8 +95,8 @@
<jersey.version>2.25.1</jersey.version>
<asm.version>7.3.1</asm.version>
<excludedGroups />
- <memoryMb>4096</memoryMb>
- <directMemoryMb>4096</directMemoryMb>
+ <memoryMb>4000</memoryMb>
+ <directMemoryMb>2500</directMemoryMb>
<rat.skip>true</rat.skip>
<license.skip>true</license.skip>
<docker.repository>apache/drill</docker.repository>