You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@druid.apache.org by ji...@apache.org on 2022/01/25 18:15:02 UTC
[druid] branch master updated: Wait for datasource to be ready for SQL in integration tests (#12189)
This is an automated email from the ASF dual-hosted git repository.
jihoonson pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new 20347e0 Wait for datasource to be ready for SQL in integration tests (#12189)
20347e0 is described below
commit 20347e0c86375693c7a23489559ff199299b7a45
Author: Jihoon Son <ji...@apache.org>
AuthorDate: Tue Jan 25 10:14:26 2022 -0800
Wait for datasource to be ready for SQL in integration tests (#12189)
* Wait for datasource to be ready for SQL in integration tests
* add limit to the check query
---
...lTestQueryHelper.java => DataLoaderHelper.java} | 33 ++++++++++++----------
.../druid/testing/utils/SqlTestQueryHelper.java | 26 +++++++++++++++++
.../tests/query/ITBroadcastJoinQueryTest.java | 8 ++++--
.../apache/druid/tests/query/ITJdbcQueryTest.java | 9 ++----
.../apache/druid/tests/query/ITQueryErrorTest.java | 9 ++----
.../apache/druid/tests/query/ITSqlCancelTest.java | 9 ++----
.../druid/tests/query/ITSystemTableQueryTest.java | 13 +++------
7 files changed, 62 insertions(+), 45 deletions(-)
diff --git a/integration-tests/src/main/java/org/apache/druid/testing/utils/SqlTestQueryHelper.java b/integration-tests/src/main/java/org/apache/druid/testing/utils/DataLoaderHelper.java
similarity index 58%
copy from integration-tests/src/main/java/org/apache/druid/testing/utils/SqlTestQueryHelper.java
copy to integration-tests/src/main/java/org/apache/druid/testing/utils/DataLoaderHelper.java
index 373f3cb..76b4300 100644
--- a/integration-tests/src/main/java/org/apache/druid/testing/utils/SqlTestQueryHelper.java
+++ b/integration-tests/src/main/java/org/apache/druid/testing/utils/DataLoaderHelper.java
@@ -19,28 +19,31 @@
package org.apache.druid.testing.utils;
-import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.inject.Inject;
import org.apache.druid.java.util.common.StringUtils;
-import org.apache.druid.testing.IntegrationTestingConfig;
-import org.apache.druid.testing.clients.SqlResourceTestClient;
+import org.apache.druid.testing.clients.CoordinatorResourceTestClient;
-public class SqlTestQueryHelper extends AbstractTestQueryHelper<SqlQueryWithResults>
+public final class DataLoaderHelper
{
+ @Inject
+ private SqlTestQueryHelper sqlTestQueryHelper;
@Inject
- public SqlTestQueryHelper(
- ObjectMapper jsonMapper,
- SqlResourceTestClient sqlClient,
- IntegrationTestingConfig config
- )
- {
- super(jsonMapper, sqlClient, config);
- }
+ private CoordinatorResourceTestClient coordinator;
- @Override
- public String getQueryURL(String schemeAndHost)
+ public void waitUntilDatasourceIsReady(String datasource)
{
- return StringUtils.format("%s/druid/v2/sql", schemeAndHost);
+ ITRetryUtil.retryUntilTrue(
+ () -> coordinator.areSegmentsLoaded(datasource),
+ StringUtils.format(
+ "Segment Load for datasource [%s]",
+ datasource
+ )
+ );
+
+ ITRetryUtil.retryUntilTrue(
+ () -> sqlTestQueryHelper.isDatasourceLoadedInSQL(datasource),
+ StringUtils.format("Waiting for [%s] to be ready for SQL queries", datasource)
+ );
}
}
diff --git a/integration-tests/src/main/java/org/apache/druid/testing/utils/SqlTestQueryHelper.java b/integration-tests/src/main/java/org/apache/druid/testing/utils/SqlTestQueryHelper.java
index 373f3cb..fe9a324 100644
--- a/integration-tests/src/main/java/org/apache/druid/testing/utils/SqlTestQueryHelper.java
+++ b/integration-tests/src/main/java/org/apache/druid/testing/utils/SqlTestQueryHelper.java
@@ -22,11 +22,14 @@ package org.apache.druid.testing.utils;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.inject.Inject;
import org.apache.druid.java.util.common.StringUtils;
+import org.apache.druid.java.util.common.logger.Logger;
+import org.apache.druid.sql.http.SqlQuery;
import org.apache.druid.testing.IntegrationTestingConfig;
import org.apache.druid.testing.clients.SqlResourceTestClient;
public class SqlTestQueryHelper extends AbstractTestQueryHelper<SqlQueryWithResults>
{
+ private static final Logger LOG = new Logger(SqlTestQueryHelper.class);
@Inject
public SqlTestQueryHelper(
@@ -43,4 +46,27 @@ public class SqlTestQueryHelper extends AbstractTestQueryHelper<SqlQueryWithResu
{
return StringUtils.format("%s/druid/v2/sql", schemeAndHost);
}
+
+ public boolean isDatasourceLoadedInSQL(String datasource)
+ {
+ final SqlQuery query = new SqlQuery(
+ "SELECT 1 FROM \"" + datasource + "\" LIMIT 1",
+ null,
+ false,
+ false,
+ false,
+ null,
+ null
+ );
+
+ try {
+ //noinspection unchecked
+ queryClient.query(getQueryURL(broker), query);
+ return true;
+ }
+ catch (Exception e) {
+ LOG.debug(e, "Check query failed");
+ return false;
+ }
+ }
}
diff --git a/integration-tests/src/test/java/org/apache/druid/tests/query/ITBroadcastJoinQueryTest.java b/integration-tests/src/test/java/org/apache/druid/tests/query/ITBroadcastJoinQueryTest.java
index 481680b..9642199 100644
--- a/integration-tests/src/test/java/org/apache/druid/tests/query/ITBroadcastJoinQueryTest.java
+++ b/integration-tests/src/test/java/org/apache/druid/tests/query/ITBroadcastJoinQueryTest.java
@@ -31,6 +31,7 @@ import org.apache.druid.testing.IntegrationTestingConfig;
import org.apache.druid.testing.clients.CoordinatorResourceTestClient;
import org.apache.druid.testing.guice.DruidTestModuleFactory;
import org.apache.druid.testing.guice.TestClient;
+import org.apache.druid.testing.utils.DataLoaderHelper;
import org.apache.druid.testing.utils.ITRetryUtil;
import org.apache.druid.testing.utils.SqlTestQueryHelper;
import org.apache.druid.tests.TestNGGroup;
@@ -60,6 +61,9 @@ public class ITBroadcastJoinQueryTest extends AbstractIndexerTest
SqlTestQueryHelper queryHelper;
@Inject
+ DataLoaderHelper dataLoaderHelper;
+
+ @Inject
@TestClient
HttpClient httpClient;
@@ -94,9 +98,7 @@ public class ITBroadcastJoinQueryTest extends AbstractIndexerTest
String taskJson = replaceJoinTemplate(getResourceAsString(BROADCAST_JOIN_TASK), BROADCAST_JOIN_DATASOURCE);
indexer.submitTask(taskJson);
- ITRetryUtil.retryUntilTrue(
- () -> coordinatorClient.areSegmentsLoaded(BROADCAST_JOIN_DATASOURCE), "broadcast segment load"
- );
+ dataLoaderHelper.waitUntilDatasourceIsReady(BROADCAST_JOIN_DATASOURCE);
// query metadata until druid schema is refreshed and datasource is available joinable
ITRetryUtil.retryUntilTrue(
diff --git a/integration-tests/src/test/java/org/apache/druid/tests/query/ITJdbcQueryTest.java b/integration-tests/src/test/java/org/apache/druid/tests/query/ITJdbcQueryTest.java
index 46fb177..6e39440 100644
--- a/integration-tests/src/test/java/org/apache/druid/tests/query/ITJdbcQueryTest.java
+++ b/integration-tests/src/test/java/org/apache/druid/tests/query/ITJdbcQueryTest.java
@@ -26,9 +26,8 @@ import org.apache.druid.https.SSLClientConfig;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.java.util.common.logger.Logger;
import org.apache.druid.testing.IntegrationTestingConfig;
-import org.apache.druid.testing.clients.CoordinatorResourceTestClient;
import org.apache.druid.testing.guice.DruidTestModuleFactory;
-import org.apache.druid.testing.utils.ITRetryUtil;
+import org.apache.druid.testing.utils.DataLoaderHelper;
import org.apache.druid.tests.TestNGGroup;
import org.testng.Assert;
import org.testng.annotations.BeforeMethod;
@@ -77,7 +76,7 @@ public class ITJdbcQueryTest
SSLClientConfig sslConfig;
@Inject
- private CoordinatorResourceTestClient coordinatorClient;
+ private DataLoaderHelper dataLoaderHelper;
@BeforeMethod
public void before()
@@ -108,9 +107,7 @@ public class ITJdbcQueryTest
)
};
// ensure that wikipedia segments are loaded completely
- ITRetryUtil.retryUntilTrue(
- () -> coordinatorClient.areSegmentsLoaded(WIKIPEDIA_DATA_SOURCE), "wikipedia segment load"
- );
+ dataLoaderHelper.waitUntilDatasourceIsReady(WIKIPEDIA_DATA_SOURCE);
}
@Test
diff --git a/integration-tests/src/test/java/org/apache/druid/tests/query/ITQueryErrorTest.java b/integration-tests/src/test/java/org/apache/druid/tests/query/ITQueryErrorTest.java
index 9abc941..fd8ce5a 100644
--- a/integration-tests/src/test/java/org/apache/druid/tests/query/ITQueryErrorTest.java
+++ b/integration-tests/src/test/java/org/apache/druid/tests/query/ITQueryErrorTest.java
@@ -24,9 +24,8 @@ import com.google.inject.Inject;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.query.QueryContexts;
import org.apache.druid.server.coordination.ServerManagerForQueryErrorTest;
-import org.apache.druid.testing.clients.CoordinatorResourceTestClient;
import org.apache.druid.testing.guice.DruidTestModuleFactory;
-import org.apache.druid.testing.utils.ITRetryUtil;
+import org.apache.druid.testing.utils.DataLoaderHelper;
import org.apache.druid.testing.utils.SqlTestQueryHelper;
import org.apache.druid.testing.utils.TestQueryHelper;
import org.apache.druid.tests.TestNGGroup;
@@ -69,7 +68,7 @@ public class ITQueryErrorTest
private static final String SQL_PLAN_FAILURE_RESOURCE = "/queries/sql_plan_failure_query.json";
@Inject
- private CoordinatorResourceTestClient coordinatorClient;
+ private DataLoaderHelper dataLoaderHelper;
@Inject
private TestQueryHelper queryHelper;
@Inject
@@ -81,9 +80,7 @@ public class ITQueryErrorTest
public void before()
{
// ensure that wikipedia segments are loaded completely
- ITRetryUtil.retryUntilTrue(
- () -> coordinatorClient.areSegmentsLoaded(WIKIPEDIA_DATA_SOURCE), "wikipedia segment load"
- );
+ dataLoaderHelper.waitUntilDatasourceIsReady(WIKIPEDIA_DATA_SOURCE);
}
@Test(expectedExceptions = {RuntimeException.class}, expectedExceptionsMessageRegExp = "(?s).*400.*")
diff --git a/integration-tests/src/test/java/org/apache/druid/tests/query/ITSqlCancelTest.java b/integration-tests/src/test/java/org/apache/druid/tests/query/ITSqlCancelTest.java
index e8a3fe5..3d3d43e 100644
--- a/integration-tests/src/test/java/org/apache/druid/tests/query/ITSqlCancelTest.java
+++ b/integration-tests/src/test/java/org/apache/druid/tests/query/ITSqlCancelTest.java
@@ -30,10 +30,9 @@ import org.apache.druid.query.QueryException;
import org.apache.druid.query.QueryInterruptedException;
import org.apache.druid.sql.http.SqlQuery;
import org.apache.druid.testing.IntegrationTestingConfig;
-import org.apache.druid.testing.clients.CoordinatorResourceTestClient;
import org.apache.druid.testing.clients.SqlResourceTestClient;
import org.apache.druid.testing.guice.DruidTestModuleFactory;
-import org.apache.druid.testing.utils.ITRetryUtil;
+import org.apache.druid.testing.utils.DataLoaderHelper;
import org.apache.druid.testing.utils.SqlTestQueryHelper;
import org.apache.druid.tests.TestNGGroup;
import org.jboss.netty.handler.codec.http.HttpResponseStatus;
@@ -61,7 +60,7 @@ public class ITSqlCancelTest
private static final int NUM_QUERIES = 3;
@Inject
- private CoordinatorResourceTestClient coordinatorClient;
+ private DataLoaderHelper dataLoaderHelper;
@Inject
private SqlTestQueryHelper sqlHelper;
@Inject
@@ -75,9 +74,7 @@ public class ITSqlCancelTest
public void before()
{
// ensure that wikipedia segments are loaded completely
- ITRetryUtil.retryUntilTrue(
- () -> coordinatorClient.areSegmentsLoaded(WIKIPEDIA_DATA_SOURCE), "wikipedia segment load"
- );
+ dataLoaderHelper.waitUntilDatasourceIsReady(WIKIPEDIA_DATA_SOURCE);
}
@Test
diff --git a/integration-tests/src/test/java/org/apache/druid/tests/query/ITSystemTableQueryTest.java b/integration-tests/src/test/java/org/apache/druid/tests/query/ITSystemTableQueryTest.java
index bb638ec..df00488 100644
--- a/integration-tests/src/test/java/org/apache/druid/tests/query/ITSystemTableQueryTest.java
+++ b/integration-tests/src/test/java/org/apache/druid/tests/query/ITSystemTableQueryTest.java
@@ -21,9 +21,8 @@ package org.apache.druid.tests.query;
import com.google.inject.Inject;
import org.apache.druid.testing.IntegrationTestingConfig;
-import org.apache.druid.testing.clients.CoordinatorResourceTestClient;
import org.apache.druid.testing.guice.DruidTestModuleFactory;
-import org.apache.druid.testing.utils.ITRetryUtil;
+import org.apache.druid.testing.utils.DataLoaderHelper;
import org.apache.druid.testing.utils.SqlTestQueryHelper;
import org.apache.druid.tests.TestNGGroup;
import org.testng.annotations.BeforeMethod;
@@ -39,7 +38,7 @@ public class ITSystemTableQueryTest
private static final String SYSTEM_QUERIES_RESOURCE = "/queries/sys_queries.json";
@Inject
- CoordinatorResourceTestClient coordinatorClient;
+ DataLoaderHelper dataLoaderHelper;
@Inject
private SqlTestQueryHelper queryHelper;
@Inject
@@ -49,14 +48,10 @@ public class ITSystemTableQueryTest
public void before()
{
// ensure that wikipedia segments are loaded completely
- ITRetryUtil.retryUntilTrue(
- () -> coordinatorClient.areSegmentsLoaded(WIKIPEDIA_DATA_SOURCE), "wikipedia segment load"
- );
+ dataLoaderHelper.waitUntilDatasourceIsReady(WIKIPEDIA_DATA_SOURCE);
// ensure that the twitter segments are loaded completely
- ITRetryUtil.retryUntilTrue(
- () -> coordinatorClient.areSegmentsLoaded(TWITTER_DATA_SOURCE), "twitter segment load"
- );
+ dataLoaderHelper.waitUntilDatasourceIsReady(TWITTER_DATA_SOURCE);
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org