You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by sb...@apache.org on 2018/12/25 14:24:19 UTC
[46/50] [abbrv] ignite git commit: IGNITE-10621: SQL: Collect running
query info for all types of queries. This closes #5620. This closes #5663.
http://git-wip-us.apache.org/repos/asf/ignite/blob/592cc346/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/RunningQueriesTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/RunningQueriesTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/RunningQueriesTest.java
index f0efb63..4c29007 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/RunningQueriesTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/RunningQueriesTest.java
@@ -17,82 +17,598 @@
package org.apache.ignite.internal.processors.query;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.sql.Statement;
import java.util.Collection;
import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.BrokenBarrierException;
import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import javax.cache.Cache;
+import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteException;
import org.apache.ignite.cache.QueryEntity;
+import org.apache.ignite.cache.query.FieldsQueryCursor;
import org.apache.ignite.cache.query.SqlFieldsQuery;
import org.apache.ignite.cache.query.SqlQuery;
-import org.apache.ignite.cache.query.annotations.QuerySqlFunction;
+import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.IgniteInterruptedCheckedException;
+import org.apache.ignite.internal.managers.communication.GridIoMessage;
+import org.apache.ignite.internal.managers.discovery.CustomMessageWrapper;
+import org.apache.ignite.internal.managers.discovery.DiscoveryCustomMessage;
+import org.apache.ignite.internal.processors.cache.DynamicCacheChangeBatch;
+import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridNearAtomicFullUpdateRequest;
+import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridNearAtomicSingleUpdateFilterRequest;
+import org.apache.ignite.internal.processors.cache.mvcc.MvccQueryTracker;
+import org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing;
+import org.apache.ignite.internal.processors.query.schema.message.SchemaProposeDiscoveryMessage;
+import org.apache.ignite.internal.util.typedef.G;
+import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
+import org.apache.ignite.spi.discovery.DiscoverySpiCustomMessage;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
import org.apache.ignite.testframework.GridTestUtils;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.jetbrains.annotations.Nullable;
+import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
+import static org.apache.ignite.internal.util.IgniteUtils.resolveIgnitePath;
+
/**
* Tests for running queries.
*/
@RunWith(JUnit4.class)
public class RunningQueriesTest extends GridCommonAbstractTest {
- /**
- *
- */
- @Test
- public void testQueriesOriginalText() throws Exception {
- IgniteEx ignite = startGrid(0);
+ /** Timeout in sec. */
+ private static final long TIMEOUT_IN_SEC = 5;
+
+ /** Timeout in sec. */
+ private static final long TIMEOUT_IN_MS = TIMEOUT_IN_SEC * 1000;
+
+ /** Barrier. */
+ private static CyclicBarrier barrier;
+
+ /** Ignite. */
+ private static IgniteEx ignite;
+
+ /** Node count. */
+ private static final int NODE_CNT = 2;
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ super.beforeTestsStarted();
+
+ GridQueryProcessor.idxCls = BlockingIndexing.class;
+
+ ignite = (IgniteEx)startGrids(NODE_CNT);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ super.beforeTest();
+
+ ignite.destroyCache(DEFAULT_CACHE_NAME);
IgniteCache<Integer, Integer> cache = ignite.getOrCreateCache(new CacheConfiguration<Integer, Integer>()
- .setName("cache")
+ .setName(DEFAULT_CACHE_NAME)
.setQueryEntities(Collections.singletonList(new QueryEntity(Integer.class, Integer.class)))
- .setSqlFunctionClasses(TestSQLFunctions.class)
);
- cache.put(0, 0);
+ cache.put(100000, 0);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTestsStopped() throws Exception {
+ super.afterTestsStopped();
- GridTestUtils.runAsync(() -> cache.query(new SqlFieldsQuery(
- "SELECT * FROM /* comment */ Integer WHERE awaitBarrier() = 0")).getAll());
+ stopAllGrids();
+ }
- GridTestUtils.runAsync(() -> cache.query(new SqlQuery<Integer, Integer>(Integer.class,
- "FROM /* comment */ Integer WHERE awaitBarrier() = 0")).getAll());
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ super.afterTest();
- TestSQLFunctions.barrier.await();
+ Assert.assertEquals(0, barrier.getNumberWaiting());
- Collection<GridRunningQueryInfo> runningQueries = ignite.context().query().runningQueries(-1);
+ assertNoRunningQueries();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ cfg.setDiscoverySpi(new TcpDiscoverySpi() {
+
+ @Override public void sendCustomEvent(DiscoverySpiCustomMessage msg) throws IgniteException {
+ if (CustomMessageWrapper.class.isAssignableFrom(msg.getClass())) {
+ DiscoveryCustomMessage delegate = ((CustomMessageWrapper)msg).delegate();
- TestSQLFunctions.barrier.await();
+ if (DynamicCacheChangeBatch.class.isAssignableFrom(delegate.getClass())) {
+ ((DynamicCacheChangeBatch)delegate).requests().stream()
+ .filter((c) -> !c.cacheName().equalsIgnoreCase("default"))
+ .findAny()
+ .ifPresent((c) -> {
+ try {
+ awaitTimeouted();
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ }
+ });
+ }
+ else if (SchemaProposeDiscoveryMessage.class.isAssignableFrom(delegate.getClass())) {
+ try {
+ awaitTimeouted();
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ }
+
+ super.sendCustomEvent(msg);
+ }
+ });
+
+ cfg.setCommunicationSpi(new TcpCommunicationSpi() {
+ /** {@inheritDoc} */
+ @Override public void sendMessage(ClusterNode node, Message msg, IgniteInClosure<IgniteException> ackC) {
+ if (GridIoMessage.class.isAssignableFrom(msg.getClass())) {
+ Message gridMsg = ((GridIoMessage)msg).message();
+
+ if (GridNearAtomicSingleUpdateFilterRequest.class.isAssignableFrom(gridMsg.getClass())
+ || GridNearAtomicFullUpdateRequest.class.isAssignableFrom(gridMsg.getClass())
+ ) {
+ try {
+ awaitTimeouted();
+ }
+ catch (Exception ignore) {
+ }
+ }
+ }
+
+ super.sendMessage(node, msg, ackC);
+ }
+ });
+
+ return cfg;
+ }
+
+ /**
+ * Check tracking running queries for Select.
+ */
+ @Test
+ public void testQueries() throws Exception {
+ newBarrier(3);
+
+ IgniteCache<Object, Object> cache = ignite.cache(DEFAULT_CACHE_NAME);
+
+ IgniteInternalFuture<List<List<?>>> fut1 = GridTestUtils.runAsync(() -> cache.query(new SqlFieldsQuery(
+ "SELECT * FROM /* comment */ Integer WHERE 1 = 1")).getAll());
+
+ IgniteInternalFuture<List<Cache.Entry<Integer, Integer>>> fut2 = GridTestUtils.runAsync(() -> cache.query(
+ new SqlQuery<Integer, Integer>(Integer.class, "FROM /* comment */ Integer WHERE 1 = 1"))
+ .getAll());
+
+ Assert.assertTrue(GridTestUtils.waitForCondition(
+ () -> barrier.getNumberWaiting() == 2, TIMEOUT_IN_MS));
+
+ Collection<GridRunningQueryInfo> runningQueries = ignite.context().query().runningQueries(-1);
assertEquals(2, runningQueries.size());
for (GridRunningQueryInfo info : runningQueries)
assertTrue("Failed to find comment in query: " + info.query(), info.query().contains("/* comment */"));
+
+ assertNoRunningQueries(ignite);
+
+ awaitTimeouted();
+
+ fut1.get(TIMEOUT_IN_MS);
+
+ fut2.get(TIMEOUT_IN_MS);
+ }
+
+ /**
+ * Check tracking running queries for DELETE.
+ */
+ @Test
+ public void testQueryDmlDelete() throws Exception {
+ testQueryDML("DELETE FROM /* comment */ Integer");
+ }
+
+ /**
+ * Check tracking running queries for INSERT.
+ */
+ @Test
+ public void testQueryDmlInsert() throws Exception {
+ testQueryDML("INSERT INTO Integer(_key, _val) VALUES(1,1)");
+ }
+
+ /**
+ * Check tracking running queries for UPDATE.
+ */
+ @Test
+ public void testQueryDmlUpdate() throws Exception {
+ testQueryDML("UPDATE Integer set _val = 1 where 1=1");
+ }
+
+ /**
+ * Check tracking running queries for DML.
+ *
+ * @param dmlQry DML query.
+ */
+ public void testQueryDML(String dmlQry) throws Exception {
+ newBarrier(2);
+
+ IgniteCache<Object, Object> cache = ignite.cache(DEFAULT_CACHE_NAME);
+
+ SqlFieldsQuery qry = new SqlFieldsQuery(dmlQry);
+
+ IgniteInternalFuture<List<List<?>>> fut = GridTestUtils.runAsync(() -> cache.query(qry).getAll());
+
+ assertWaitingOnBarrier();
+
+ Collection<GridRunningQueryInfo> runningQueries = ignite.context().query().runningQueries(-1);
+
+ assertEquals(1, runningQueries.size());
+
+ assertNoRunningQueries(ignite);
+
+ runningQueries.forEach((info) -> Assert.assertEquals(qry.getSql(), info.query()));
+
+ IgniteInternalFuture<Integer> fut1 = GridTestUtils.runAsync(() -> barrier.await());
+
+ awaitTimeouted();
+
+ fut1.get(TIMEOUT_IN_MS);
+
+ fut.get(TIMEOUT_IN_MS);
}
/**
- * Utility class with custom SQL functions.
+ * Check tracking running queries for DROP INDEX.
*/
- public static class TestSQLFunctions {
- /** Barrier. */
- static CyclicBarrier barrier = new CyclicBarrier(3);
+ @Test
+ public void testQueryDdlDropIndex() throws Exception {
+ newBarrier(1);
+
+ ignite.cache(DEFAULT_CACHE_NAME).query(new SqlFieldsQuery("CREATE TABLE tst_idx_drop(id long PRIMARY KEY, cnt integer)"));
+
+ ignite.cache(DEFAULT_CACHE_NAME).query(new SqlFieldsQuery("CREATE INDEX tst_idx_drop_idx ON default.tst_idx_drop(cnt)"));
+
+ testQueryDDL("DROP INDEX default.tst_idx_drop_idx");
+ }
+
+ /**
+ * Check tracking running queries for CREATE INDEX.
+ */
+ @Test
+ public void testQueryDdlCreateIndex() throws Exception {
+ newBarrier(1);
+
+ ignite.cache(DEFAULT_CACHE_NAME).query(new SqlFieldsQuery("CREATE TABLE tst_idx_create(id long PRIMARY KEY, cnt integer)"));
+
+ testQueryDDL("CREATE INDEX tst_idx_create_idx ON default.tst_idx_create(cnt)");
+ }
+
+ /**
+ * Check tracking running queries for DROP TABLE.
+ */
+ @Test
+ public void testQueryDdlDropTable() throws Exception {
+ newBarrier(1);
+
+ ignite.cache(DEFAULT_CACHE_NAME).query(new SqlFieldsQuery("CREATE TABLE tst_drop(id long PRIMARY KEY, cnt integer)"));
+
+ testQueryDDL("DROP TABLE default.tst_drop");
+ }
+
+ /**
+ * Check tracking running queries for CREATE TABLE.
+ */
+ @Test
+ public void testQueryDdlCreateTable() throws Exception {
+ testQueryDDL("CREATE TABLE tst_create(id long PRIMARY KEY, cnt integer)");
+ }
+
+ /**
+ * Check tracking running queries for DDL.
+ */
+ public void testQueryDDL(String sql) throws Exception {
+ newBarrier(2);
+
+ IgniteCache<Object, Object> cache = ignite.cache(DEFAULT_CACHE_NAME);
+
+ SqlFieldsQuery qry = new SqlFieldsQuery(sql);
+
+ IgniteInternalFuture<List<List<?>>> fut = GridTestUtils.runAsync(() -> cache.query(qry).getAll());
+
+ assertWaitingOnBarrier();
+
+ Collection<GridRunningQueryInfo> runningQueries = ignite.context().query().runningQueries(-1);
+
+ assertEquals(1, runningQueries.size());
+
+ assertNoRunningQueries(ignite);
+
+ runningQueries.forEach((info) -> Assert.assertEquals(qry.getSql(), info.query()));
+
+ awaitTimeouted();
+
+ awaitTimeouted();
+
+ fut.get(TIMEOUT_IN_MS);
+ }
+
+ /**
+ * Check tracking running queries for batches.
+ *
+ * @throws Exception Exception in case of failure.
+ */
+ @Test
+ public void testJdbcBatchDML() throws Exception {
+ newBarrier(2);
+
+ try (Connection conn = GridTestUtils.connect(ignite, null); Statement stmt = conn.createStatement()) {
+ conn.setSchema("\"default\"");
+
+ final int BATCH_SIZE = 10;
+
+ int key = 0;
+
+ for (int i = 0; i < BATCH_SIZE; i++) {
+ while (ignite.affinity(DEFAULT_CACHE_NAME).isPrimary(ignite.localNode(), key))
+ key++;
+
+ stmt.addBatch("insert into Integer (_key, _val) values (" + key + "," + key + ")");
+
+ key++;
+ }
+
+ IgniteInternalFuture<int[]> fut = GridTestUtils.runAsync(stmt::executeBatch);
+
+ for (int i = 0; i < BATCH_SIZE; i++) {
+ assertWaitingOnBarrier();
+
+ Collection<GridRunningQueryInfo> runningQueries = ignite.context().query().runningQueries(-1);
+
+ assertEquals(1, runningQueries.size());
+
+ awaitTimeouted();
+
+ assertWaitingOnBarrier();
+
+ awaitTimeouted();
+ }
+
+ fut.get(TIMEOUT_IN_MS);
+ }
+ }
+
+ /**
+ * Check tracking running queries for multistatements.
+ *
+ * @throws Exception Exception in case of failure.
+ */
+ @Test
+ public void testMultiStatement() throws Exception {
+ newBarrier(2);
+
+ int key = 0;
+
+ int[] notAffinityKey = new int[2];
+
+ for (int i = 0; i < notAffinityKey.length; i++) {
+ while (ignite.affinity(DEFAULT_CACHE_NAME).isPrimary(ignite.localNode(), key))
+ key++;
+
+ notAffinityKey[i] = key;
+
+ key++;
+ }
+
+ String[] queries = {
+ "create table test(ID int primary key, NAME varchar(20))",
+ "insert into test (ID, NAME) values (" + notAffinityKey[0] + ", 'name')",
+ "insert into test (ID, NAME) values (" + notAffinityKey[1] + ", 'name')",
+ "SELECT * FROM test"
+ };
+
+ String sql = String.join(";", queries);
+
+ try (Connection conn = GridTestUtils.connect(ignite, null); Statement stmt = conn.createStatement()) {
+ IgniteInternalFuture<Boolean> fut = GridTestUtils.runAsync(() -> stmt.execute(sql));
+
+ for (String query : queries) {
+ assertWaitingOnBarrier();
+
+ List<GridRunningQueryInfo> runningQueries = (List<GridRunningQueryInfo>)ignite.context().query()
+ .runningQueries(-1);
+
+ assertEquals(1, runningQueries.size());
+
+ assertEquals(query, runningQueries.get(0).query());
+
+ awaitTimeouted();
+ }
+
+ fut.get(TIMEOUT_IN_MS);
+ }
+ }
+
+ /**
+ * Check tracking running queries for stream batching.
+ *
+ * @throws Exception in case of failure.
+ */
+ @Test
+ public void testJdbcStreamBatchUpdate() throws Exception {
+ try (Connection conn = GridTestUtils.connect(ignite, null); Statement stmt = conn.createStatement()) {
+ conn.setSchema("\"default\"");
+
+ newBarrier(1);
+
+ final int BATCH_SIZE = 10;
+
+ stmt.executeUpdate("SET STREAMING ON BATCH_SIZE " + BATCH_SIZE);
+
+ newBarrier(2);
+
+ for (int i = 0; i < BATCH_SIZE; i++)
+ stmt.addBatch("insert into Integer (_key, _val) values (" + i + "," + i + ")");
+
+ for (int i = 0; i < BATCH_SIZE; i++) {
+ assertWaitingOnBarrier();
+
+ awaitTimeouted();
+
+ assertWaitingOnBarrier();
+
+ Collection<GridRunningQueryInfo> runningQueries = ignite.context().query().runningQueries(-1);
+
+ assertEquals(1, runningQueries.size());
+
+ awaitTimeouted();
+ }
+ }
+ }
+
+ /**
+ * Check tracking running queries for stream COPY command.
+ *
+ * @throws SQLException If failed.
+ */
+ @Test
+ public void testCopyCommand() throws Exception {
+ try (Connection conn = GridTestUtils.connect(ignite, null); Statement stmt = conn.createStatement()) {
+ conn.setSchema("\"default\"");
+
+ newBarrier(1);
+
+ stmt.execute("CREATE TABLE Person(id integer primary key, age integer, firstName varchar, lastname varchar)");
+
+ String path = Objects.requireNonNull(resolveIgnitePath("/modules/clients/src/test/resources/bulkload1.csv"))
+ .getAbsolutePath();
+
+ newBarrier(2);
+
+ String sql = "copy from '" + path + "'" +
+ " into Person" +
+ " (_key, age, firstName, lastName)" +
+ " format csv charset 'ascii'";
+
+ IgniteInternalFuture<Integer> fut = GridTestUtils.runAsync(() -> stmt.executeUpdate(sql));
+
+ assertWaitingOnBarrier();
+
+ List<GridRunningQueryInfo> runningQueries = (List<GridRunningQueryInfo>)ignite.context().query().runningQueries(-1);
+
+ assertEquals(1, runningQueries.size());
+
+ assertEquals(sql, runningQueries.get(0).query());
+
+ awaitTimeouted();
+
+ fut.get(TIMEOUT_IN_MS);
+ }
+ }
+
+ /**
+ * Assert that on barrier waiting one thread.
+ *
+ * @throws IgniteInterruptedCheckedException In case of failure.
+ */
+ private void assertWaitingOnBarrier() throws IgniteInterruptedCheckedException {
+ Assert.assertTrue("Still waiting " + barrier.getNumberWaiting() + " parties",
+ GridTestUtils.waitForCondition(() -> barrier.getNumberWaiting() == 1, TIMEOUT_IN_MS));
+ }
+
+ /**
+ * Check all nodes except passed as parameter on no any running queries.
+ *
+ * @param excludeNodes Nodes shich will be excluded from check.
+ */
+ private void assertNoRunningQueries(IgniteEx... excludeNodes) {
+ Set<UUID> excludeIds = Stream.of(excludeNodes).map((ignite) -> ignite.localNode().id()).collect(Collectors.toSet());
+
+ for (Ignite g : G.allGrids()) {
+ IgniteEx node = (IgniteEx)g;
+
+ if (!excludeIds.contains(node.localNode().id())) {
+ Collection<GridRunningQueryInfo> runningQueries = node.context().query().runningQueries(-1);
+
+ Assert.assertEquals(0, runningQueries.size());
+ }
+ }
+ }
+
+ /**
+ * Create and set new CyclicBarrier for the function.
+ *
+ * @param parties the number of threads that must invoke await method before the barrier is tripped
+ */
+ private static void newBarrier(int parties) {
+ barrier = new CyclicBarrier(parties);
+ }
+
+ /**
+ * @throws InterruptedException In case of failure.
+ * @throws TimeoutException In case of failure.
+ * @throws BrokenBarrierException In case of failure.
+ */
+ private static void awaitTimeouted() throws InterruptedException, TimeoutException, BrokenBarrierException {
+ barrier.await(TIMEOUT_IN_MS, TimeUnit.SECONDS);
+ }
+
+ /**
+ * Blocking indexing processor.
+ */
+ private static class BlockingIndexing extends IgniteH2Indexing {
+ /** {@inheritDoc} */
+ @Override public void checkStatementStreamable(PreparedStatement nativeStmt) {
+ super.checkStatementStreamable(nativeStmt);
- /**
- * Await cyclic barrier twice, first time to wait for enter method, second time to wait for collecting running
- * queries.
- */
- @QuerySqlFunction
- public static long awaitBarrier() {
try {
barrier.await();
- barrier.await();
}
- catch (Exception ignored) {
- // No-op.
+ catch (Exception e) {
+ throw new IgniteException(e);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public List<FieldsQueryCursor<List<?>>> querySqlFields(String schemaName, SqlFieldsQuery qry,
+ @Nullable SqlClientContext cliCtx, boolean keepBinary, boolean failOnMultipleStmts,
+ MvccQueryTracker tracker, GridQueryCancel cancel, boolean registerAsNewQry) {
+ List<FieldsQueryCursor<List<?>>> res = super.querySqlFields(schemaName, qry, cliCtx, keepBinary,
+ failOnMultipleStmts, tracker, cancel, registerAsNewQry);
+ try {
+ awaitTimeouted();
+ }
+ catch (Exception e) {
+ throw new IgniteException(e);
}
- return 0;
+ return res;
}
}
}