You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by am...@apache.org on 2019/07/19 12:20:02 UTC
[ignite] branch gg-21368 created (now 1855175)
This is an automated email from the ASF dual-hosted git repository.
amashenkov pushed a change to branch gg-21368
in repository https://gitbox.apache.org/repos/asf/ignite.git.
at 1855175 Merge branch 'master' into gg-21368
This branch includes the following new commits:
new 5c5769e GG-21394: missed slf4j-simple-1.7.7.jar in binary packages
new 6fb6519 GG-14870 AuthenticationEvent address does not contain full list of addresses
new 4c3d3be GG-20242 Updated dependencies to fix possible vulnerabilities
new 1b9a13b GG-21310 Deployed on GCP console can't display connected cluster (#284)
new a5f16e7 GG-20691 New source of query metrics. (#245)
new 5417a6c GG-20750: SQL: Fix performance drop with enabling global SQL memory quota.
new 4b045f5 GG-19117 GridCacheProcessor should add info about cache in exception message, if applicable.
new 04f305d GG-21455: ML Example TutorialStepByStepExample 0 node, linux fail
new f3017ea GG-21225 control.sh validate_indexes SQL Index issue add information about group and cache id.
new f7f0b89 GG-21254 Fixed *.testFailAfterStart failed on TC
new 00e531c GG-20750: SQL: Fix performance drop with enabling global SQL memory quota.
new fb2553a GG-21338 Fixed checkpointReadLock() may hang during node stop
new 5f7a54b GG-21468 Fix javadocs. (#289)
new 350aac24 GG-21210: IgniteCacheQueriesLoadTest1 scale factor
new 86ee0b5 GG-21232 control.sh if experimental command disabled - don't show help for experemental commands.
new c5a113f GG-21561 Remove slf4j-simple binding (Reverts GG-21394)
new 04c001e GG-21424 Fixed compatibility of cache filter of visor idle verify arguments
new bad5683 GG-21356 Usability: control utility new printed argument format.
new 1855175 Merge branch 'master' into gg-21368
The 19 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[ignite] 12/19: GG-21338 Fixed checkpointReadLock() may hang during
node stop
Posted by am...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
amashenkov pushed a commit to branch gg-21368
in repository https://gitbox.apache.org/repos/asf/ignite.git
commit fb2553a00aa4885280dcde883248742f9b753c07
Author: Ivan Bessonov <be...@gmail.com>
AuthorDate: Wed Jul 17 14:59:57 2019 +0300
GG-21338 Fixed checkpointReadLock() may hang during node stop
---
.../dht/topology/GridDhtPartitionTopologyImpl.java | 23 +++++++++++++++++-----
.../GridCacheDatabaseSharedManager.java | 6 +++---
2 files changed, 21 insertions(+), 8 deletions(-)
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java
index 134feac..2dec335 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java
@@ -2245,6 +2245,8 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
ctx.database().checkpointReadLock();
try {
+ Map<UUID, Set<Integer>> addToWaitGroups = new HashMap<>();
+
lock.writeLock().lock();
try {
@@ -2282,7 +2284,7 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
GridDhtPartitionState state = partMap.get(part);
- if (state == null || state != OWNING)
+ if (state != OWNING)
continue;
if (!newOwners.contains(remoteNodeId)) {
@@ -2302,9 +2304,7 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
UUID nodeId = entry.getKey();
Set<Integer> rebalancedParts = entry.getValue();
- // Add to wait groups to ensure late assignment switch after all partitions are rebalanced.
- for (Integer part : rebalancedParts)
- ctx.cache().context().affinity().addToWaitGroup(groupId(), part, nodeId, topologyVersionFuture().initialVersion());
+ addToWaitGroups.put(nodeId, new HashSet<>(rebalancedParts));
if (!rebalancedParts.isEmpty()) {
Set<Integer> historical = rebalancedParts.stream()
@@ -2323,9 +2323,22 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
}
node2part = new GridDhtPartitionFullMap(node2part, updateSeq.incrementAndGet());
- } finally {
+ }
+ finally {
lock.writeLock().unlock();
}
+
+ for (Map.Entry<UUID, Set<Integer>> entry : addToWaitGroups.entrySet()) {
+ // Add to wait groups to ensure late assignment switch after all partitions are rebalanced.
+ for (Integer part : entry.getValue()) {
+ ctx.cache().context().affinity().addToWaitGroup(
+ groupId(),
+ part,
+ entry.getKey(),
+ topologyVersionFuture().initialVersion()
+ );
+ }
+ }
}
finally {
ctx.database().checkpointReadUnlock();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index 42a9cdc..6e6992e 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -94,6 +94,7 @@ import org.apache.ignite.internal.NodeStoppingException;
import org.apache.ignite.internal.managers.discovery.GridDiscoveryManager;
import org.apache.ignite.internal.mem.DirectMemoryProvider;
import org.apache.ignite.internal.mem.DirectMemoryRegion;
+import org.apache.ignite.internal.metric.IoStatisticsHolderNoOp;
import org.apache.ignite.internal.pagemem.FullPageId;
import org.apache.ignite.internal.pagemem.PageIdAllocator;
import org.apache.ignite.internal.pagemem.PageIdUtils;
@@ -154,7 +155,6 @@ import org.apache.ignite.internal.processors.cache.persistence.wal.crc.IgniteDat
import org.apache.ignite.internal.processors.compress.CompressionProcessor;
import org.apache.ignite.internal.processors.port.GridPortRecord;
import org.apache.ignite.internal.processors.query.GridQueryProcessor;
-import org.apache.ignite.internal.metric.IoStatisticsHolderNoOp;
import org.apache.ignite.internal.util.GridConcurrentHashSet;
import org.apache.ignite.internal.util.GridMultiCollectionWrapper;
import org.apache.ignite.internal.util.GridReadOnlyArrayView;
@@ -950,7 +950,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
long time = System.currentTimeMillis();
- checkpointReadLock();
+ CHECKPOINT_LOCK_HOLD_COUNT.set(CHECKPOINT_LOCK_HOLD_COUNT.get() + 1);
try {
for (DatabaseLifecycleListener lsnr : getDatabaseListeners(cctx.kernalContext()))
@@ -982,7 +982,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
throw e;
}
finally {
- checkpointReadUnlock();
+ CHECKPOINT_LOCK_HOLD_COUNT.set(CHECKPOINT_LOCK_HOLD_COUNT.get() - 1);
}
}
[ignite] 03/19: GG-20242 Updated dependencies to fix possible
vulnerabilities
Posted by am...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
amashenkov pushed a commit to branch gg-21368
in repository https://gitbox.apache.org/repos/asf/ignite.git
commit 4c3d3be8fccfb7391aea634677bae042a20a57de
Author: Evgeniy Rudenko <er...@mifinity.com>
AuthorDate: Wed Jul 17 13:51:28 2019 +1000
GG-20242 Updated dependencies to fix possible vulnerabilities
---
modules/kafka/pom.xml | 2 +-
modules/mesos/pom.xml | 2 +-
modules/osgi-karaf/src/main/resources/features.xml | 4 ++--
modules/zookeeper/pom.xml | 4 ++--
parent/pom.xml | 14 +++++++-------
5 files changed, 13 insertions(+), 13 deletions(-)
diff --git a/modules/kafka/pom.xml b/modules/kafka/pom.xml
index a488fea..95fae4b 100644
--- a/modules/kafka/pom.xml
+++ b/modules/kafka/pom.xml
@@ -78,7 +78,7 @@
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-test</artifactId>
- <version>${curator.version}</version>
+ <version>${curator.test.version}</version>
<scope>test</scope>
<!-- https://github.com/confluentinc/kafka-connect-elasticsearch/issues/143 -->
<exclusions>
diff --git a/modules/mesos/pom.xml b/modules/mesos/pom.xml
index 687b14b..44525e2 100644
--- a/modules/mesos/pom.xml
+++ b/modules/mesos/pom.xml
@@ -34,7 +34,7 @@
<url>http://ignite.apache.org</url>
<properties>
- <mesos.version>1.5.0</mesos.version>
+ <mesos.version>1.5.2</mesos.version>
</properties>
<dependencies>
diff --git a/modules/osgi-karaf/src/main/resources/features.xml b/modules/osgi-karaf/src/main/resources/features.xml
index 76bbd33..3ea28e5 100644
--- a/modules/osgi-karaf/src/main/resources/features.xml
+++ b/modules/osgi-karaf/src/main/resources/features.xml
@@ -281,7 +281,7 @@
<![CDATA[The Apache Ignite Twitter module + dependencies.]]>
</details>
<feature prerequisite="true">wrap</feature>
- <bundle start="true" dependency="true">mvn:com.google.guava/guava/${guava14.version}</bundle>
+ <bundle start="true" dependency="true">mvn:com.google.guava/guava/${guava.version}</bundle>
<bundle start="true" dependency="true">wrap:mvn:com.twitter/hbc-core/${twitter.hbc.version}$Bundle-SymbolicName=Hosebird Client Core&Bundle-Version=${twitter.hbc.version}</bundle>
<bundle start="true" dependency="true">wrap:mvn:com.twitter/hbc-twitter4j/${twitter.hbc.version}$Bundle-SymbolicName=Hosebird Client Twitter4J&Bundle-Version=${twitter.hbc.version}</bundle>
<bundle start="true">mvn:org.apache.ignite/ignite-twitter/${project.version}</bundle>
@@ -322,7 +322,7 @@
<![CDATA[The Apache Ignite ZooKeeper module + dependencies.]]>
</details>
<bundle start="true" dependency="true">mvn:org.apache.zookeeper/zookeeper/${zookeeper.version}</bundle>
- <bundle start="true" dependency="true">mvn:com.google.guava/guava/${guava16.version}</bundle>
+ <bundle start="true" dependency="true">mvn:com.google.guava/guava/${guava.version}</bundle>
<bundle start="true" dependency="true">mvn:com.fasterxml.jackson.core/jackson-core/${jackson.version}</bundle>
<bundle start="true" dependency="true">mvn:com.fasterxml.jackson.core/jackson-databind/${jackson.version}</bundle>
<bundle start="true" dependency="true">mvn:org.apache.curator/curator-client/${curator.version}</bundle>
diff --git a/modules/zookeeper/pom.xml b/modules/zookeeper/pom.xml
index f627665..b801378 100644
--- a/modules/zookeeper/pom.xml
+++ b/modules/zookeeper/pom.xml
@@ -67,7 +67,7 @@
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
- <version>${guava16.version}</version>
+ <version>${guava.version}</version>
</dependency>
<!-- Do not remove org.codehaus.jackson:jackson-core-asl it is required by Apache Curator at runtime -->
@@ -110,7 +110,7 @@
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-test</artifactId>
- <version>${curator.version}</version>
+ <version>${curator.test.version}</version>
<scope>test</scope>
</dependency>
diff --git a/parent/pom.xml b/parent/pom.xml
index cda75c3..453ec89 100644
--- a/parent/pom.xml
+++ b/parent/pom.xml
@@ -63,15 +63,15 @@
<commons.lang3.version>3.9</commons.lang3.version>
<commons.io.version>2.6</commons.io.version>
<cron4j.version>2.2.5</cron4j.version>
- <curator.version>2.9.1</curator.version>
+ <curator.version>4.2.0</curator.version>
+ <!--Need this version to test ZooKeeper 3.4.x-->
+ <curator.test.version>2.9.1</curator.test.version>
<easymock.version>3.4</easymock.version>
<ezmorph.bundle.version>1.0.6_1</ezmorph.bundle.version>
<ezmorph.version>1.0.6</ezmorph.version>
<flume.ng.version>1.7.0</flume.ng.version>
<guava.retrying.version>2.0.0</guava.retrying.version>
<guava.version>25.1-jre</guava.version>
- <guava14.version>14.0.1</guava14.version>
- <guava16.version>16.0.1</guava16.version>
<hadoop.version>2.9.1</hadoop.version>
<hamcrest.version>1.2</hamcrest.version>
<httpclient.version>4.5.1</httpclient.version>
@@ -123,22 +123,22 @@
<slf4j16.version>1.6.4</slf4j16.version>
<snappy.version>1.1.7.2</snappy.version>
<spark.hadoop.version>2.6.5</spark.hadoop.version>
- <spark.version>2.3.0</spark.version>
+ <spark.version>2.3.3</spark.version>
<spring.boot.version>1.5.20.RELEASE</spring.boot.version>
<spring.data.version>1.13.14.RELEASE</spring.data.version> <!-- don't forget to update spring version -->
<spring.version>4.3.18.RELEASE</spring.version><!-- don't forget to update spring-data version -->
<spring.data-2.0.version>2.0.9.RELEASE</spring.data-2.0.version> <!-- don't forget to update spring-5.0 version -->
<spring-5.0.version>5.0.8.RELEASE</spring-5.0.version><!-- don't forget to update spring-data-2.0 version -->
<spring41.osgi.feature.version>4.1.7.RELEASE_1</spring41.osgi.feature.version>
- <storm.version>1.1.1</storm.version>
+ <storm.version>1.1.3</storm.version>
<swagger2.version>2.9.2</swagger2.version>
- <tomcat.version>9.0.10</tomcat.version>
+ <tomcat.version>9.0.21</tomcat.version>
<twitter.hbc.version>2.2.0</twitter.hbc.version>
<yammer.metrics.annotation.version>2.2.0</yammer.metrics.annotation.version>
<yammer.metrics.core.version>2.2.0</yammer.metrics.core.version>
<yardstick.version>0.8.3</yardstick.version>
<zkclient.version>0.5</zkclient.version>
- <zookeeper.version>3.4.13</zookeeper.version>
+ <zookeeper.version>3.4.14</zookeeper.version>
<zstd.version>1.3.7-2</zstd.version>
<web-console.ignite.version>8.7.5</web-console.ignite.version>
<opencensus.version>0.22.0</opencensus.version>
[ignite] 18/19: GG-21356 Usability: control utility new printed
argument format.
Posted by am...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
amashenkov pushed a commit to branch gg-21368
in repository https://gitbox.apache.org/repos/asf/ignite.git
commit bad5683ffde0874a709a7d9ae488c9328962f9a2
Author: mstepachev <ma...@gmail.com>
AuthorDate: Thu Jul 18 12:33:40 2019 +0300
GG-21356 Usability: control utility new printed argument format.
---
.../ignite/internal/commandline/Command.java | 7 -----
.../internal/commandline/CommandHandler.java | 34 +++++++++++++++++++---
.../internal/commandline/CommonArgParser.java | 16 ++++++++++
.../ignite/internal/commandline/WalCommands.java | 5 ----
.../internal/commandline/cache/CacheCommands.java | 6 ----
.../commandline/cache/ResetLostPartitions.java | 5 ----
.../commandline/diagnostic/DiagnosticCommand.java | 6 ----
.../apache/ignite/util/GridCommandHandlerTest.java | 2 +-
8 files changed, 47 insertions(+), 34 deletions(-)
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/Command.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/Command.java
index 21c9deb..be47696 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/Command.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/Command.java
@@ -101,13 +101,6 @@ public interface Command<T> {
public void printUsage(Logger logger);
/**
- * @return String which reflect of command-specific arguments in human-friendly format.
- */
- public default String argumentString() {
- return arg() == null? "" : arg().toString();
- }
-
- /**
* @return command name.
*/
String name();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
index 98e3f91..d0fe93e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
@@ -43,6 +43,7 @@ import org.apache.ignite.internal.client.impl.connection.GridClientConnectionRes
import org.apache.ignite.internal.client.ssl.GridSslBasicContextFactory;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.X;
+import org.apache.ignite.internal.util.typedef.internal.SB;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.internal.visor.util.VisorIllegalStateException;
import org.apache.ignite.logger.java.JavaLoggerFileHandler;
@@ -245,10 +246,7 @@ public class CommandHandler {
GridClientConfiguration clientCfg = getClientConfiguration(args);
logger.info("Command [" + commandName + "] started");
- logger.info("Common arguments: " + args.toString());
-
- if (command.arg() != null)
- logger.info("Command arguments: " + command.argumentString());
+ logger.info("Arguments: " + argumentsToString(rawArgs));
logger.info(DELIM);
@@ -347,6 +345,34 @@ public class CommandHandler {
}
/**
+ * @param rawArgs Arguments which user has provided.
+ * @return String which could be shown in console and pritned to log.
+ */
+ private String argumentsToString(List<String> rawArgs) {
+ boolean hide = false;
+
+ SB sb = new SB();
+
+ for (int i = 0; i < rawArgs.size(); i++) {
+ if (hide) {
+ sb.a("***** ");
+
+ hide = false;
+
+ continue;
+ }
+
+ String arg = rawArgs.get(i);
+
+ sb.a(arg).a(' ');
+
+ hide = CommonArgParser.isSensitiveArgument(arg);
+ }
+
+ return sb.toString();
+ }
+
+ /**
* Does one of three things:
* <ul>
* <li>returns user name from connection parameters if it is there;</li>
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommonArgParser.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommonArgParser.java
index 6fec5a0..1cf04da 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommonArgParser.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommonArgParser.java
@@ -92,6 +92,9 @@ public class CommonArgParser {
/** List of optional auxiliary commands. */
private static final Set<String> AUX_COMMANDS = new HashSet<>();
+ /** Set of sensitive arguments */
+ private static final Set<String> SENSITIVE_ARGUMENTS = new HashSet<>();
+
static {
AUX_COMMANDS.add(CMD_HOST);
AUX_COMMANDS.add(CMD_PORT);
@@ -115,9 +118,22 @@ public class CommonArgParser {
AUX_COMMANDS.add(CMD_TRUSTSTORE);
AUX_COMMANDS.add(CMD_TRUSTSTORE_PASSWORD);
AUX_COMMANDS.add(CMD_TRUSTSTORE_TYPE);
+
+ SENSITIVE_ARGUMENTS.add(CMD_PASSWORD);
+ SENSITIVE_ARGUMENTS.add(CMD_KEYSTORE_PASSWORD);
+ SENSITIVE_ARGUMENTS.add(CMD_TRUSTSTORE_PASSWORD);
}
/**
+ * @param arg To check.
+ * @return True if provided argument is among sensitive one and not should be displayed.
+ */
+ public static boolean isSensitiveArgument(String arg) {
+ return SENSITIVE_ARGUMENTS.contains(arg);
+ }
+
+
+ /**
* @param logger Logger.
*/
public CommonArgParser(Logger logger) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/WalCommands.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/WalCommands.java
index 6c03c4d..95b1377 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/WalCommands.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/WalCommands.java
@@ -144,11 +144,6 @@ public class WalCommands implements Command<T2<String, String>> {
return new T2<>(walAct, walArgs);
}
- /** {@inheritDoc} */
- @Override public String argumentString() {
- return "walAction=" + walAct + ", walArgs=" + walArgs;
- }
-
/**
* Execute delete unused WAL segments task.
* @param client Client.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheCommands.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheCommands.java
index 54c9004..bd1bdcf 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheCommands.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheCommands.java
@@ -211,12 +211,6 @@ public class CacheCommands implements Command<CacheSubcommands> {
}
/** {@inheritDoc} */
- @Override public String argumentString() {
- return "cacheSubcommand=" + subcommand.name() +
- (subcommand.subcommand() == null ? "" : ", " + subcommand.subcommand().argumentString());
- }
-
- /** {@inheritDoc} */
@Override public String name() {
return CACHE.toCommandName();
}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/ResetLostPartitions.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/ResetLostPartitions.java
index bf712f7..d6bee13 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/ResetLostPartitions.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/ResetLostPartitions.java
@@ -53,11 +53,6 @@ public class ResetLostPartitions implements Command<Set<String>> {
}
/** {@inheritDoc} */
- @Override public String argumentString() {
- return "caches=" + caches;
- }
-
- /** {@inheritDoc} */
@Override public Object execute(GridClientConfiguration clientCfg, Logger logger) throws Exception {
CacheResetLostPartitionsTaskArg taskArg = new CacheResetLostPartitionsTaskArg(caches);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/diagnostic/DiagnosticCommand.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/diagnostic/DiagnosticCommand.java
index e2379bb..e4b11b0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/diagnostic/DiagnosticCommand.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/diagnostic/DiagnosticCommand.java
@@ -59,12 +59,6 @@ public class DiagnosticCommand implements Command<DiagnosticSubCommand> {
}
/** {@inheritDoc} */
- @Override public String argumentString() {
- return "diagnosticSubcommand=" + subcommand.name() +
- (subcommand.subcommand() == null ? "" : ", " + subcommand.subcommand().argumentString());
- }
-
- /** {@inheritDoc} */
@Override public void parseArguments(CommandArgIterator argIter) {
if (!argIter.hasNextSubArg()) {
subcommand = HELP;
diff --git a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
index 780dfee..55c50a5 100644
--- a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
@@ -1243,7 +1243,7 @@ public class GridCommandHandlerTest extends GridCommandHandlerAbstractTest {
// Ignite instase 1 can be logged only in arguments list.
boolean isInstanse1Found = Arrays.stream(testOutStr.split("\n"))
- .filter(s -> s.contains("Command arguments:"))
+ .filter(s -> s.contains("Arguments:"))
.noneMatch(s -> s.contains(getTestIgniteInstanceName() + "1"));
assertTrue(testOutStr, testOutStr.contains("Node not found for consistent ID:"));
[ignite] 04/19: GG-21310 Deployed on GCP console can't display
connected cluster (#284)
Posted by am...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
amashenkov pushed a commit to branch gg-21368
in repository https://gitbox.apache.org/repos/asf/ignite.git
commit 1b9a13b2e1421fcd7b9e63969e1b2dc94d70ee9f
Author: Andrey Novikov <an...@gridgain.com>
AuthorDate: Wed Jul 17 12:59:09 2019 +0700
GG-21310 Deployed on GCP console can't display connected cluster (#284)
---
.../ignite/console/agent/handlers/ClusterHandler.java | 4 ++--
.../ignite/console/agent/handlers/WebSocketRouter.java | 8 +++-----
.../apache/ignite/console/agent/rest/RestExecutor.java | 16 +++++-----------
.../ignite/console/web/socket/WebSocketsManager.java | 10 +++++++++-
4 files changed, 19 insertions(+), 19 deletions(-)
diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/ClusterHandler.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/ClusterHandler.java
index 0bb54e3..451e064 100644
--- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/ClusterHandler.java
+++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/ClusterHandler.java
@@ -102,10 +102,10 @@ public class ClusterHandler extends AbstractClusterHandler {
return res;
}
- catch (InterruptedException | TimeoutException ignored) {
+ catch (ConnectException | InterruptedException | TimeoutException ignored) {
// No-op.
}
- catch (Exception e) {
+ catch (Throwable e) {
LT.error(log, e, "Failed execute request on node [url=" + nodeUrl + ", parameters=" + params + "]");
}
}
diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/WebSocketRouter.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/WebSocketRouter.java
index 4dde8f8..4039936 100644
--- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/WebSocketRouter.java
+++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/WebSocketRouter.java
@@ -415,13 +415,11 @@ public class WebSocketRouter implements AutoCloseable {
return;
}
-
- String errMsg = ERROR_MSGS.get(evt.getEventType());
-
- log.error(errMsg, e);
+
+ log.error("Failed to send response: " + evt, e);
try {
- send(ses, evt.withError(extractErrorMessage(errMsg, e)));
+ send(ses, evt.withError(extractErrorMessage(ERROR_MSGS.get(evt.getEventType()), e)));
}
catch (Exception ex) {
log.error("Failed to send response with error", e);
diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/rest/RestExecutor.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/rest/RestExecutor.java
index 7e06722..68141bc 100644
--- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/rest/RestExecutor.java
+++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/rest/RestExecutor.java
@@ -19,11 +19,10 @@ package org.apache.ignite.console.agent.rest;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
+import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
import org.apache.ignite.IgniteLogger;
import org.apache.ignite.console.json.JsonObject;
-import org.apache.ignite.internal.util.typedef.X;
import org.apache.ignite.logger.slf4j.Slf4jLogger;
import org.eclipse.jetty.client.HttpClient;
import org.eclipse.jetty.client.api.Response;
@@ -97,9 +96,9 @@ public class RestExecutor implements AutoCloseable {
* @param params Request parameters.
* @return Request result.
* @throws IOException If failed to parse REST result.
- * @throws Exception If failed to send request.
+ * @throws Throwable If failed to send request.
*/
- public RestResult sendRequest(String url, JsonObject params) throws Exception {
+ public RestResult sendRequest(String url, JsonObject params) throws Throwable {
if (!httpClient.isRunning())
httpClient.start();
@@ -120,13 +119,8 @@ public class RestExecutor implements AutoCloseable {
return parseResponse(res, lsnr.getInputStream());
}
- catch (Exception e) {
- TimeoutException e0 = X.cause(e, TimeoutException.class);
-
- if (e0 != null)
- throw e0;
-
- throw e;
+ catch (ExecutionException e) {
+ throw e.getCause();
}
}
}
diff --git a/modules/web-console/web-console-server/src/main/java/org/apache/ignite/console/web/socket/WebSocketsManager.java b/modules/web-console/web-console-server/src/main/java/org/apache/ignite/console/web/socket/WebSocketsManager.java
index 6c2992e..bf850cd 100644
--- a/modules/web-console/web-console-server/src/main/java/org/apache/ignite/console/web/socket/WebSocketsManager.java
+++ b/modules/web-console/web-console-server/src/main/java/org/apache/ignite/console/web/socket/WebSocketsManager.java
@@ -139,6 +139,8 @@ public class WebSocketsManager {
*/
public void onBrowserConnectionClosed(WebSocketSession ws) {
browsers.remove(ws);
+
+ requests.values().remove(ws);
}
/**
@@ -148,7 +150,7 @@ public class WebSocketsManager {
WebSocketSession ws = requests.remove(evt.getRequestId());
if (ws == null) {
- log.warn("Failed to send event to browser: " + evt);
+ log.warn("Failed to send response to browser, connection was already closed: " + evt);
return;
}
@@ -426,6 +428,12 @@ public class WebSocketsManager {
*/
@SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter")
protected void sendMessage(WebSocketSession ws, WebSocketEvent evt) throws IOException {
+ if (!ws.isOpen()) {
+ log.warn("Failed to send event because websocket is already closed [session=" + ws + ", evt=" + evt + "]");
+
+ return;
+ }
+
synchronized (ws) {
ws.sendMessage(new TextMessage(toJson(evt)));
}
[ignite] 05/19: GG-20691 New source of query metrics. (#245)
Posted by am...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
amashenkov pushed a commit to branch gg-21368
in repository https://gitbox.apache.org/repos/asf/ignite.git
commit a5f16e7cfd3c8caf91611663194fc20a131cbb76
Author: Vasiliy Sisko <vs...@gridgain.com>
AuthorDate: Wed Jul 17 09:45:29 2019 +0300
GG-20691 New source of query metrics. (#245)
---
.../org/apache/ignite/internal/IgniteFeatures.java | 14 +-
.../org/apache/ignite/internal/IgniteKernal.java | 2 +-
.../VisorQueryDetailMetricsCollectorTask.java | 3 +-
.../query/VisorQueryResetDetailMetricsTask.java | 3 +-
.../product/IndexingFeatureIsNotAvailableTest.java | 50 +++++++
.../ignite/testsuites/IgniteBasicTestSuite.java | 3 +
.../VisorQueryHistoryMetricsCollectorTask.java} | 73 ++++++++--
.../query/VisorQueryHistoryMetricsResetTask.java} | 9 +-
.../query/SqlMetricsOnWebConsoleSelfTest.java | 151 +++++++++++++++++++++
.../product/IndexingFeatureIsAvailableTest.java | 50 +++++++
.../IgniteBinaryCacheQueryTestSuite.java | 5 +
11 files changed, 346 insertions(+), 17 deletions(-)
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteFeatures.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteFeatures.java
index c0df3a0..a5c339f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteFeatures.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteFeatures.java
@@ -64,11 +64,14 @@ public enum IgniteFeatures {
/** Supports tracking update counter for transactions. */
TX_TRACKING_UPDATE_COUNTER(12),
- /** Support new security processor */
+ /** Support new security processor. */
IGNITE_SECURITY_PROCESSOR(13),
/** Replacing TcpDiscoveryNode field with nodeId field in discovery messages. */
- TCP_DISCOVERY_MESSAGE_NODE_COMPACT_REPRESENTATION(14);
+ TCP_DISCOVERY_MESSAGE_NODE_COMPACT_REPRESENTATION(14),
+
+ /** Indexing enabled. */
+ INDEXING(15);
/**
* Unique feature identifier.
@@ -159,9 +162,10 @@ public enum IgniteFeatures {
/**
* Features supported by the current node.
*
+ * @param ctx Kernal context.
* @return Byte array representing all supported features by current node.
*/
- public static byte[] allFeatures() {
+ public static byte[] allFeatures(GridKernalContext ctx) {
final BitSet set = new BitSet();
for (IgniteFeatures value : IgniteFeatures.values()) {
@@ -169,6 +173,10 @@ public enum IgniteFeatures {
if (IGNITE_SECURITY_PROCESSOR == value && !getBoolean(IGNITE_SECURITY_PROCESSOR.name(), true))
continue;
+ // Add only when indexing is enabled.
+ if (INDEXING == value && !ctx.query().moduleEnabled())
+ continue;
+
final int featureId = value.getFeatureId();
assert !set.get(featureId) : "Duplicate feature ID found for [" + value + "] having same ID ["
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index f283d86..48e996a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -1755,7 +1755,7 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
add(ATTR_TX_CONFIG, cfg.getTransactionConfiguration());
// Supported features.
- add(ATTR_IGNITE_FEATURES, IgniteFeatures.allFeatures());
+ add(ATTR_IGNITE_FEATURES, IgniteFeatures.allFeatures(context()));
// Stick in SPI versions and classes attributes.
addSpiAttributes(cfg.getCollisionSpi());
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryDetailMetricsCollectorTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryDetailMetricsCollectorTask.java
index ed425db..5548b26 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryDetailMetricsCollectorTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryDetailMetricsCollectorTask.java
@@ -37,7 +37,8 @@ import org.jetbrains.annotations.Nullable;
import static org.apache.ignite.internal.processors.cache.GridCacheUtils.isSystemCache;
/**
- * Task to collect cache query metrics.
+ * Collect query detailed metrics from cache context only.
+ * This task should be used to collect query metrics when indexing feature not available.
*/
@GridInternal
public class VisorQueryDetailMetricsCollectorTask extends VisorMultiNodeTask<VisorQueryDetailMetricsCollectorTaskArg,
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryResetDetailMetricsTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryResetDetailMetricsTask.java
index 03a21ae..c8cf29e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryResetDetailMetricsTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryResetDetailMetricsTask.java
@@ -23,7 +23,8 @@ import org.apache.ignite.internal.visor.VisorJob;
import org.apache.ignite.internal.visor.VisorOneNodeTask;
/**
- * Reset query detail metrics.
+ * Reset query detailed metrics from cache context only
+ * This task should be used to reset query metrics when indexing feature not available.
*/
@GridInternal
public class VisorQueryResetDetailMetricsTask extends VisorOneNodeTask<Void, Void> {
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/product/IndexingFeatureIsNotAvailableTest.java b/modules/core/src/test/java/org/apache/ignite/internal/product/IndexingFeatureIsNotAvailableTest.java
new file mode 100644
index 0000000..ef4ad75
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/product/IndexingFeatureIsNotAvailableTest.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright 2019 GridGain Systems, Inc. and Contributors.
+ *
+ * Licensed under the GridGain Community Edition License (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.gridgain.com/products/software/community-edition/gridgain-community-edition-license
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.product;
+
+import org.apache.ignite.internal.IgniteFeatures;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.junit.Test;
+
+/**
+ * The test checks that indexing feature is not available in ignite core module.
+ */
+public class IndexingFeatureIsNotAvailableTest extends GridCommonAbstractTest {
+ /** Value of expected INDEXING feature availability check. */
+ private static final boolean INDEXING_FEATURE_IS_NOT_AVAILABLE = false;
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ startGrid();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTestsStopped() throws Exception {
+ stopGrid();
+ }
+
+ /**
+ * @throws Exception If fail.
+ */
+ @Test
+ public void testIndexingFeatureIsNotAvailable() throws Exception {
+ assertEquals(
+ INDEXING_FEATURE_IS_NOT_AVAILABLE,
+ IgniteFeatures.nodeSupports(IgniteFeatures.allFeatures(grid().context()), IgniteFeatures.INDEXING)
+ );
+ }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java
index 0f71ba0..aa2ea2f 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java
@@ -86,6 +86,7 @@ import org.apache.ignite.internal.processors.odbc.OdbcConfigurationValidationSel
import org.apache.ignite.internal.processors.odbc.OdbcEscapeSequenceSelfTest;
import org.apache.ignite.internal.processors.odbc.SqlListenerUtilsTest;
import org.apache.ignite.internal.product.GridProductVersionSelfTest;
+import org.apache.ignite.internal.product.IndexingFeatureIsNotAvailableTest;
import org.apache.ignite.internal.util.BitSetIntSetTest;
import org.apache.ignite.internal.util.GridCleanerTest;
import org.apache.ignite.internal.util.nio.IgniteExceptionInNioWorkerSelfTest;
@@ -171,6 +172,8 @@ import org.junit.runners.Suite;
SetTxTimeoutOnPartitionMapExchangeTest.class,
DiscoveryDataDeserializationFailureHanderTest.class,
+ IndexingFeatureIsNotAvailableTest.class,
+
IgniteExceptionInNioWorkerSelfTest.class,
IgniteLocalNodeMapBeforeStartTest.class,
OdbcConfigurationValidationSelfTest.class,
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryDetailMetricsCollectorTask.java b/modules/indexing/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryHistoryMetricsCollectorTask.java
similarity index 57%
copy from modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryDetailMetricsCollectorTask.java
copy to modules/indexing/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryHistoryMetricsCollectorTask.java
index ed425db..631f8cf 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryDetailMetricsCollectorTask.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryHistoryMetricsCollectorTask.java
@@ -28,6 +28,9 @@ import org.apache.ignite.internal.processors.cache.GridCacheProcessor;
import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
import org.apache.ignite.internal.processors.cache.query.GridCacheQueryDetailMetricsAdapter;
import org.apache.ignite.internal.processors.cache.query.GridCacheQueryDetailMetricsKey;
+import org.apache.ignite.internal.processors.query.GridQueryIndexing;
+import org.apache.ignite.internal.processors.query.QueryHistoryMetrics;
+import org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing;
import org.apache.ignite.internal.processors.task.GridInternal;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.internal.visor.VisorJob;
@@ -35,19 +38,21 @@ import org.apache.ignite.internal.visor.VisorMultiNodeTask;
import org.jetbrains.annotations.Nullable;
import static org.apache.ignite.internal.processors.cache.GridCacheUtils.isSystemCache;
+import static org.apache.ignite.internal.processors.cache.query.GridCacheQueryType.SQL;
+import static org.apache.ignite.internal.processors.cache.query.GridCacheQueryType.SQL_FIELDS;
/**
* Task to collect cache query metrics.
*/
@GridInternal
-public class VisorQueryDetailMetricsCollectorTask extends VisorMultiNodeTask<VisorQueryDetailMetricsCollectorTaskArg,
+public class VisorQueryHistoryMetricsCollectorTask extends VisorMultiNodeTask<VisorQueryDetailMetricsCollectorTaskArg,
Collection<VisorQueryDetailMetrics>, Collection<? extends QueryDetailMetrics>> {
/** */
private static final long serialVersionUID = 0L;
/** {@inheritDoc} */
- @Override protected VisorCacheQueryDetailMetricsCollectorJob job(VisorQueryDetailMetricsCollectorTaskArg arg) {
- return new VisorCacheQueryDetailMetricsCollectorJob(arg, debug);
+ @Override protected VisorQueryHistoryMetricsCollectorJob job(VisorQueryDetailMetricsCollectorTaskArg arg) {
+ return new VisorQueryHistoryMetricsCollectorJob(arg, debug);
}
/** {@inheritDoc} */
@@ -61,7 +66,7 @@ public class VisorQueryDetailMetricsCollectorTask extends VisorMultiNodeTask<Vis
Collection<GridCacheQueryDetailMetricsAdapter> metrics = res.getData();
- VisorCacheQueryDetailMetricsCollectorJob.aggregateMetrics(-1, taskRes, metrics);
+ VisorQueryHistoryMetricsCollectorJob.aggregateMetrics(-1, taskRes, metrics, false);
}
Collection<GridCacheQueryDetailMetricsAdapter> aggMetrics = taskRes.values();
@@ -77,7 +82,7 @@ public class VisorQueryDetailMetricsCollectorTask extends VisorMultiNodeTask<Vis
/**
* Job that will actually collect query metrics.
*/
- private static class VisorCacheQueryDetailMetricsCollectorJob
+ private static class VisorQueryHistoryMetricsCollectorJob
extends VisorJob<VisorQueryDetailMetricsCollectorTaskArg, Collection<? extends QueryDetailMetrics>> {
/** */
private static final long serialVersionUID = 0L;
@@ -88,7 +93,7 @@ public class VisorQueryDetailMetricsCollectorTask extends VisorMultiNodeTask<Vis
* @param arg Last time when metrics were collected.
* @param debug Debug flag.
*/
- protected VisorCacheQueryDetailMetricsCollectorJob(@Nullable VisorQueryDetailMetricsCollectorTaskArg arg, boolean debug) {
+ protected VisorQueryHistoryMetricsCollectorJob(@Nullable VisorQueryDetailMetricsCollectorTaskArg arg, boolean debug) {
super(arg, debug);
}
@@ -96,13 +101,21 @@ public class VisorQueryDetailMetricsCollectorTask extends VisorMultiNodeTask<Vis
* @param since Time when metrics were collected last time.
* @param res Response.
* @param metrics Metrics.
+ * @param collectNotSqlMetrics When {@code true} collect metrics for not SQL queries only.
*/
- private static void aggregateMetrics(long since, Map<GridCacheQueryDetailMetricsKey,
- GridCacheQueryDetailMetricsAdapter> res, Collection<GridCacheQueryDetailMetricsAdapter> metrics) {
+ private static void aggregateMetrics(
+ long since, Map<GridCacheQueryDetailMetricsKey,
+ GridCacheQueryDetailMetricsAdapter> res,
+ Collection<GridCacheQueryDetailMetricsAdapter> metrics,
+ boolean collectNotSqlMetrics
+ ) {
for (GridCacheQueryDetailMetricsAdapter m : metrics) {
if (m.lastStartTime() > since) {
GridCacheQueryDetailMetricsKey key = m.key();
+ if (collectNotSqlMetrics && (key.getQueryType() == SQL || key.getQueryType() == SQL_FIELDS))
+ continue;
+
GridCacheQueryDetailMetricsAdapter aggMetrics = res.get(key);
res.put(key, aggMetrics == null ? m : aggMetrics.aggregate(m));
@@ -129,7 +142,47 @@ public class VisorQueryDetailMetricsCollectorTask extends VisorMultiNodeTask<Vis
if (cache == null || !cache.context().started())
continue;
- aggregateMetrics(arg.getSince(), aggMetrics, cache.context().queries().detailMetrics());
+ aggregateMetrics(arg.getSince(), aggMetrics, cache.context().queries().detailMetrics(), true);
+ }
+ }
+
+ GridQueryIndexing indexing = ignite.context().query().getIndexing();
+
+ if (indexing instanceof IgniteH2Indexing) {
+ Collection<QueryHistoryMetrics> metrics = ((IgniteH2Indexing)indexing)
+ .runningQueryManager().queryHistoryMetrics().values();
+
+ for (QueryHistoryMetrics m : metrics) {
+ GridCacheQueryDetailMetricsKey key = new GridCacheQueryDetailMetricsKey(SQL_FIELDS, m.query());
+
+ GridCacheQueryDetailMetricsAdapter oldMetrics = aggMetrics.get(key);
+
+ GridCacheQueryDetailMetricsAdapter total = oldMetrics != null
+ ? new GridCacheQueryDetailMetricsAdapter(
+ SQL_FIELDS, m.query(),
+ null,
+ oldMetrics.executions() + (int)m.executions(),
+ oldMetrics.completions() + (int)(m.executions() - m.failures()),
+ oldMetrics.failures() + (int)m.failures(),
+ Long.min(oldMetrics.minimumTime(), m.minimumTime()),
+ Long.max(oldMetrics.maximumTime(), m.maximumTime()),
+ oldMetrics.totalTime(),
+ Long.max(oldMetrics.lastStartTime(), m.lastStartTime()),
+ key)
+ : new GridCacheQueryDetailMetricsAdapter(
+ SQL_FIELDS,
+ m.query(),
+ null,
+ (int)m.executions(),
+ (int)(m.executions() - m.failures()),
+ (int)m.failures(),
+ m.minimumTime(),
+ m.maximumTime(),
+ 0L,
+ m.lastStartTime(),
+ key);
+
+ aggMetrics.put(key, total);
}
}
@@ -138,7 +191,7 @@ public class VisorQueryDetailMetricsCollectorTask extends VisorMultiNodeTask<Vis
/** {@inheritDoc} */
@Override public String toString() {
- return S.toString(VisorCacheQueryDetailMetricsCollectorJob.class, this);
+ return S.toString(VisorQueryHistoryMetricsCollectorJob.class, this);
}
}
}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryResetDetailMetricsTask.java b/modules/indexing/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryHistoryMetricsResetTask.java
similarity index 83%
copy from modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryResetDetailMetricsTask.java
copy to modules/indexing/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryHistoryMetricsResetTask.java
index 03a21ae..180ea1a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryResetDetailMetricsTask.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryHistoryMetricsResetTask.java
@@ -17,6 +17,8 @@
package org.apache.ignite.internal.visor.query;
import org.apache.ignite.IgniteCache;
+import org.apache.ignite.internal.processors.query.GridQueryIndexing;
+import org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing;
import org.apache.ignite.internal.processors.task.GridInternal;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.internal.visor.VisorJob;
@@ -26,7 +28,7 @@ import org.apache.ignite.internal.visor.VisorOneNodeTask;
* Reset query detail metrics.
*/
@GridInternal
-public class VisorQueryResetDetailMetricsTask extends VisorOneNodeTask<Void, Void> {
+public class VisorQueryHistoryMetricsResetTask extends VisorOneNodeTask<Void, Void> {
/** */
private static final long serialVersionUID = 0L;
@@ -52,6 +54,11 @@ public class VisorQueryResetDetailMetricsTask extends VisorOneNodeTask<Void, Voi
/** {@inheritDoc} */
@Override protected Void run(Void arg) {
+ GridQueryIndexing indexing = ignite.context().query().getIndexing();
+
+ if (indexing instanceof IgniteH2Indexing)
+ ((IgniteH2Indexing)indexing).runningQueryManager().resetQueryHistoryMetrics();
+
for (String cacheName : ignite.cacheNames()) {
IgniteCache cache = ignite.cache(cacheName);
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlMetricsOnWebConsoleSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlMetricsOnWebConsoleSelfTest.java
new file mode 100644
index 0000000..a096a5f
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlMetricsOnWebConsoleSelfTest.java
@@ -0,0 +1,151 @@
+/*
+ * Copyright 2019 GridGain Systems, Inc. and Contributors.
+ *
+ * Licensed under the GridGain Community Edition License (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.gridgain.com/products/software/community-edition/gridgain-community-edition-license
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.query;
+
+import java.util.Collection;
+import java.util.List;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.query.FieldsQueryCursor;
+import org.apache.ignite.cache.query.QueryCursor;
+import org.apache.ignite.cache.query.ScanQuery;
+import org.apache.ignite.cache.query.SqlFieldsQuery;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.cache.index.AbstractIndexingCommonTest;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.visor.VisorTaskArgument;
+import org.apache.ignite.internal.visor.query.VisorQueryDetailMetrics;
+import org.apache.ignite.internal.visor.query.VisorQueryDetailMetricsCollectorTaskArg;
+import org.apache.ignite.internal.visor.query.VisorQueryHistoryMetricsCollectorTask;
+import org.apache.ignite.internal.visor.query.VisorQueryHistoryMetricsResetTask;
+import org.junit.Test;
+
+/**
+ * Tests for ignite SQL system views.
+ */
+public class SqlMetricsOnWebConsoleSelfTest extends AbstractIndexingCommonTest {
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ super.beforeTest();
+
+ cleanPersistenceDir();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ stopAllGrids();
+
+ cleanPersistenceDir();
+ }
+
+ /**
+ * @return System schema name.
+ */
+ protected String systemSchemaName() {
+ return "SYS";
+ }
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration() throws Exception {
+ return super.getConfiguration().setSqlQueryHistorySize(10);
+ }
+
+ /**
+ * Test Query history system view.
+ */
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testQueryHistoryMetricsTasks() throws Exception {
+ IgniteEx ignite = startGrid(0);
+
+ final String SCHEMA_NAME = "TEST_SCHEMA";
+
+ IgniteCache cache = ignite.createCache(
+ new CacheConfiguration<>(DEFAULT_CACHE_NAME)
+ .setIndexedTypes(Integer.class, String.class)
+ .setQueryDetailMetricsSize(10)
+ .setSqlSchema(SCHEMA_NAME)
+ );
+
+ cache.put(100, "200");
+
+ String sqlHist = "SELECT SCHEMA_NAME, SQL, LOCAL, EXECUTIONS, FAILURES, DURATION_MIN, DURATION_MAX, LAST_START_TIME " +
+ "FROM " + systemSchemaName() + ".LOCAL_SQL_QUERY_HISTORY ORDER BY LAST_START_TIME";
+
+ // Execute query without cache context.
+ cache.query(new SqlFieldsQuery(sqlHist).setLocal(true)).getAll();
+
+ // Execute query with cache context.
+ List<FieldsQueryCursor<List<?>>> cursors = ignite.context().query().querySqlFields(
+ ignite.context().cache().cache(DEFAULT_CACHE_NAME).context(),
+ new SqlFieldsQuery(sqlHist),
+ null,
+ true,
+ false,
+ null
+ );
+
+ cursors.forEach(U::closeQuiet);
+
+ // Execute scan query.
+ ScanQuery<Object, Object> scanQry = new ScanQuery<>();
+
+ QueryCursor scanCur = cache.query(scanQry);
+
+ scanCur.getAll();
+ U.closeQuiet(scanCur);
+
+ // Collect query metrics.
+ Collection<VisorQueryDetailMetrics> res = ignite.compute().execute(
+ VisorQueryHistoryMetricsCollectorTask.class,
+ new VisorTaskArgument<>(
+ ignite.localNode().id(),
+ new VisorQueryDetailMetricsCollectorTaskArg(-1),
+ false
+ )
+ );
+
+ // First and second queries have the same query string and are grouped in single row.
+ assertEquals(2, res.size());
+
+ for (VisorQueryDetailMetrics row: res) {
+ if ("SQL_FIELDS".equals(row.getQueryType()))
+ assertEquals(2, row.getExecutions());
+ else
+ assertEquals(1, row.getExecutions());
+ }
+
+ // Clear query metrics.
+ ignite.compute().execute(
+ VisorQueryHistoryMetricsResetTask.class,
+ new VisorTaskArgument<>(ignite.localNode().id(), null, false)
+ );
+
+ // Collect query metrics after clean.
+ res = ignite.compute().execute(
+ VisorQueryHistoryMetricsCollectorTask.class,
+ new VisorTaskArgument<>(
+ ignite.localNode().id(),
+ new VisorQueryDetailMetricsCollectorTaskArg(-1),
+ false
+ )
+ );
+
+ assertEquals(0, res.size());
+ }
+}
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/product/IndexingFeatureIsAvailableTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/product/IndexingFeatureIsAvailableTest.java
new file mode 100644
index 0000000..895d106
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/product/IndexingFeatureIsAvailableTest.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright 2019 GridGain Systems, Inc. and Contributors.
+ *
+ * Licensed under the GridGain Community Edition License (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.gridgain.com/products/software/community-edition/gridgain-community-edition-license
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.product;
+
+import org.apache.ignite.internal.IgniteFeatures;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.junit.Test;
+
+/**
+ * The test checks that indexing feature is available in ignite indexing module.
+ */
+public class IndexingFeatureIsAvailableTest extends GridCommonAbstractTest {
+ /** Value of expected INDEXING feature availability check. */
+ private static final boolean INDEXING_FEATURE_IS_AVAILABLE = true;
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ startGrid();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTestsStopped() throws Exception {
+ stopGrid();
+ }
+
+ /**
+ * @throws Exception If fail.
+ */
+ @Test
+ public void testIndexingFeatureIsAvailable() throws Exception {
+ assertEquals(
+ INDEXING_FEATURE_IS_AVAILABLE,
+ IgniteFeatures.nodeSupports(IgniteFeatures.allFeatures(grid().context()), IgniteFeatures.INDEXING)
+ );
+ }
+}
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteBinaryCacheQueryTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteBinaryCacheQueryTestSuite.java
index ff8d454..032ff0b 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteBinaryCacheQueryTestSuite.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteBinaryCacheQueryTestSuite.java
@@ -204,6 +204,7 @@ import org.apache.ignite.internal.processors.query.MultipleStatementsSqlQuerySel
import org.apache.ignite.internal.processors.query.RunningQueriesTest;
import org.apache.ignite.internal.processors.query.SqlIllegalSchemaSelfTest;
import org.apache.ignite.internal.processors.query.SqlIncompatibleDataTypeExceptionTest;
+import org.apache.ignite.internal.processors.query.SqlMetricsOnWebConsoleSelfTest;
import org.apache.ignite.internal.processors.query.SqlNestedQuerySelfTest;
import org.apache.ignite.internal.processors.query.SqlPushDownFunctionTest;
import org.apache.ignite.internal.processors.query.SqlQueryHistoryFromClientSelfTest;
@@ -242,6 +243,7 @@ import org.apache.ignite.internal.processors.sql.IgniteCacheReplicatedTransactio
import org.apache.ignite.internal.processors.sql.IgniteSQLColumnConstraintsTest;
import org.apache.ignite.internal.processors.sql.IgniteTransactionSQLColumnConstraintTest;
import org.apache.ignite.internal.processors.sql.SqlConnectorConfigurationValidationSelfTest;
+import org.apache.ignite.internal.product.IndexingFeatureIsAvailableTest;
import org.apache.ignite.internal.sql.SqlParserBulkLoadSelfTest;
import org.apache.ignite.internal.sql.SqlParserCreateIndexSelfTest;
import org.apache.ignite.internal.sql.SqlParserDropIndexSelfTest;
@@ -282,6 +284,8 @@ import org.junit.runners.Suite;
SqlConnectorConfigurationValidationSelfTest.class,
ClientConnectorConfigurationValidationSelfTest.class,
+ IndexingFeatureIsAvailableTest.class,
+
SqlSchemaSelfTest.class,
SqlIllegalSchemaSelfTest.class,
MultipleStatementsSqlQuerySelfTest.class,
@@ -501,6 +505,7 @@ import org.junit.runners.Suite;
IgniteCheckClusterStateBeforeExecuteQueryTest.class,
OptimizedMarshallerIndexNameTest.class,
SqlSystemViewsSelfTest.class,
+ SqlMetricsOnWebConsoleSelfTest.class,
GridIndexRebuildSelfTest.class,
GridIndexFullRebuildTest.class,
[ignite] 08/19: GG-21455: ML Example TutorialStepByStepExample 0
node, linux fail
Posted by am...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
amashenkov pushed a commit to branch gg-21368
in repository https://gitbox.apache.org/repos/asf/ignite.git
commit 04f305d543a6ec0525993570eee988fce9c65dce
Author: Alexey Platonov <40...@users.noreply.github.com>
AuthorDate: Wed Jul 17 12:05:22 2019 +0300
GG-21455: ML Example TutorialStepByStepExample 0 node, linux fail
---
.../apache/ignite/examples/ml/tutorial/TutorialStepByStepExample.java | 1 +
1 file changed, 1 insertion(+)
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/TutorialStepByStepExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/TutorialStepByStepExample.java
index 2dc9b37..b8886b8 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/TutorialStepByStepExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/TutorialStepByStepExample.java
@@ -33,5 +33,6 @@ public class TutorialStepByStepExample {
Step_7_Split_train_test.main(args);
Step_8_CV.main(args);
Step_8_CV_with_Param_Grid.main(args);
+ Step_9_Scaling_With_Stacking.main(args);
}
}
[ignite] 16/19: GG-21561 Remove slf4j-simple binding (Reverts
GG-21394)
Posted by am...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
amashenkov pushed a commit to branch gg-21368
in repository https://gitbox.apache.org/repos/asf/ignite.git
commit c5a113f93fe26625676738eb2fb61230758ab3f3
Author: Alexey Goncharuk <al...@gmail.com>
AuthorDate: Thu Jul 18 11:37:03 2019 +0300
GG-21561 Remove slf4j-simple binding (Reverts GG-21394)
---
modules/slf4j/pom.xml | 6 ------
1 file changed, 6 deletions(-)
diff --git a/modules/slf4j/pom.xml b/modules/slf4j/pom.xml
index 84a470c..4f148fc 100644
--- a/modules/slf4j/pom.xml
+++ b/modules/slf4j/pom.xml
@@ -47,12 +47,6 @@
</dependency>
<dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-simple</artifactId>
- <version>${slf4j.version}</version>
- </dependency>
-
- <dependency>
<groupId>org.gridgain</groupId>
<artifactId>ignite-core</artifactId>
<version>${project.version}</version>
[ignite] 15/19: GG-21232 control.sh if experimental command
disabled - don't show help for experemental commands.
Posted by am...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
amashenkov pushed a commit to branch gg-21368
in repository https://gitbox.apache.org/repos/asf/ignite.git
commit 86ee0b508e206cfcc325b4898c2732d511dfceff
Author: ktkalenko <kt...@gridgain.com>
AuthorDate: Thu Jul 18 08:50:49 2019 +0300
GG-21232 control.sh if experimental command disabled - don't show help for experemental commands.
---
bin/control.bat | 11 +++--
.../ignite/internal/commandline/WalCommands.java | 48 ++++++++++++---------
.../commandline/CommandHandlerParsingTest.java | 21 ---------
.../apache/ignite/util/GridCommandHandlerTest.java | 50 ++++++++++++++++++++--
4 files changed, 82 insertions(+), 48 deletions(-)
diff --git a/bin/control.bat b/bin/control.bat
index ac34475..0015323 100644
--- a/bin/control.bat
+++ b/bin/control.bat
@@ -1,12 +1,12 @@
::
:: Copyright 2019 GridGain Systems, Inc. and Contributors.
-::
+::
:: Licensed under the GridGain Community Edition License (the "License");
:: you may not use this file except in compliance with the License.
:: You may obtain a copy of the License at
-::
+::
:: https://www.gridgain.com/products/software/community-edition/gridgain-community-edition-license
-::
+::
:: Unless required by applicable law or agreed to in writing, software
:: distributed under the License is distributed on an "AS IS" BASIS,
:: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -168,6 +168,11 @@ if %ERRORLEVEL% equ 0 (
)
::
+:: Uncomment to enable experimental commands [--wal]
+::
+:: set JVM_OPTS=%JVM_OPTS% -DIGNITE_ENABLE_EXPERIMENTAL_COMMAND=true
+
+::
:: Uncomment the following GC settings if you see spikes in your throughput due to Garbage Collection.
::
:: set JVM_OPTS=%JVM_OPTS% -XX:+UseG1GC
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/WalCommands.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/WalCommands.java
index 99c804a..6c03c4d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/WalCommands.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/WalCommands.java
@@ -35,6 +35,7 @@ import org.apache.ignite.internal.visor.misc.VisorWalTaskResult;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_ENABLE_EXPERIMENTAL_COMMAND;
import static org.apache.ignite.internal.commandline.CommandArgIterator.isCommandOrOption;
+import static org.apache.ignite.internal.commandline.CommandHandler.UTILITY_NAME;
import static org.apache.ignite.internal.commandline.CommandList.WAL;
import static org.apache.ignite.internal.commandline.CommandLogger.DOUBLE_INDENT;
import static org.apache.ignite.internal.commandline.CommandLogger.INDENT;
@@ -67,12 +68,13 @@ public class WalCommands implements Command<T2<String, String>> {
/** {@inheritDoc} */
@Override public void printUsage(Logger logger) {
- if (enableExperimental()) {
- Command.usage(logger, "Print absolute paths of unused archived wal segments on each node:", WAL,
- WAL_PRINT, "[consistentId1,consistentId2,....,consistentIdN]");
- Command.usage(logger, "Delete unused archived wal segments on each node:", WAL, WAL_DELETE,
- "[consistentId1,consistentId2,....,consistentIdN]", optional(CMD_AUTO_CONFIRMATION));
- }
+ if (!enableExperimental())
+ return;
+
+ Command.usage(logger, "Print absolute paths of unused archived wal segments on each node:", WAL,
+ WAL_PRINT, "[consistentId1,consistentId2,....,consistentIdN]");
+ Command.usage(logger, "Delete unused archived wal segments on each node:", WAL, WAL_DELETE,
+ "[consistentId1,consistentId2,....,consistentIdN]", optional(CMD_AUTO_CONFIRMATION));
}
/**
@@ -82,21 +84,26 @@ public class WalCommands implements Command<T2<String, String>> {
* @throws Exception If failed to execute wal action.
*/
@Override public Object execute(GridClientConfiguration clientCfg, Logger logger) throws Exception {
- this.logger = logger;
+ if (enableExperimental()) {
+ this.logger = logger;
- try (GridClient client = Command.startClient(clientCfg)) {
- switch (walAct) {
- case WAL_DELETE:
- deleteUnusedWalSegments(client, walArgs, clientCfg);
+ try (GridClient client = Command.startClient(clientCfg)) {
+ switch (walAct) {
+ case WAL_DELETE:
+ deleteUnusedWalSegments(client, walArgs, clientCfg);
- break;
+ break;
- case WAL_PRINT:
- default:
- printUnusedWalSegments(client, walArgs, clientCfg);
+ case WAL_PRINT:
+ default:
+ printUnusedWalSegments(client, walArgs, clientCfg);
- break;
+ break;
+ }
}
+ } else {
+ logger.warning(String.format("For use experimental command add %s=true to JVM_OPTS in %s",
+ IGNITE_ENABLE_EXPERIMENTAL_COMMAND, UTILITY_NAME));
}
return null;
@@ -112,9 +119,6 @@ public class WalCommands implements Command<T2<String, String>> {
/** {@inheritDoc} */
@Override public void parseArguments(CommandArgIterator argIter) {
- if (!enableExperimental())
- throw new IllegalArgumentException("Experimental command is disabled.");
-
String str = argIter.nextArg("Expected arguments for " + WAL.text());
String walAct = str.toLowerCase();
@@ -124,8 +128,10 @@ public class WalCommands implements Command<T2<String, String>> {
? argIter.nextArg("Unexpected argument for " + WAL.text() + ": " + walAct)
: "";
- this.walAct = walAct;
- this.walArgs = walArgs;
+ if (enableExperimental()) {
+ this.walAct = walAct;
+ this.walArgs = walArgs;
+ }
}
else
throw new IllegalArgumentException("Unexpected action " + walAct + " for " + WAL.text());
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java
index 4ee391e..5b713c9 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java
@@ -275,27 +275,6 @@ public class CommandHandlerParsingTest {
}
/**
- * Test that experimental command (i.e. WAL command) is disabled by default.
- */
- @Test
- @WithSystemProperty(key = IGNITE_ENABLE_EXPERIMENTAL_COMMAND, value = "false")
- public void testExperimentalCommandIsDisabled() {
- GridTestUtils.assertThrows(
- null,
- () -> parseArgs(Arrays.asList(WAL.text(), WAL_PRINT)),
- IllegalArgumentException.class,
- null
- );
-
- GridTestUtils.assertThrows(
- null,
- () -> parseArgs(Arrays.asList(WAL.text(), WAL_DELETE)),
- IllegalArgumentException.class,
- null
- );
- }
-
- /**
* Tests parsing and validation for the SSL arguments.
*/
@Test
diff --git a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
index 152c654..780dfee 100644
--- a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
@@ -43,6 +43,7 @@ import java.util.concurrent.atomic.LongAdder;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
+import java.util.stream.Stream;
import javax.cache.processor.EntryProcessor;
import javax.cache.processor.EntryProcessorException;
import javax.cache.processor.MutableEntry;
@@ -103,21 +104,28 @@ import org.apache.ignite.lang.IgnitePredicate;
import org.apache.ignite.lang.IgniteUuid;
import org.apache.ignite.plugin.extensions.communication.Message;
import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.SystemPropertiesRule;
+import org.apache.ignite.testframework.junits.WithSystemProperty;
import org.apache.ignite.transactions.Transaction;
import org.apache.ignite.transactions.TransactionRollbackException;
import org.apache.ignite.transactions.TransactionState;
import org.apache.ignite.transactions.TransactionTimeoutException;
import org.jetbrains.annotations.NotNull;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.TestRule;
import static java.io.File.separatorChar;
import static java.util.Arrays.asList;
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_ENABLE_EXPERIMENTAL_COMMAND;
import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_ILLEGAL_SATE_ERROR;
import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_INVALID_ARGUMENTS;
import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_OK;
import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_UNEXPECTED_ERROR;
+import static org.apache.ignite.internal.commandline.CommandHandler.UTILITY_NAME;
+import static org.apache.ignite.internal.commandline.CommandList.WAL;
import static org.apache.ignite.internal.commandline.OutputFormat.MULTI_LINE;
import static org.apache.ignite.internal.commandline.OutputFormat.SINGLE_LINE;
import static org.apache.ignite.internal.commandline.cache.CacheSubcommands.HELP;
@@ -134,6 +142,10 @@ import static org.apache.ignite.transactions.TransactionIsolation.READ_COMMITTED
* Command line handler test.
*/
public class GridCommandHandlerTest extends GridCommandHandlerAbstractTest {
+
+ /** */
+ @Rule public final TestRule methodRule = new SystemPropertiesRule();
+
/** */
private File defaultDiagnosticDir;
/** */
@@ -326,21 +338,21 @@ public class GridCommandHandlerTest extends GridCommandHandlerAbstractTest {
assertEquals(EXIT_CODE_OK, execute("--cache", "find_garbage", "--port", "11212"));
- assertTrue(testOut.toString().contains("garbage not found"));
+ assertContains(log, testOut.toString(), "garbage not found");
testOut.reset();
assertEquals(EXIT_CODE_OK, execute("--cache", "find_garbage",
ignite(0).localNode().id().toString(), "--port", "11212"));
- assertTrue(testOut.toString().contains("garbage not found"));
+ assertContains(log, testOut.toString(), "garbage not found");
testOut.reset();
assertEquals(EXIT_CODE_OK, execute("--cache", "find_garbage",
"groupGarbage", "--port", "11212"));
- assertTrue(testOut.toString().contains("garbage not found"));
+ assertContains(log, testOut.toString(), "garbage not found");
}
/**
@@ -3042,4 +3054,36 @@ public class GridCommandHandlerTest extends GridCommandHandlerAbstractTest {
e.printStackTrace();
}
}
+
+ /**
+ * Don't show wal commands by --help in case
+ * {@link org.apache.ignite.IgniteSystemProperties#IGNITE_ENABLE_EXPERIMENTAL_COMMAND} = false or empty.
+ */
+ @Test
+ @WithSystemProperty(key = IGNITE_ENABLE_EXPERIMENTAL_COMMAND, value = "false")
+ public void testHideWalInHelpWhenDisableExperimentalCommand() {
+ injectTestSystemOut();
+
+ execute("--help");
+
+ assertNotContains(log, testOut.toString(), WAL.text());
+ }
+
+ /**
+ * Wal commands should ignored and print warning in case
+ * {@link org.apache.ignite.IgniteSystemProperties#IGNITE_ENABLE_EXPERIMENTAL_COMMAND} = false or empty.
+ * */
+ @Test
+ @WithSystemProperty(key = IGNITE_ENABLE_EXPERIMENTAL_COMMAND, value = "false")
+ public void testWalCommandsInCaseDisableExperimentalCommand() {
+ injectTestSystemOut();
+
+ String warning = String.format("For use experimental command add %s=true to JVM_OPTS in %s",
+ IGNITE_ENABLE_EXPERIMENTAL_COMMAND, UTILITY_NAME);
+
+ Stream.of("print", "delete")
+ .peek(c -> testOut.reset())
+ .peek(c -> assertEquals(EXIT_CODE_OK, execute(WAL.text(), c)))
+ .forEach(c -> assertContains(log, testOut.toString(), warning));
+ }
}
[ignite] 14/19: GG-21210: IgniteCacheQueriesLoadTest1 scale factor
Posted by am...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
amashenkov pushed a commit to branch gg-21368
in repository https://gitbox.apache.org/repos/asf/ignite.git
commit 350aac24202f80c511c6383865d79f341aa33138
Author: pavel-kuznetsov <34...@users.noreply.github.com>
AuthorDate: Wed Jul 17 18:59:49 2019 +0300
GG-21210: IgniteCacheQueriesLoadTest1 scale factor
---
.../cache/IgniteCacheQueriesLoadTest1.java | 21 +++++++++++++--------
1 file changed, 13 insertions(+), 8 deletions(-)
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheQueriesLoadTest1.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheQueriesLoadTest1.java
index d1c6ca0..4277075 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheQueriesLoadTest1.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheQueriesLoadTest1.java
@@ -50,6 +50,7 @@ import org.apache.ignite.lang.IgniteCallable;
import org.apache.ignite.lang.IgniteRunnable;
import org.apache.ignite.resources.IgniteInstanceResource;
import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.GridTestUtils.SF;
import org.junit.Test;
import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
@@ -133,7 +134,7 @@ public class IgniteCacheQueriesLoadTest1 extends AbstractIndexingCommonTest {
private static Map<UUID, List<Integer>> partitionsMap;
/** Preload amount. */
- private static final int preloadAmount = 10_000;
+ private static final int preloadAmount = SF.applyLB(10_000, 2_000);
/** {@inheritDoc} */
@Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
@@ -167,23 +168,27 @@ public class IgniteCacheQueriesLoadTest1 extends AbstractIndexingCommonTest {
*/
@Test
public void testQueries() throws Exception {
- runQueries(1, true, 10_000);
+ int runSingleMs = SF.applyLB(10_000, 1_000);
- runQueries(10, false, 30_000);
+ runQueries(1, true, runSingleMs);
+
+ int runParMs = SF.applyLB(30_000, 3_000);
+
+ runQueries(10, false, runParMs);
}
/**
- * @param threads Threads number.
- * @param checkBalance Check balance flag.
- * @param time Execution time.
+ * @param threads number of threads executing the queries.
+ * @param checkBalance whether or not to perform balance check.
+ * @param runAtLeast minimum number of milliseconds thread should perform scan query in a loop.
* @throws Exception If failed.
*/
- private void runQueries(int threads, final boolean checkBalance, final long time) throws Exception {
+ private void runQueries(int threads, final boolean checkBalance, final long runAtLeast) throws Exception {
final Ignite ignite = grid(0);
GridTestUtils.runMultiThreaded(new Callable<Object>() {
@Override public Object call() {
- long endTime = System.currentTimeMillis() + time;
+ long endTime = System.currentTimeMillis() + runAtLeast;
while (System.currentTimeMillis() < endTime) {
ScanQueryBroadcastClosure c = new ScanQueryBroadcastClosure(partitionsMap, checkBalance);
[ignite] 10/19: GG-21254 Fixed *.testFailAfterStart failed on TC
Posted by am...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
amashenkov pushed a commit to branch gg-21368
in repository https://gitbox.apache.org/repos/asf/ignite.git
commit f7f0b89c947ac421c3e49a45a5357413198c59ba
Author: Dmitriy Govorukhin <dm...@gmail.com>
AuthorDate: Wed Jul 17 12:50:00 2019 +0300
GG-21254 Fixed *.testFailAfterStart failed on TC
---
...eWalFlushMultiNodeFailoverAbstractSelfTest.java | 59 ++++++++++++----------
1 file changed, 31 insertions(+), 28 deletions(-)
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java
index 815de97..d71d563 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java
@@ -25,8 +25,6 @@ import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.IgniteSystemProperties;
-import org.apache.ignite.cache.CacheAtomicityMode;
-import org.apache.ignite.cache.CacheRebalanceMode;
import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.configuration.DataRegionConfiguration;
@@ -43,6 +41,7 @@ import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAhea
import org.apache.ignite.internal.util.lang.GridAbsPredicate;
import org.apache.ignite.testframework.GridTestUtils;
import org.apache.ignite.testframework.MvccFeatureChecker;
+import org.apache.ignite.testframework.junits.WithSystemProperty;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
import org.apache.ignite.transactions.Transaction;
import org.apache.ignite.transactions.TransactionConcurrency;
@@ -51,15 +50,14 @@ import org.junit.Assume;
import org.junit.Test;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_BASELINE_AUTO_ADJUST_ENABLED;
+import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
+import static org.apache.ignite.cache.CacheRebalanceMode.SYNC;
/**
* Tests error recovery while node flushing
*/
public abstract class IgniteWalFlushMultiNodeFailoverAbstractSelfTest extends GridCommonAbstractTest {
/** */
- private static final String TEST_CACHE = "testCache";
-
- /** */
private static final int ITRS = 2000;
/** */
@@ -131,22 +129,23 @@ public abstract class IgniteWalFlushMultiNodeFailoverAbstractSelfTest extends Gr
cfg.setConsistentId(gridName);
- CacheConfiguration cacheCfg = new CacheConfiguration(TEST_CACHE)
- .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL)
+ cfg.setCacheConfiguration(
+ new CacheConfiguration(DEFAULT_CACHE_NAME)
+ .setAtomicityMode(TRANSACTIONAL)
.setBackups(1)
- .setRebalanceMode(CacheRebalanceMode.SYNC)
- .setAffinity(new RendezvousAffinityFunction(false, 32));
+ .setRebalanceMode(SYNC)
+ .setAffinity(new RendezvousAffinityFunction(false, 32))
+ );
- cfg.setCacheConfiguration(cacheCfg);
-
- DataStorageConfiguration memCfg = new DataStorageConfiguration()
+ cfg.setDataStorageConfiguration(
+ new DataStorageConfiguration()
.setDefaultDataRegionConfiguration(
- new DataRegionConfiguration().setMaxSize(2048L * 1024 * 1024).setPersistenceEnabled(true))
- .setWalMode(this.walMode())
+ new DataRegionConfiguration()
+ .setMaxSize(2048L * 1024 * 1024)
+ .setPersistenceEnabled(true))
+ .setWalMode(walMode())
.setWalSegmentSize(512 * 1024)
- .setWalBufferSize(512 * 1024);
-
- cfg.setDataStorageConfiguration(memCfg);
+ .setWalBufferSize(512 * 1024));
cfg.setFailureHandler(new StopNodeFailureHandler());
@@ -159,6 +158,7 @@ public abstract class IgniteWalFlushMultiNodeFailoverAbstractSelfTest extends Gr
* @throws Exception In case of fail
*/
@Test
+ @WithSystemProperty(key = "IGNITE_DISABLE_WAL_DURING_REBALANCING", value = "false")
public void testFailWhileStart() throws Exception {
failWhilePut(true);
}
@@ -169,6 +169,7 @@ public abstract class IgniteWalFlushMultiNodeFailoverAbstractSelfTest extends Gr
* @throws Exception In case of fail
*/
@Test
+ @WithSystemProperty(key = "IGNITE_DISABLE_WAL_DURING_REBALANCING", value = "false")
public void testFailAfterStart() throws Exception {
failWhilePut(false);
}
@@ -177,15 +178,15 @@ public abstract class IgniteWalFlushMultiNodeFailoverAbstractSelfTest extends Gr
* @throws Exception if failed.
*/
private void failWhilePut(boolean failWhileStart) throws Exception {
- final Ignite grid = startGridsMultiThreaded(gridCount());
+ Ignite ig = startGrids(gridCount());
- grid.cluster().active(true);
+ ig.cluster().active(true);
- IgniteCache<Object, Object> cache = grid.cache(TEST_CACHE);
+ IgniteCache<Object, Object> cache = ig.cache(DEFAULT_CACHE_NAME);
for (int i = 0; i < ITRS; i++) {
while (!Thread.currentThread().isInterrupted()) {
- try (Transaction tx = grid.transactions().txStart(
+ try (Transaction tx = ig.transactions().txStart(
TransactionConcurrency.PESSIMISTIC, TransactionIsolation.REPEATABLE_READ)) {
cache.put(i, "testValue" + i);
@@ -207,7 +208,7 @@ public abstract class IgniteWalFlushMultiNodeFailoverAbstractSelfTest extends Gr
setFileIOFactory(grid(gridCount()).context().cache().context().wal());
- grid.cluster().setBaselineTopology(grid.cluster().topologyVersion());
+ ig.cluster().setBaselineTopology(ig.cluster().topologyVersion());
awaitPartitionMapExchange();
}
@@ -223,7 +224,7 @@ public abstract class IgniteWalFlushMultiNodeFailoverAbstractSelfTest extends Gr
// We should await successful stop of node.
GridTestUtils.waitForCondition(new GridAbsPredicate() {
@Override public boolean apply() {
- return grid.cluster().nodes().size() == gridCount();
+ return ig.cluster().nodes().size() == gridCount();
}
}, getTestTimeout());
@@ -233,11 +234,9 @@ public abstract class IgniteWalFlushMultiNodeFailoverAbstractSelfTest extends Gr
Ignite grid0 = startGrids(gridCount() + 1);
- setFileIOFactory(grid(gridCount()).context().cache().context().wal());
-
grid0.cluster().active(true);
- cache = grid0.cache(TEST_CACHE);
+ cache = grid0.cache(DEFAULT_CACHE_NAME);
for (int i = 0; i < ITRS; i++)
assertEquals(cache.get(i), "testValue" + i);
@@ -276,7 +275,9 @@ public abstract class IgniteWalFlushMultiNodeFailoverAbstractSelfTest extends Gr
return new FileIODecorator(delegate) {
/** {@inheritDoc} */
@Override public int write(ByteBuffer srcBuf) throws IOException {
- if (fail != null && fail.get())
+ System.out.println(">>>!!!! W "+file.getName());
+
+ if (fail != null && file.getName().endsWith(".wal") && fail.get())
throw new IOException("No space left on device");
return super.write(srcBuf);
@@ -284,7 +285,9 @@ public abstract class IgniteWalFlushMultiNodeFailoverAbstractSelfTest extends Gr
/** {@inheritDoc} */
@Override public MappedByteBuffer map(int sizeBytes) throws IOException {
- if (fail != null && fail.get())
+ System.out.println(">>>!!!! M "+file.getName());
+
+ if (fail != null && file.getName().endsWith(".wal") && fail.get())
throw new IOException("No space left on deive");
return delegate.map(sizeBytes);
[ignite] 02/19: GG-14870 AuthenticationEvent address does not
contain full list of addresses
Posted by am...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
amashenkov pushed a commit to branch gg-21368
in repository https://gitbox.apache.org/repos/asf/ignite.git
commit 6fb651916987d6f1aea47378af1d5e29f4629cae
Author: akorensh <ak...@gmail.com>
AuthorDate: Tue Jul 16 13:23:30 2019 -0400
GG-14870 AuthenticationEvent address does not contain full list of addresses
---
.../plugin/security/AuthenticationContext.java | 23 ++++++++++++++++++++++
1 file changed, 23 insertions(+)
diff --git a/modules/core/src/main/java/org/apache/ignite/plugin/security/AuthenticationContext.java b/modules/core/src/main/java/org/apache/ignite/plugin/security/AuthenticationContext.java
index f9199f2..d7a528f 100644
--- a/modules/core/src/main/java/org/apache/ignite/plugin/security/AuthenticationContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/plugin/security/AuthenticationContext.java
@@ -18,6 +18,7 @@ package org.apache.ignite.plugin.security;
import java.net.InetSocketAddress;
import java.util.Collections;
+import java.util.List;
import java.util.Map;
import java.util.UUID;
import org.apache.ignite.internal.processors.authentication.AuthorizationContext;
@@ -38,6 +39,9 @@ public class AuthenticationContext {
/** Subject address. */
private InetSocketAddress addr;
+ /** All Subject address. */
+ private List<InetSocketAddress> allAddr;
+
/** */
private Map<String, Object> nodeAttrs;
@@ -169,4 +173,23 @@ public class AuthenticationContext {
return this;
}
+
+ /**
+ * Gets All subject network addresses.
+ *
+ * @return All subject network addresses.
+ */
+ public List<InetSocketAddress> allAddresses() {
+ return allAddr;
+ }
+
+ /**
+ * Sets All subject network addresses.
+ *
+ * @param allAddr List of All subject network addresses.
+ */
+ public void allAddresses(List<InetSocketAddress> allAddr) {
+ this.allAddr = allAddr;
+ }
+
}
\ No newline at end of file
[ignite] 01/19: GG-21394: missed slf4j-simple-1.7.7.jar in binary
packages
Posted by am...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
amashenkov pushed a commit to branch gg-21368
in repository https://gitbox.apache.org/repos/asf/ignite.git
commit 5c5769e893d0f72b24afd2396153f4cfe4633b55
Author: Alexey Platonov <40...@users.noreply.github.com>
AuthorDate: Tue Jul 16 17:58:33 2019 +0300
GG-21394: missed slf4j-simple-1.7.7.jar in binary packages
---
modules/slf4j/pom.xml | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/modules/slf4j/pom.xml b/modules/slf4j/pom.xml
index 4f148fc..84a470c 100644
--- a/modules/slf4j/pom.xml
+++ b/modules/slf4j/pom.xml
@@ -47,6 +47,12 @@
</dependency>
<dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-simple</artifactId>
+ <version>${slf4j.version}</version>
+ </dependency>
+
+ <dependency>
<groupId>org.gridgain</groupId>
<artifactId>ignite-core</artifactId>
<version>${project.version}</version>
[ignite] 06/19: GG-20750: SQL: Fix performance drop with enabling
global SQL memory quota.
Posted by am...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
amashenkov pushed a commit to branch gg-21368
in repository https://gitbox.apache.org/repos/asf/ignite.git
commit 5417a6c5e109e1e3bb3eaa402bf1275026dc000f
Author: Andrey V. Mashenkov <an...@gmail.com>
AuthorDate: Wed Jul 17 11:09:10 2019 +0300
GG-20750: SQL: Fix performance drop with enabling global SQL memory quota.
---
.../processors/query/h2/H2MemoryTracker.java | 3 +-
.../processors/query/h2/QueryMemoryManager.java | 72 +++++++++----
.../processors/query/h2/QueryMemoryTracker.java | 113 +++++++++++----------
.../oom/AbstractQueryMemoryTrackerSelfTest.java | 27 ++++-
...yMemoryTrackerWithQueryParallelismSelfTest.java | 36 +++++++
.../query/oom/QueryMemoryTrackerSelfTest.java | 11 ++
6 files changed, 183 insertions(+), 79 deletions(-)
diff --git a/modules/h2/src/main/java/org/apache/ignite/internal/processors/query/h2/H2MemoryTracker.java b/modules/h2/src/main/java/org/apache/ignite/internal/processors/query/h2/H2MemoryTracker.java
index e1d3aac..bbcc5f4 100644
--- a/modules/h2/src/main/java/org/apache/ignite/internal/processors/query/h2/H2MemoryTracker.java
+++ b/modules/h2/src/main/java/org/apache/ignite/internal/processors/query/h2/H2MemoryTracker.java
@@ -21,6 +21,8 @@ package org.apache.ignite.internal.processors.query.h2;
*/
public abstract class H2MemoryTracker implements AutoCloseable {
/**
+ * Reserve memory.
+ *
* @param size Memory to reserve in bytes.
*/
public abstract void reserve(long size);
@@ -30,6 +32,5 @@ public abstract class H2MemoryTracker implements AutoCloseable {
*
* @param size Memory to release in bytes.
*/
-
public abstract void release(long size);
}
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/QueryMemoryManager.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/QueryMemoryManager.java
index 58ae9e2..727ed35 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/QueryMemoryManager.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/QueryMemoryManager.java
@@ -17,6 +17,7 @@
package org.apache.ignite.internal.processors.query.h2;
import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.LongBinaryOperator;
import org.apache.ignite.IgniteLogger;
import org.apache.ignite.IgniteSystemProperties;
import org.apache.ignite.configuration.IgniteConfiguration;
@@ -37,6 +38,19 @@ public class QueryMemoryManager extends H2MemoryTracker {
*/
private static final long DFLT_MEMORY_RESERVATION_BLOCK_SIZE = 512 * KB;
+ /** */
+ static final LongBinaryOperator RELEASE_OP = new LongBinaryOperator() {
+ @Override public long applyAsLong(long prev, long x) {
+ long res = prev - x;
+
+ if (res < 0)
+ throw new IllegalStateException("Try to free more memory that ever be reserved: [" +
+ "reserved=" + prev + ", toFree=" + x + ']');
+
+ return res;
+ }
+ };
+
/**
* Default query memory limit.
*
@@ -48,6 +62,9 @@ public class QueryMemoryManager extends H2MemoryTracker {
/** Logger. */
private final IgniteLogger log;
+ /** */
+ private final LongBinaryOperator reserveOp;
+
/** Global query memory quota. */
//TODO GG-18629: it looks safe to make this configurable at runtime.
private final long globalQuota;
@@ -74,48 +91,38 @@ public class QueryMemoryManager extends H2MemoryTracker {
(long)(Runtime.getRuntime().maxMemory() * 0.6d));
}
- long dfltSqlQryMemoryLimit = Long.getLong(IgniteSystemProperties.IGNITE_DEFAULT_SQL_QUERY_MEMORY_LIMIT, 0);
+ long dfltMemLimit = Long.getLong(IgniteSystemProperties.IGNITE_DEFAULT_SQL_QUERY_MEMORY_LIMIT, 0);
- if (dfltSqlQryMemoryLimit == 0)
- dfltSqlQryMemoryLimit = globalQuota > 0 ? globalQuota / IgniteConfiguration.DFLT_QUERY_THREAD_POOL_SIZE : -1;
+ if (dfltMemLimit == 0)
+ dfltMemLimit = globalQuota > 0 ? globalQuota / IgniteConfiguration.DFLT_QUERY_THREAD_POOL_SIZE : -1;
this.blockSize = Long.getLong(IgniteSystemProperties.IGNITE_SQL_MEMORY_RESERVATION_BLOCK_SIZE, DFLT_MEMORY_RESERVATION_BLOCK_SIZE);
this.globalQuota = globalQuota;
- this.dfltSqlQryMemoryLimit = dfltSqlQryMemoryLimit;
+ this.dfltSqlQryMemoryLimit = dfltMemLimit;
+
+ this.reserveOp = new ReservationOp(globalQuota);
this.log = ctx.log(QueryMemoryManager.class);
}
/** {@inheritDoc} */
@Override public void reserve(long size) {
- assert size >= 0;
-
if (size == 0)
return; // Nothing to do.
- reserved.accumulateAndGet(size, (prev, x) -> {
- if (prev + x > globalQuota)
- throw new IgniteSQLException("SQL query run out of memory: Global quota exceeded.",
- IgniteQueryErrorCode.QUERY_OUT_OF_MEMORY);
+ assert size > 0;
- return prev + x;
- });
+ reserved.accumulateAndGet(size, reserveOp);
}
/** {@inheritDoc} */
@Override public void release(long size) {
- assert size >= 0;
-
if (size == 0)
return; // Nothing to do.
- reserved.accumulateAndGet(-size, (prev, x) -> {
- if (prev + x < 0)
- throw new IllegalStateException("Try to free more memory that ever be reserved: [" +
- "reserved=" + prev + ", toFree=" + x + ']');
+ assert size > 0;
- return prev + x;
- });
+ reserved.accumulateAndGet(size, RELEASE_OP);
}
/**
@@ -172,4 +179,29 @@ public class QueryMemoryManager extends H2MemoryTracker {
if (log.isDebugEnabled() && reserved.get() != 0)
log.debug("Potential memory leak in SQL processor. Some query cursors were not closed or forget to free memory.");
}
+
+ /** */
+ private static class ReservationOp implements LongBinaryOperator {
+ /** Operation result high bound.*/
+ private final long limit;
+
+ /**
+ * Constructor.
+ * @param limit Operation result high bound.
+ */
+ ReservationOp(long limit) {
+ this.limit = limit;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long applyAsLong(long prev, long x) {
+ long res = prev + x;
+
+ if (res > limit)
+ throw new IgniteSQLException("SQL query run out of memory: Global quota exceeded.",
+ IgniteQueryErrorCode.QUERY_OUT_OF_MEMORY);
+
+ return res;
+ }
+ }
}
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/QueryMemoryTracker.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/QueryMemoryTracker.java
index d0ff5c2..ea31930 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/QueryMemoryTracker.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/QueryMemoryTracker.java
@@ -16,8 +16,6 @@
package org.apache.ignite.internal.processors.query.h2;
-import java.util.concurrent.atomic.AtomicLongFieldUpdater;
-import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
import org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode;
import org.apache.ignite.internal.processors.query.IgniteSQLException;
import org.apache.ignite.internal.util.typedef.internal.S;
@@ -28,14 +26,6 @@ import org.apache.ignite.internal.util.typedef.internal.S;
* Track query memory usage and throws an exception if query tries to allocate memory over limit.
*/
public class QueryMemoryTracker extends H2MemoryTracker implements AutoCloseable {
- /** Resered field updater. */
- private static final AtomicLongFieldUpdater<QueryMemoryTracker> RESERVED_UPD =
- AtomicLongFieldUpdater.newUpdater(QueryMemoryTracker.class, "reserved");
-
- /** Closed flag updater. */
- private static final AtomicReferenceFieldUpdater<QueryMemoryTracker, Boolean> CLOSED_UPD =
- AtomicReferenceFieldUpdater.newUpdater(QueryMemoryTracker.class, Boolean.class, "closed");
-
/** Parent tracker. */
private final H2MemoryTracker parent;
@@ -46,13 +36,13 @@ public class QueryMemoryTracker extends H2MemoryTracker implements AutoCloseable
private final long blockSize;
/** Memory reserved on parent. */
- private volatile long reservedFromParent;
+ private long reservedFromParent;
/** Memory reserved by query. */
- private volatile long reserved;
+ private long reserved;
/** Close flag to prevent tracker reuse. */
- private volatile Boolean closed = Boolean.FALSE;
+ private Boolean closed = Boolean.FALSE;
/**
* Constructor.
@@ -71,40 +61,31 @@ public class QueryMemoryTracker extends H2MemoryTracker implements AutoCloseable
/** {@inheritDoc} */
@Override public void reserve(long size) {
- assert !closed && size >= 0;
-
if (size == 0)
return;
- long reserved0 = RESERVED_UPD.accumulateAndGet(this, size, (prev, x) -> {
- if (prev + x > maxMem) {
- throw new IgniteSQLException("SQL query run out of memory: Query quota exceeded.",
- IgniteQueryErrorCode.QUERY_OUT_OF_MEMORY);
- }
+ assert size > 0;
- return prev + x;
- });
+ synchronized (this) {
+ if (closed)
+ throw new IllegalStateException("Memory tracker has been closed concurrently.");
- if (parent != null && reserved0 > reservedFromParent) {
- synchronized (this) {
- assert !closed;
-
- if (reserved0 <= reservedFromParent)
- return;
-
- // If single block size is too small.
- long blockSize = Math.max(reserved0 - reservedFromParent, this.blockSize);
- // If we are too close to limit.
- blockSize = Math.min(blockSize, maxMem - reservedFromParent);
+ long reserved0 = reserve0(size);
+ if (parent != null && reserved0 > reservedFromParent) {
try {
+ // If single block size is too small.
+ long blockSize = Math.max(reserved0 - reservedFromParent, this.blockSize);
+ // If we are too close to limit.
+ blockSize = Math.min(blockSize, maxMem - reservedFromParent);
+
parent.reserve(blockSize);
reservedFromParent += blockSize;
}
catch (Throwable e) {
// Fallback if failed to reserve.
- RESERVED_UPD.addAndGet(this, -size);
+ release0(size);
throw e;
}
@@ -114,31 +95,50 @@ public class QueryMemoryTracker extends H2MemoryTracker implements AutoCloseable
/** {@inheritDoc} */
@Override public void release(long size) {
- assert size >= 0;
-
if (size == 0)
return;
- long reserved = RESERVED_UPD.accumulateAndGet(this, -size, (prev, x) -> {
- if (prev + x < 0)
- throw new IllegalStateException("Try to release more memory that were reserved: [" +
- "reserved=" + prev + ", toRelease=" + x + ']');
+ assert size > 0;
- return prev + x;
- });
+ synchronized (this) {
+ if (closed)
+ throw new IllegalStateException("Memory tracker has been closed concurrently.");
- assert !closed && reserved >= 0 || reserved == 0 : "Invalid reserved memory size:" + reserved;
+ long reserved = release0(size);
- // For now, won'tQ release memory to parent until tracker closed.
- /* if (parent != null && preAllocated - reserved >= 2 * blockSize) {
- synchronized (this) {
- if (preAllocated - reserved >= 2 * blockSize) {
- parent.release(blockSize);
+ assert reserved >= 0;
+ }
+ }
- preAllocated -= blockSize;
- }
- }
- }*/
+ /**
+ * @param size Memory to reserve in bytes.
+ * @return Reserved memory after release.
+ */
+ private long reserve0(long size) {
+ long res = reserved + size;
+
+ if (res > maxMem)
+ throw new IgniteSQLException("SQL query run out of memory: Query quota exceeded.",
+ IgniteQueryErrorCode.QUERY_OUT_OF_MEMORY);
+
+ return reserved = res;
+ }
+
+ /**
+ * Release reserved memory.
+ *
+ * @param size Memory to release in bytes.
+ * @return Reserved memory after release.
+ */
+ private long release0(long size) {
+
+ long res = reserved - size;
+
+ if (res < 0)
+ throw new IllegalStateException("Try to free more memory that ever be reserved: [" +
+ "reserved=" + reserved + ", toFree=" + size + ']');
+
+ return reserved = res;
}
/**
@@ -152,8 +152,13 @@ public class QueryMemoryTracker extends H2MemoryTracker implements AutoCloseable
@Override public void close() {
// It is not expected to be called concurrently with reserve\release.
// But query can be cancelled concurrently on query finish.
- if (CLOSED_UPD.compareAndSet(this, Boolean.FALSE, Boolean.TRUE)) {
- release(RESERVED_UPD.get(this));
+ synchronized (this) {
+ if (closed)
+ return;
+
+ closed = true;
+
+ release0(reserved);
if (parent != null)
parent.release(reservedFromParent);
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/oom/AbstractQueryMemoryTrackerSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/oom/AbstractQueryMemoryTrackerSelfTest.java
index c77ed15..2fc6231 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/oom/AbstractQueryMemoryTrackerSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/oom/AbstractQueryMemoryTrackerSelfTest.java
@@ -44,6 +44,7 @@ import org.h2.engine.Session;
import org.h2.expression.Expression;
import org.h2.result.H2BaseLocalResult;
import org.h2.result.LocalResult;
+import org.junit.Ignore;
import org.junit.Test;
import sun.reflect.generics.reflectiveObjects.NotImplementedException;
@@ -371,9 +372,10 @@ public abstract class AbstractQueryMemoryTrackerSelfTest extends GridCommonAbstr
/** Check GROUP BY operation on indexed col. */
@Test
+ @Ignore("https://ggsystems.atlassian.net/browse/GG-19071")
public void testQueryWithGroupByPrimaryKey() throws Exception {
- //TODO: GG-19071: make next test pass without hint.
- execQuery("select K.indexed, sum(K.id) from K USE INDEX (K_IDX) GROUP BY K.indexed", true);
+ //TODO: GG-19071: make next query use correct index (K_IDX instead of primary).
+ execQuery("select K.indexed, sum(K.id) from K GROUP BY K.indexed", true);
assertEquals(0, localResults.size());
}
@@ -396,7 +398,7 @@ public abstract class AbstractQueryMemoryTrackerSelfTest extends GridCommonAbstr
// Local result is quite small.
assertEquals(1, localResults.size());
- assertTrue(maxMem > localResults.get(0).memoryReserved());
+ assertTrue(maxMem > localResults.get(0).memoryReserved() + 1000);
assertTrue(BIG_TABLE_SIZE > localResults.get(0).getRowCount());
}
@@ -407,7 +409,7 @@ public abstract class AbstractQueryMemoryTrackerSelfTest extends GridCommonAbstr
// Local result is quite small.
assertEquals(1, localResults.size());
- assertTrue(maxMem > localResults.get(0).memoryReserved());
+ assertTrue(maxMem > localResults.get(0).memoryReserved() + 1000);
assertTrue(100 > localResults.get(0).getRowCount());
}
@@ -428,6 +430,7 @@ public abstract class AbstractQueryMemoryTrackerSelfTest extends GridCommonAbstr
"GROUP BY K.indexed ORDER BY a DESC", true);
assertEquals(1, localResults.size());
+ assertTrue(maxMem < localResults.get(0).memoryReserved() + 1000);
assertTrue(BIG_TABLE_SIZE > localResults.get(0).getRowCount());
}
@@ -447,6 +450,9 @@ public abstract class AbstractQueryMemoryTrackerSelfTest extends GridCommonAbstr
public void testQueryWithDistinctAndLowCardinality() throws Exception {
// Distinct on indexed column with small cardinality.
execQuery("select DISTINCT K.grp_indexed from K", false);
+
+ assertEquals(1, localResults.size());
+ assertEquals(100, localResults.get(0).getRowCount());
}
/** Check query failure with DISTINCT constraint. */
@@ -454,6 +460,9 @@ public abstract class AbstractQueryMemoryTrackerSelfTest extends GridCommonAbstr
public void testQueryWithDistinctAndHighCardinality() throws Exception {
// Distinct on indexed column with unique values.
checkQueryExpectOOM("select DISTINCT K.id from K", true);
+
+ assertEquals(1, localResults.size());
+ assertTrue(BIG_TABLE_SIZE > localResults.get(0).getRowCount());
}
/** Check HashJoin with large table. */
@@ -473,6 +482,16 @@ public abstract class AbstractQueryMemoryTrackerSelfTest extends GridCommonAbstr
}
}
+ /** Check Join with large table. */
+ @Test
+ public void testJoinWithLargeTable() throws Exception {
+ maxMem = 512 * KB;
+
+ execQuery("select * from T, K where T.id = K.grp_indexed", true);
+
+ assertEquals(0, localResults.size());
+ }
+
/** Check query failure due to global memory quota exceeded. */
@Test
public void testGlobalQuota() throws Exception {
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/oom/LocalQueryMemoryTrackerWithQueryParallelismSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/oom/LocalQueryMemoryTrackerWithQueryParallelismSelfTest.java
index acb1fc0..4569792 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/oom/LocalQueryMemoryTrackerWithQueryParallelismSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/oom/LocalQueryMemoryTrackerWithQueryParallelismSelfTest.java
@@ -339,4 +339,40 @@ public class LocalQueryMemoryTrackerWithQueryParallelismSelfTest extends Abstrac
assertEquals(1, localResults.size());
assertTrue(100 > localResults.get(0).getRowCount());
}
+
+ /** Check simple query with DISTINCT constraint. */
+ @Test
+ @Override public void testQueryWithDistinctAndLowCardinality() throws Exception {
+ // Distinct on indexed column with small cardinality.
+ execQuery("select DISTINCT K.grp_indexed from K", false);
+
+ assertEquals(5, localResults.size());
+ assertEquals(100, localResults.get(4).getRowCount());
+ }
+
+ /** Check query failure with DISTINCT constraint. */
+ @Test
+ @Override public void testQueryWithDistinctAndHighCardinality() throws Exception {
+ // Distinct on indexed column with unique values.
+ checkQueryExpectOOM("select DISTINCT K.id from K", true);
+
+ assertEquals(5, localResults.size());
+ assertFalse(localResults.stream().limit(4).anyMatch(r -> r.memoryReserved() + 1000 > maxMem));
+ assertTrue(BIG_TABLE_SIZE > localResults.get(4).getRowCount());
+ }
+
+ /** {@inheritDoc} */
+ @Test
+ @Override public void testLazyQueryWithGroupByThenSort() {
+ maxMem = MB / 2;
+
+ // Query failed on map side due too many groups.
+ checkQueryExpectOOM("select K.indexed, sum(K.grp) as a from K " +
+ "GROUP BY K.indexed ORDER BY a DESC", true);
+
+ // Result on reduce side.
+ assertEquals(1, localResults.size());
+ assertEquals(0, localResults.get(0).memoryReserved());
+ assertEquals(0, localResults.get(0).getRowCount());
+ }
}
\ No newline at end of file
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/oom/QueryMemoryTrackerSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/oom/QueryMemoryTrackerSelfTest.java
index 6c29013..f612df9 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/oom/QueryMemoryTrackerSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/oom/QueryMemoryTrackerSelfTest.java
@@ -176,6 +176,17 @@ public class QueryMemoryTrackerSelfTest extends AbstractQueryMemoryTrackerSelfTe
assertTrue(BIG_TABLE_SIZE > localResults.get(0).getRowCount());
}
+ /** Check simple query with DISTINCT constraint. */
+ @Test
+ @Override public void testQueryWithDistinctAndLowCardinality() throws Exception {
+ // Distinct on indexed column with small cardinality.
+ execQuery("select DISTINCT K.grp_indexed from K", false);
+
+ assertEquals(2, localResults.size());
+ assertEquals(100, localResults.get(0).getRowCount());
+ assertEquals(100, localResults.get(1).getRowCount());
+ }
+
/** {@inheritDoc} */
@Test
@Override public void testLazyQueryWithGroupByIndexedColAndDistinctAggregates() {
[ignite] 17/19: GG-21424 Fixed compatibility of cache filter of
visor idle verify arguments
Posted by am...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
amashenkov pushed a commit to branch gg-21368
in repository https://gitbox.apache.org/repos/asf/ignite.git
commit 04c001eb4f5c1c87834ddcad85c236a2f591ecc1
Author: Anton Kalashnikov <ka...@yandex.ru>
AuthorDate: Thu Jul 18 12:23:14 2019 +0300
GG-21424 Fixed compatibility of cache filter of visor idle verify arguments
---
.../cache/verify/VerifyBackupPartitionsTaskV2.java | 7 +------
.../internal/visor/verify/VisorIdleVerifyTaskArg.java | 15 ++++++++-------
.../ignite/util/GridCommandHandlerAbstractTest.java | 4 ++--
3 files changed, 11 insertions(+), 15 deletions(-)
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTaskV2.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTaskV2.java
index f0b240d..6b7c125 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTaskV2.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTaskV2.java
@@ -504,12 +504,7 @@ public class VerifyBackupPartitionsTaskV2 extends ComputeTaskAdapter<VisorIdleVe
CacheConfiguration cc = desc.cacheConfiguration();
- CacheFilterEnum cacheFilter = arg.cacheFilterEnum();
-
- if (cacheFilter == null)
- cacheFilter = CacheFilterEnum.DEFAULT;
-
- switch (cacheFilter) {
+ switch (arg.cacheFilterEnum()) {
case DEFAULT:
return desc.cacheType().userCache() || !F.isEmpty(arg.caches());
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyTaskArg.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyTaskArg.java
index 8b706f7..86b8940 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyTaskArg.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyTaskArg.java
@@ -23,10 +23,10 @@ import java.util.Set;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.internal.visor.VisorDataTransferObject;
+import org.jetbrains.annotations.NotNull;
/**
- * Arguments for task {@link VisorIdleVerifyTask}.
- * <br/>
+ * Arguments for task {@link VisorIdleVerifyTask}. <br/>
*/
public class VisorIdleVerifyTaskArg extends VisorDataTransferObject {
/** */
@@ -71,8 +71,9 @@ public class VisorIdleVerifyTaskArg extends VisorDataTransferObject {
this.caches = caches;
this.excludeCaches = excludeCaches;
this.skipZeros = skipZeros;
- this.cacheFilterEnum = (cacheFilterEnum == null ? CacheFilterEnum.DEFAULT : cacheFilterEnum);
this.checkCrc = checkCrc;
+
+ cacheFilterEnum(cacheFilterEnum);
}
/**
@@ -159,10 +160,10 @@ public class VisorIdleVerifyTaskArg extends VisorDataTransferObject {
if (protoVer >= V4) {
skipZeros = in.readBoolean();
- CacheFilterEnum cfe = CacheFilterEnum.fromOrdinal(in.readByte());
-
- cacheFilterEnum = (cfe == null ? CacheFilterEnum.DEFAULT : cfe);
+ cacheFilterEnum = CacheFilterEnum.fromOrdinal(in.readByte());
}
+
+ cacheFilterEnum(cacheFilterEnum);
}
}
@@ -196,7 +197,7 @@ public class VisorIdleVerifyTaskArg extends VisorDataTransferObject {
/**
* @return Kind fo cache.
*/
- public CacheFilterEnum cacheFilterEnum() {
+ @NotNull public CacheFilterEnum cacheFilterEnum() {
return cacheFilterEnum;
}
diff --git a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerAbstractTest.java
index 7b385d3..e1c0284 100644
--- a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerAbstractTest.java
@@ -123,13 +123,13 @@ public class GridCommandHandlerAbstractTest extends GridCommonAbstractTest {
log.info("Test output for " + currentTestMethod());
log.info("----------------------------------------");
+ System.setOut(sysOut);
+
if (testOut != null)
System.out.println(testOut.toString());
testOut = null;
- System.setOut(sysOut);
-
stopAllGrids();
cleanPersistenceDir();
[ignite] 07/19: GG-19117 GridCacheProcessor should add info about
cache in exception message, if applicable.
Posted by am...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
amashenkov pushed a commit to branch gg-21368
in repository https://gitbox.apache.org/repos/asf/ignite.git
commit 4b045f5a0f04ce89a3266f825bebd3136af5c0b0
Author: ktkalenko <kt...@gridgain.com>
AuthorDate: Wed Jul 17 11:54:50 2019 +0300
GG-19117 GridCacheProcessor should add info about cache in exception message, if applicable.
---
.../processors/cache/GridCacheProcessor.java | 63 ++++++++++++++++++----
1 file changed, 52 insertions(+), 11 deletions(-)
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index 7399123..ee17408 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -35,6 +35,7 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
+import java.util.function.Supplier;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import javax.cache.configuration.FactoryBuilder;
@@ -240,6 +241,12 @@ public class GridCacheProcessor extends GridProcessorAdapter {
"(encryption settings are different for cache '%s' : local=%s, remote=%s.)";
/** */
+ private static final String CACHE_NAME_AND_OPERATION_FORMAT = "[cacheName=%s, operation=%s]";
+
+ /** */
+ private static final String CACHE_NAMES_AND_OPERATION_FORMAT = "[cacheNames=%s, operation=%s]";
+
+ /** */
private final boolean startClientCaches =
IgniteSystemProperties.getBoolean(IgniteSystemProperties.IGNITE_START_CACHES_ON_JOIN, false);
@@ -4052,8 +4059,10 @@ public class GridCacheProcessor extends GridProcessorAdapter {
) {
assert cacheName != null;
- if (checkThreadTx)
- checkEmptyTransactions();
+ if (checkThreadTx) {
+ checkEmptyTransactionsEx(() -> String.format(CACHE_NAME_AND_OPERATION_FORMAT, cacheName,
+ "dynamicStartCache"));
+ }
GridPlainClosure<Collection<byte[]>, IgniteInternalFuture<Boolean>> startCacheClsr = (grpKeys) -> {
assert ccfg == null || !ccfg.isEncryptionEnabled() || !grpKeys.isEmpty();
@@ -4202,8 +4211,16 @@ public class GridCacheProcessor extends GridProcessorAdapter {
boolean disabledAfterStart,
IgniteUuid restartId
) {
- if (checkThreadTx)
- checkEmptyTransactions();
+ if (checkThreadTx) {
+ checkEmptyTransactionsEx(() -> {
+ List<String> cacheNames = storedCacheDataList.stream()
+ .map(StoredCacheData::config)
+ .map(CacheConfiguration::getName)
+ .collect(Collectors.toList());
+
+ return String.format(CACHE_NAMES_AND_OPERATION_FORMAT, cacheNames, "dynamicStartCachesByStoredConf");
+ });
+ }
GridPlainClosure<Collection<byte[]>, IgniteInternalFuture<Boolean>> startCacheClsr = (grpKeys) -> {
List<DynamicCacheChangeRequest> srvReqs = null;
@@ -4305,8 +4322,10 @@ public class GridCacheProcessor extends GridProcessorAdapter {
) {
assert cacheName != null;
- if (checkThreadTx)
- checkEmptyTransactions();
+ if (checkThreadTx) {
+ checkEmptyTransactionsEx(() -> String.format(CACHE_NAME_AND_OPERATION_FORMAT, cacheName,
+ "dynamicDestroyCache"));
+ }
DynamicCacheChangeRequest req = DynamicCacheChangeRequest.stopRequest(ctx, cacheName, sql, true);
@@ -4338,8 +4357,10 @@ public class GridCacheProcessor extends GridProcessorAdapter {
boolean checkThreadTx,
boolean destroy
) {
- if (checkThreadTx)
- checkEmptyTransactions();
+ if (checkThreadTx) {
+ checkEmptyTransactionsEx(() -> String.format(CACHE_NAMES_AND_OPERATION_FORMAT, cacheNames,
+ "dynamicDestroyCaches"));
+ }
List<DynamicCacheChangeRequest> reqs = new ArrayList<>(cacheNames.size());
@@ -4425,7 +4446,7 @@ public class GridCacheProcessor extends GridProcessorAdapter {
if (proxy == null || proxy.isProxyClosed())
return new GridFinishedFuture<>(); // No-op.
- checkEmptyTransactions();
+ checkEmptyTransactionsEx(() -> String.format(CACHE_NAME_AND_OPERATION_FORMAT, cacheName, "dynamicCloseCache"));
if (proxy.context().isLocal())
return dynamicDestroyCache(cacheName, false, true, false, null);
@@ -4440,11 +4461,14 @@ public class GridCacheProcessor extends GridProcessorAdapter {
* @return Future that will be completed when state is changed for all caches.
*/
public IgniteInternalFuture<?> resetCacheState(Collection<String> cacheNames) {
- checkEmptyTransactions();
-
if (F.isEmpty(cacheNames))
cacheNames = cachesInfo.registeredCaches().keySet();
+ Collection<String> forCheckCacheNames = cacheNames;
+
+ checkEmptyTransactionsEx(() -> String.format(CACHE_NAME_AND_OPERATION_FORMAT, forCheckCacheNames,
+ "resetCacheState"));
+
Collection<DynamicCacheChangeRequest> reqs = new ArrayList<>(cacheNames.size());
for (String cacheName : cacheNames) {
@@ -5640,6 +5664,23 @@ public class GridCacheProcessor extends GridProcessorAdapter {
}
/**
+ * Method invoke {@link #checkEmptyTransactions()} and add message in case exception.
+ *
+ * @param eMsgSupplier supplier additional text message
+ * @throws IgniteException If {@link #checkEmptyTransactions()} throw {@link IgniteException}
+ * */
+ private void checkEmptyTransactionsEx(final Supplier<String> eMsgSupplier) throws IgniteException {
+ assert eMsgSupplier != null;
+
+ try {
+ checkEmptyTransactions();
+ }
+ catch (IgniteException e) {
+ throw new IgniteException(e.getMessage() + ' ' + eMsgSupplier.get(), e);
+ }
+ }
+
+ /**
* @param val Object to check.
* @return Configuration copy.
* @throws IgniteCheckedException If validation failed.
[ignite] 13/19: GG-21468 Fix javadocs. (#289)
Posted by am...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
amashenkov pushed a commit to branch gg-21368
in repository https://gitbox.apache.org/repos/asf/ignite.git
commit 5f7a54bf4db25aa982759b6c06e3ffc2bebf3e09
Author: smoldachev <47...@users.noreply.github.com>
AuthorDate: Wed Jul 17 15:19:03 2019 +0300
GG-21468 Fix javadocs. (#289)
---
.../java/org/apache/ignite/console/websocket/WebSocketEvent.java | 5 +++++
.../apache/ignite/console/config/MailServiceAutoConfiguration.java | 4 +++-
parent/pom.xml | 4 ++++
3 files changed, 12 insertions(+), 1 deletion(-)
diff --git a/modules/web-console/web-console-common/src/main/java/org/apache/ignite/console/websocket/WebSocketEvent.java b/modules/web-console/web-console-common/src/main/java/org/apache/ignite/console/websocket/WebSocketEvent.java
index d586682..91429f0 100644
--- a/modules/web-console/web-console-common/src/main/java/org/apache/ignite/console/websocket/WebSocketEvent.java
+++ b/modules/web-console/web-console-common/src/main/java/org/apache/ignite/console/websocket/WebSocketEvent.java
@@ -16,6 +16,11 @@
package org.apache.ignite.console.websocket;
+/**
+ * Websocket event with payload.
+ *
+ * @param <T> payload type.
+ */
public interface WebSocketEvent<T> {
/**
* @return Request ID.
diff --git a/modules/web-console/web-console-mail/src/main/java/org/apache/ignite/console/config/MailServiceAutoConfiguration.java b/modules/web-console/web-console-mail/src/main/java/org/apache/ignite/console/config/MailServiceAutoConfiguration.java
index ba9a33c..9f8cba6 100644
--- a/modules/web-console/web-console-mail/src/main/java/org/apache/ignite/console/config/MailServiceAutoConfiguration.java
+++ b/modules/web-console/web-console-mail/src/main/java/org/apache/ignite/console/config/MailServiceAutoConfiguration.java
@@ -32,7 +32,9 @@ import org.springframework.context.annotation.Primary;
import org.springframework.mail.MailSender;
import org.springframework.mail.javamail.JavaMailSender;
-/** */
+/**
+ * Mail service auto configuration.
+ */
@Configuration
@Conditional(MailServiceAutoConfiguration.MailSenderCondition.class)
@ConditionalOnClass({ MimeMessage.class, MimeType.class, MailSender.class, MailService.class })
diff --git a/parent/pom.xml b/parent/pom.xml
index 453ec89..cc0cbf8 100644
--- a/parent/pom.xml
+++ b/parent/pom.xml
@@ -409,6 +409,10 @@
<packages>org.apache.ignite.spi.eventstorage*</packages>
</group>
<group>
+ <title>SPI: Metric</title>
+ <packages>org.apache.ignite.spi.metric*</packages>
+ </group>
+ <group>
<title>Communication Failure Detection</title>
<packages>org.apache.ignite.failure</packages>
</group>
[ignite] 19/19: Merge branch 'master' into gg-21368
Posted by am...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
amashenkov pushed a commit to branch gg-21368
in repository https://gitbox.apache.org/repos/asf/ignite.git
commit 18551758ad75af2fa3b11af5baf64c82ae8acd8f
Merge: 5417a6c bad5683
Author: Andrey V. Mashenkov <an...@gmail.com>
AuthorDate: Thu Jul 18 14:18:00 2019 +0300
Merge branch 'master' into gg-21368
bin/control.bat | 11 ++--
.../ml/tutorial/TutorialStepByStepExample.java | 1 +
.../ignite/internal/commandline/Command.java | 7 ---
.../internal/commandline/CommandHandler.java | 34 ++++++++++--
.../internal/commandline/CommonArgParser.java | 16 ++++++
.../ignite/internal/commandline/WalCommands.java | 53 +++++++++---------
.../internal/commandline/cache/CacheCommands.java | 6 ---
.../commandline/cache/ResetLostPartitions.java | 5 --
.../commandline/diagnostic/DiagnosticCommand.java | 6 ---
.../processors/cache/GridCacheProcessor.java | 63 ++++++++++++++++++----
.../dht/topology/GridDhtPartitionTopologyImpl.java | 23 ++++++--
.../GridCacheDatabaseSharedManager.java | 6 +--
.../cache/verify/VerifyBackupPartitionsTaskV2.java | 7 +--
.../visor/verify/VisorIdleVerifyTaskArg.java | 15 +++---
.../commandline/CommandHandlerParsingTest.java | 21 --------
...eWalFlushMultiNodeFailoverAbstractSelfTest.java | 59 ++++++++++----------
.../util/GridCommandHandlerAbstractTest.java | 4 +-
.../apache/ignite/util/GridCommandHandlerTest.java | 52 ++++++++++++++++--
.../visor/verify/ValidateIndexesClosure.java | 11 +++-
.../cache/IgniteCacheQueriesLoadTest1.java | 21 +++++---
modules/slf4j/pom.xml | 6 ---
.../ignite/console/websocket/WebSocketEvent.java | 5 ++
.../config/MailServiceAutoConfiguration.java | 4 +-
parent/pom.xml | 4 ++
24 files changed, 280 insertions(+), 160 deletions(-)
[ignite] 11/19: GG-20750: SQL: Fix performance drop with enabling
global SQL memory quota.
Posted by am...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
amashenkov pushed a commit to branch gg-21368
in repository https://gitbox.apache.org/repos/asf/ignite.git
commit 00e531cec9d41cae7ef267a32768e5820501e884
Author: Andrey V. Mashenkov <an...@gmail.com>
AuthorDate: Wed Jul 17 11:09:10 2019 +0300
GG-20750: SQL: Fix performance drop with enabling global SQL memory quota.
---
.../processors/query/h2/H2MemoryTracker.java | 3 +-
.../processors/query/h2/QueryMemoryManager.java | 72 +++++++++----
.../processors/query/h2/QueryMemoryTracker.java | 113 +++++++++++----------
.../oom/AbstractQueryMemoryTrackerSelfTest.java | 27 ++++-
...yMemoryTrackerWithQueryParallelismSelfTest.java | 36 +++++++
.../query/oom/QueryMemoryTrackerSelfTest.java | 11 ++
6 files changed, 183 insertions(+), 79 deletions(-)
diff --git a/modules/h2/src/main/java/org/apache/ignite/internal/processors/query/h2/H2MemoryTracker.java b/modules/h2/src/main/java/org/apache/ignite/internal/processors/query/h2/H2MemoryTracker.java
index e1d3aac..bbcc5f4 100644
--- a/modules/h2/src/main/java/org/apache/ignite/internal/processors/query/h2/H2MemoryTracker.java
+++ b/modules/h2/src/main/java/org/apache/ignite/internal/processors/query/h2/H2MemoryTracker.java
@@ -21,6 +21,8 @@ package org.apache.ignite.internal.processors.query.h2;
*/
public abstract class H2MemoryTracker implements AutoCloseable {
/**
+ * Reserve memory.
+ *
* @param size Memory to reserve in bytes.
*/
public abstract void reserve(long size);
@@ -30,6 +32,5 @@ public abstract class H2MemoryTracker implements AutoCloseable {
*
* @param size Memory to release in bytes.
*/
-
public abstract void release(long size);
}
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/QueryMemoryManager.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/QueryMemoryManager.java
index 58ae9e2..727ed35 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/QueryMemoryManager.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/QueryMemoryManager.java
@@ -17,6 +17,7 @@
package org.apache.ignite.internal.processors.query.h2;
import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.LongBinaryOperator;
import org.apache.ignite.IgniteLogger;
import org.apache.ignite.IgniteSystemProperties;
import org.apache.ignite.configuration.IgniteConfiguration;
@@ -37,6 +38,19 @@ public class QueryMemoryManager extends H2MemoryTracker {
*/
private static final long DFLT_MEMORY_RESERVATION_BLOCK_SIZE = 512 * KB;
+ /** */
+ static final LongBinaryOperator RELEASE_OP = new LongBinaryOperator() {
+ @Override public long applyAsLong(long prev, long x) {
+ long res = prev - x;
+
+ if (res < 0)
+ throw new IllegalStateException("Try to free more memory that ever be reserved: [" +
+ "reserved=" + prev + ", toFree=" + x + ']');
+
+ return res;
+ }
+ };
+
/**
* Default query memory limit.
*
@@ -48,6 +62,9 @@ public class QueryMemoryManager extends H2MemoryTracker {
/** Logger. */
private final IgniteLogger log;
+ /** */
+ private final LongBinaryOperator reserveOp;
+
/** Global query memory quota. */
//TODO GG-18629: it looks safe to make this configurable at runtime.
private final long globalQuota;
@@ -74,48 +91,38 @@ public class QueryMemoryManager extends H2MemoryTracker {
(long)(Runtime.getRuntime().maxMemory() * 0.6d));
}
- long dfltSqlQryMemoryLimit = Long.getLong(IgniteSystemProperties.IGNITE_DEFAULT_SQL_QUERY_MEMORY_LIMIT, 0);
+ long dfltMemLimit = Long.getLong(IgniteSystemProperties.IGNITE_DEFAULT_SQL_QUERY_MEMORY_LIMIT, 0);
- if (dfltSqlQryMemoryLimit == 0)
- dfltSqlQryMemoryLimit = globalQuota > 0 ? globalQuota / IgniteConfiguration.DFLT_QUERY_THREAD_POOL_SIZE : -1;
+ if (dfltMemLimit == 0)
+ dfltMemLimit = globalQuota > 0 ? globalQuota / IgniteConfiguration.DFLT_QUERY_THREAD_POOL_SIZE : -1;
this.blockSize = Long.getLong(IgniteSystemProperties.IGNITE_SQL_MEMORY_RESERVATION_BLOCK_SIZE, DFLT_MEMORY_RESERVATION_BLOCK_SIZE);
this.globalQuota = globalQuota;
- this.dfltSqlQryMemoryLimit = dfltSqlQryMemoryLimit;
+ this.dfltSqlQryMemoryLimit = dfltMemLimit;
+
+ this.reserveOp = new ReservationOp(globalQuota);
this.log = ctx.log(QueryMemoryManager.class);
}
/** {@inheritDoc} */
@Override public void reserve(long size) {
- assert size >= 0;
-
if (size == 0)
return; // Nothing to do.
- reserved.accumulateAndGet(size, (prev, x) -> {
- if (prev + x > globalQuota)
- throw new IgniteSQLException("SQL query run out of memory: Global quota exceeded.",
- IgniteQueryErrorCode.QUERY_OUT_OF_MEMORY);
+ assert size > 0;
- return prev + x;
- });
+ reserved.accumulateAndGet(size, reserveOp);
}
/** {@inheritDoc} */
@Override public void release(long size) {
- assert size >= 0;
-
if (size == 0)
return; // Nothing to do.
- reserved.accumulateAndGet(-size, (prev, x) -> {
- if (prev + x < 0)
- throw new IllegalStateException("Try to free more memory that ever be reserved: [" +
- "reserved=" + prev + ", toFree=" + x + ']');
+ assert size > 0;
- return prev + x;
- });
+ reserved.accumulateAndGet(size, RELEASE_OP);
}
/**
@@ -172,4 +179,29 @@ public class QueryMemoryManager extends H2MemoryTracker {
if (log.isDebugEnabled() && reserved.get() != 0)
log.debug("Potential memory leak in SQL processor. Some query cursors were not closed or forget to free memory.");
}
+
+ /** */
+ private static class ReservationOp implements LongBinaryOperator {
+ /** Operation result high bound.*/
+ private final long limit;
+
+ /**
+ * Constructor.
+ * @param limit Operation result high bound.
+ */
+ ReservationOp(long limit) {
+ this.limit = limit;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long applyAsLong(long prev, long x) {
+ long res = prev + x;
+
+ if (res > limit)
+ throw new IgniteSQLException("SQL query run out of memory: Global quota exceeded.",
+ IgniteQueryErrorCode.QUERY_OUT_OF_MEMORY);
+
+ return res;
+ }
+ }
}
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/QueryMemoryTracker.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/QueryMemoryTracker.java
index d0ff5c2..ea31930 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/QueryMemoryTracker.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/QueryMemoryTracker.java
@@ -16,8 +16,6 @@
package org.apache.ignite.internal.processors.query.h2;
-import java.util.concurrent.atomic.AtomicLongFieldUpdater;
-import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
import org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode;
import org.apache.ignite.internal.processors.query.IgniteSQLException;
import org.apache.ignite.internal.util.typedef.internal.S;
@@ -28,14 +26,6 @@ import org.apache.ignite.internal.util.typedef.internal.S;
* Track query memory usage and throws an exception if query tries to allocate memory over limit.
*/
public class QueryMemoryTracker extends H2MemoryTracker implements AutoCloseable {
- /** Resered field updater. */
- private static final AtomicLongFieldUpdater<QueryMemoryTracker> RESERVED_UPD =
- AtomicLongFieldUpdater.newUpdater(QueryMemoryTracker.class, "reserved");
-
- /** Closed flag updater. */
- private static final AtomicReferenceFieldUpdater<QueryMemoryTracker, Boolean> CLOSED_UPD =
- AtomicReferenceFieldUpdater.newUpdater(QueryMemoryTracker.class, Boolean.class, "closed");
-
/** Parent tracker. */
private final H2MemoryTracker parent;
@@ -46,13 +36,13 @@ public class QueryMemoryTracker extends H2MemoryTracker implements AutoCloseable
private final long blockSize;
/** Memory reserved on parent. */
- private volatile long reservedFromParent;
+ private long reservedFromParent;
/** Memory reserved by query. */
- private volatile long reserved;
+ private long reserved;
/** Close flag to prevent tracker reuse. */
- private volatile Boolean closed = Boolean.FALSE;
+ private Boolean closed = Boolean.FALSE;
/**
* Constructor.
@@ -71,40 +61,31 @@ public class QueryMemoryTracker extends H2MemoryTracker implements AutoCloseable
/** {@inheritDoc} */
@Override public void reserve(long size) {
- assert !closed && size >= 0;
-
if (size == 0)
return;
- long reserved0 = RESERVED_UPD.accumulateAndGet(this, size, (prev, x) -> {
- if (prev + x > maxMem) {
- throw new IgniteSQLException("SQL query run out of memory: Query quota exceeded.",
- IgniteQueryErrorCode.QUERY_OUT_OF_MEMORY);
- }
+ assert size > 0;
- return prev + x;
- });
+ synchronized (this) {
+ if (closed)
+ throw new IllegalStateException("Memory tracker has been closed concurrently.");
- if (parent != null && reserved0 > reservedFromParent) {
- synchronized (this) {
- assert !closed;
-
- if (reserved0 <= reservedFromParent)
- return;
-
- // If single block size is too small.
- long blockSize = Math.max(reserved0 - reservedFromParent, this.blockSize);
- // If we are too close to limit.
- blockSize = Math.min(blockSize, maxMem - reservedFromParent);
+ long reserved0 = reserve0(size);
+ if (parent != null && reserved0 > reservedFromParent) {
try {
+ // If single block size is too small.
+ long blockSize = Math.max(reserved0 - reservedFromParent, this.blockSize);
+ // If we are too close to limit.
+ blockSize = Math.min(blockSize, maxMem - reservedFromParent);
+
parent.reserve(blockSize);
reservedFromParent += blockSize;
}
catch (Throwable e) {
// Fallback if failed to reserve.
- RESERVED_UPD.addAndGet(this, -size);
+ release0(size);
throw e;
}
@@ -114,31 +95,50 @@ public class QueryMemoryTracker extends H2MemoryTracker implements AutoCloseable
/** {@inheritDoc} */
@Override public void release(long size) {
- assert size >= 0;
-
if (size == 0)
return;
- long reserved = RESERVED_UPD.accumulateAndGet(this, -size, (prev, x) -> {
- if (prev + x < 0)
- throw new IllegalStateException("Try to release more memory that were reserved: [" +
- "reserved=" + prev + ", toRelease=" + x + ']');
+ assert size > 0;
- return prev + x;
- });
+ synchronized (this) {
+ if (closed)
+ throw new IllegalStateException("Memory tracker has been closed concurrently.");
- assert !closed && reserved >= 0 || reserved == 0 : "Invalid reserved memory size:" + reserved;
+ long reserved = release0(size);
- // For now, won'tQ release memory to parent until tracker closed.
- /* if (parent != null && preAllocated - reserved >= 2 * blockSize) {
- synchronized (this) {
- if (preAllocated - reserved >= 2 * blockSize) {
- parent.release(blockSize);
+ assert reserved >= 0;
+ }
+ }
- preAllocated -= blockSize;
- }
- }
- }*/
+ /**
+ * @param size Memory to reserve in bytes.
+ * @return Reserved memory after release.
+ */
+ private long reserve0(long size) {
+ long res = reserved + size;
+
+ if (res > maxMem)
+ throw new IgniteSQLException("SQL query run out of memory: Query quota exceeded.",
+ IgniteQueryErrorCode.QUERY_OUT_OF_MEMORY);
+
+ return reserved = res;
+ }
+
+ /**
+ * Release reserved memory.
+ *
+ * @param size Memory to release in bytes.
+ * @return Reserved memory after release.
+ */
+ private long release0(long size) {
+
+ long res = reserved - size;
+
+ if (res < 0)
+ throw new IllegalStateException("Try to free more memory that ever be reserved: [" +
+ "reserved=" + reserved + ", toFree=" + size + ']');
+
+ return reserved = res;
}
/**
@@ -152,8 +152,13 @@ public class QueryMemoryTracker extends H2MemoryTracker implements AutoCloseable
@Override public void close() {
// It is not expected to be called concurrently with reserve\release.
// But query can be cancelled concurrently on query finish.
- if (CLOSED_UPD.compareAndSet(this, Boolean.FALSE, Boolean.TRUE)) {
- release(RESERVED_UPD.get(this));
+ synchronized (this) {
+ if (closed)
+ return;
+
+ closed = true;
+
+ release0(reserved);
if (parent != null)
parent.release(reservedFromParent);
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/oom/AbstractQueryMemoryTrackerSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/oom/AbstractQueryMemoryTrackerSelfTest.java
index c77ed15..2fc6231 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/oom/AbstractQueryMemoryTrackerSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/oom/AbstractQueryMemoryTrackerSelfTest.java
@@ -44,6 +44,7 @@ import org.h2.engine.Session;
import org.h2.expression.Expression;
import org.h2.result.H2BaseLocalResult;
import org.h2.result.LocalResult;
+import org.junit.Ignore;
import org.junit.Test;
import sun.reflect.generics.reflectiveObjects.NotImplementedException;
@@ -371,9 +372,10 @@ public abstract class AbstractQueryMemoryTrackerSelfTest extends GridCommonAbstr
/** Check GROUP BY operation on indexed col. */
@Test
+ @Ignore("https://ggsystems.atlassian.net/browse/GG-19071")
public void testQueryWithGroupByPrimaryKey() throws Exception {
- //TODO: GG-19071: make next test pass without hint.
- execQuery("select K.indexed, sum(K.id) from K USE INDEX (K_IDX) GROUP BY K.indexed", true);
+ //TODO: GG-19071: make next query use correct index (K_IDX instead of primary).
+ execQuery("select K.indexed, sum(K.id) from K GROUP BY K.indexed", true);
assertEquals(0, localResults.size());
}
@@ -396,7 +398,7 @@ public abstract class AbstractQueryMemoryTrackerSelfTest extends GridCommonAbstr
// Local result is quite small.
assertEquals(1, localResults.size());
- assertTrue(maxMem > localResults.get(0).memoryReserved());
+ assertTrue(maxMem > localResults.get(0).memoryReserved() + 1000);
assertTrue(BIG_TABLE_SIZE > localResults.get(0).getRowCount());
}
@@ -407,7 +409,7 @@ public abstract class AbstractQueryMemoryTrackerSelfTest extends GridCommonAbstr
// Local result is quite small.
assertEquals(1, localResults.size());
- assertTrue(maxMem > localResults.get(0).memoryReserved());
+ assertTrue(maxMem > localResults.get(0).memoryReserved() + 1000);
assertTrue(100 > localResults.get(0).getRowCount());
}
@@ -428,6 +430,7 @@ public abstract class AbstractQueryMemoryTrackerSelfTest extends GridCommonAbstr
"GROUP BY K.indexed ORDER BY a DESC", true);
assertEquals(1, localResults.size());
+ assertTrue(maxMem < localResults.get(0).memoryReserved() + 1000);
assertTrue(BIG_TABLE_SIZE > localResults.get(0).getRowCount());
}
@@ -447,6 +450,9 @@ public abstract class AbstractQueryMemoryTrackerSelfTest extends GridCommonAbstr
public void testQueryWithDistinctAndLowCardinality() throws Exception {
// Distinct on indexed column with small cardinality.
execQuery("select DISTINCT K.grp_indexed from K", false);
+
+ assertEquals(1, localResults.size());
+ assertEquals(100, localResults.get(0).getRowCount());
}
/** Check query failure with DISTINCT constraint. */
@@ -454,6 +460,9 @@ public abstract class AbstractQueryMemoryTrackerSelfTest extends GridCommonAbstr
public void testQueryWithDistinctAndHighCardinality() throws Exception {
// Distinct on indexed column with unique values.
checkQueryExpectOOM("select DISTINCT K.id from K", true);
+
+ assertEquals(1, localResults.size());
+ assertTrue(BIG_TABLE_SIZE > localResults.get(0).getRowCount());
}
/** Check HashJoin with large table. */
@@ -473,6 +482,16 @@ public abstract class AbstractQueryMemoryTrackerSelfTest extends GridCommonAbstr
}
}
+ /** Check Join with large table. */
+ @Test
+ public void testJoinWithLargeTable() throws Exception {
+ maxMem = 512 * KB;
+
+ execQuery("select * from T, K where T.id = K.grp_indexed", true);
+
+ assertEquals(0, localResults.size());
+ }
+
/** Check query failure due to global memory quota exceeded. */
@Test
public void testGlobalQuota() throws Exception {
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/oom/LocalQueryMemoryTrackerWithQueryParallelismSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/oom/LocalQueryMemoryTrackerWithQueryParallelismSelfTest.java
index acb1fc0..4569792 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/oom/LocalQueryMemoryTrackerWithQueryParallelismSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/oom/LocalQueryMemoryTrackerWithQueryParallelismSelfTest.java
@@ -339,4 +339,40 @@ public class LocalQueryMemoryTrackerWithQueryParallelismSelfTest extends Abstrac
assertEquals(1, localResults.size());
assertTrue(100 > localResults.get(0).getRowCount());
}
+
+ /** Check simple query with DISTINCT constraint. */
+ @Test
+ @Override public void testQueryWithDistinctAndLowCardinality() throws Exception {
+ // Distinct on indexed column with small cardinality.
+ execQuery("select DISTINCT K.grp_indexed from K", false);
+
+ assertEquals(5, localResults.size());
+ assertEquals(100, localResults.get(4).getRowCount());
+ }
+
+ /** Check query failure with DISTINCT constraint. */
+ @Test
+ @Override public void testQueryWithDistinctAndHighCardinality() throws Exception {
+ // Distinct on indexed column with unique values.
+ checkQueryExpectOOM("select DISTINCT K.id from K", true);
+
+ assertEquals(5, localResults.size());
+ assertFalse(localResults.stream().limit(4).anyMatch(r -> r.memoryReserved() + 1000 > maxMem));
+ assertTrue(BIG_TABLE_SIZE > localResults.get(4).getRowCount());
+ }
+
+ /** {@inheritDoc} */
+ @Test
+ @Override public void testLazyQueryWithGroupByThenSort() {
+ maxMem = MB / 2;
+
+ // Query failed on map side due too many groups.
+ checkQueryExpectOOM("select K.indexed, sum(K.grp) as a from K " +
+ "GROUP BY K.indexed ORDER BY a DESC", true);
+
+ // Result on reduce side.
+ assertEquals(1, localResults.size());
+ assertEquals(0, localResults.get(0).memoryReserved());
+ assertEquals(0, localResults.get(0).getRowCount());
+ }
}
\ No newline at end of file
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/oom/QueryMemoryTrackerSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/oom/QueryMemoryTrackerSelfTest.java
index 6c29013..f612df9 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/oom/QueryMemoryTrackerSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/oom/QueryMemoryTrackerSelfTest.java
@@ -176,6 +176,17 @@ public class QueryMemoryTrackerSelfTest extends AbstractQueryMemoryTrackerSelfTe
assertTrue(BIG_TABLE_SIZE > localResults.get(0).getRowCount());
}
+ /** Check simple query with DISTINCT constraint. */
+ @Test
+ @Override public void testQueryWithDistinctAndLowCardinality() throws Exception {
+ // Distinct on indexed column with small cardinality.
+ execQuery("select DISTINCT K.grp_indexed from K", false);
+
+ assertEquals(2, localResults.size());
+ assertEquals(100, localResults.get(0).getRowCount());
+ assertEquals(100, localResults.get(1).getRowCount());
+ }
+
/** {@inheritDoc} */
@Test
@Override public void testLazyQueryWithGroupByIndexedColAndDistinctAggregates() {
[ignite] 09/19: GG-21225 control.sh validate_indexes SQL Index
issue add information about group and cache id.
Posted by am...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
amashenkov pushed a commit to branch gg-21368
in repository https://gitbox.apache.org/repos/asf/ignite.git
commit f3017ea805e1334e2bdcf5a083aa830a28be3013
Author: ktkalenko <kt...@gridgain.com>
AuthorDate: Wed Jul 17 12:34:53 2019 +0300
GG-21225 control.sh validate_indexes SQL Index issue add information about group and cache id.
---
.../ignite/internal/visor/verify/ValidateIndexesClosure.java | 11 ++++++++++-
1 file changed, 10 insertions(+), 1 deletion(-)
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java b/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java
index 521ad4d..eb2e6f5 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java
@@ -737,7 +737,16 @@ public class ValidateIndexesClosure implements IgniteCallable<VisorValidateIndex
}
}
- String uniqueIdxName = "[cacheGroup=" + ctx.group().name() + ", cache=" + ctx.name() + ", idx=" + idx.getName() + "]";
+ CacheGroupContext group = ctx.group();
+
+ String uniqueIdxName = String.format(
+ "[cacheGroup=%s, cacheGroupId=%s, cache=%s, cacheId=%s, idx=%s]",
+ group.name(),
+ group.groupId(),
+ ctx.name(),
+ ctx.cacheId(),
+ idx.getName()
+ );
processedIndexes.incrementAndGet();