You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by ni...@apache.org on 2021/02/25 14:38:19 UTC
[ignite] branch master updated: IGNITE-14221 Hide SQL Constants
when IGNITE_TO_STRING_INCLUDE_SENSITIVE = false (#8821)
This is an automated email from the ASF dual-hosted git repository.
nizhikov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push:
new dad9c8c IGNITE-14221 Hide SQL Constants when IGNITE_TO_STRING_INCLUDE_SENSITIVE = false (#8821)
dad9c8c is described below
commit dad9c8c42326cd703359ce0b74129ebebbdbfb11
Author: Nikolay <ni...@apache.org>
AuthorDate: Thu Feb 25 17:38:00 2021 +0300
IGNITE-14221 Hide SQL Constants when IGNITE_TO_STRING_INCLUDE_SENSITIVE = false (#8821)
---
.../internal/processors/query/QueryUtils.java | 46 ++++
.../AbstractPerformanceStatisticsTest.java | 4 +-
.../processors/query/h2/IgniteH2Indexing.java | 46 +++-
.../processors/query/h2/sql/GridSqlConst.java | 5 +
.../processors/query/h2/sql/GridSqlDelete.java | 10 +-
.../processors/query/h2/sql/GridSqlInsert.java | 18 +-
.../processors/query/h2/sql/GridSqlJoin.java | 8 +-
.../processors/query/h2/sql/GridSqlMerge.java | 16 +-
.../processors/query/h2/sql/GridSqlQuery.java | 4 +-
.../processors/query/h2/sql/GridSqlSelect.java | 16 +-
.../processors/query/h2/sql/GridSqlUnion.java | 14 +-
.../processors/query/h2/sql/GridSqlUpdate.java | 12 +-
.../query/RemoveConstantsFromQueryTest.java | 235 +++++++++++++++++++++
.../IgniteBinaryCacheQueryTestSuite.java | 2 +
14 files changed, 393 insertions(+), 43 deletions(-)
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/QueryUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/QueryUtils.java
index bff1ae2..39a50f4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/QueryUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/QueryUtils.java
@@ -79,10 +79,12 @@ import org.jetbrains.annotations.Nullable;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_INDEXING_DISCOVERY_HISTORY_SIZE;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_SQL_SYSTEM_SCHEMA_NAME_IGNITE;
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_TO_STRING_INCLUDE_SENSITIVE;
import static org.apache.ignite.IgniteSystemProperties.getBoolean;
import static org.apache.ignite.IgniteSystemProperties.getInteger;
import static org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode.TOO_LONG_VALUE;
import static org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode.VALUE_SCALE_OUT_OF_RANGE;
+import static org.apache.ignite.internal.util.tostring.GridToStringBuilder.DFLT_TO_STRING_INCLUDE_SENSITIVE;
/**
* Utility methods for queries.
@@ -131,6 +133,29 @@ public class QueryUtils {
/** */
private static final Set<Class<?>> SQL_TYPES = createSqlTypes();
+ /** Default SQL delimeter. */
+ public static final char DEFAULT_DELIM = '\n';
+
+ /** Space SQL delimeter. */
+ public static final char SPACE_DELIM = ' ';
+
+ /** Setting to {@code true} enables writing sensitive information in {@code toString()} output. */
+ public static boolean INCLUDE_SENSITIVE =
+ IgniteSystemProperties.getBoolean(IGNITE_TO_STRING_INCLUDE_SENSITIVE, DFLT_TO_STRING_INCLUDE_SENSITIVE);
+
+ /**
+ * Enables {@link IgniteSystemProperties#IGNITE_TO_STRING_INCLUDE_SENSITIVE} mode for current thread.
+ * Note, setting {@code INCL_SENS_TL} to {@code false} will lead to generation of invalid SQL query.
+ * For example:<br>
+ * source query - "SELECT * FROM TBL WHERE name = 'Name'"<br>
+ * generated query - "SELECT * FROM TBL WHERE name = ?" - there is no parameter value in query.<br>
+ * It's a desired behaviour, because, when {@link IgniteSystemProperties#IGNITE_TO_STRING_INCLUDE_SENSITIVE} {@code = false}
+ * we want to filter out all sensitive data and those data can be sitting in SQL constants.
+ * Please, see {@code GridSqlConst#getSQL()}, {@code IgniteH2Indexing#sqlWithoutConst(GridSqlStatement)}.
+ */
+ public static final ThreadLocal<Boolean> INCLUDE_SENSITIVE_TL =
+ ThreadLocal.withInitial(() -> DFLT_TO_STRING_INCLUDE_SENSITIVE);
+
/**
* Creates SQL types set.
*
@@ -1611,6 +1636,27 @@ public class QueryUtils {
}
/**
+ * @return {@code True} if output sensitive data allowed.
+ */
+ public static boolean includeSensitive() {
+ return INCLUDE_SENSITIVE || INCLUDE_SENSITIVE_TL.get();
+ }
+
+ /**
+ * Return space character as an SQL delimeter in case {@link #includeSensitive()} is {@code false}
+ * to make output SQL one line. Default multiline SQL output looks ugly in system view and other view tool.
+ * See, {@code GridSqlConst} and {@code IgniteH2Indexing#sqlWithoutConst()} for details.
+ *
+ * @return Delimeter to use.
+ */
+ public static char delimeter() {
+ if (!includeSensitive())
+ return SPACE_DELIM;
+
+ return DEFAULT_DELIM;
+ }
+
+ /**
* Private constructor.
*/
private QueryUtils() {
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/performancestatistics/AbstractPerformanceStatisticsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/performancestatistics/AbstractPerformanceStatisticsTest.java
index bdb6a05..4a29989 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/performancestatistics/AbstractPerformanceStatisticsTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/performancestatistics/AbstractPerformanceStatisticsTest.java
@@ -71,7 +71,7 @@ public abstract class AbstractPerformanceStatisticsTest extends GridCommonAbstra
}
/** Starts collecting performance statistics. */
- protected static void startCollectStatistics() throws Exception {
+ public static void startCollectStatistics() throws Exception {
List<Ignite> grids = G.allGrids();
assertFalse(grids.isEmpty());
@@ -82,7 +82,7 @@ public abstract class AbstractPerformanceStatisticsTest extends GridCommonAbstra
}
/** Stops and reads collecting performance statistics. */
- protected static void stopCollectStatisticsAndRead(TestHandler... handlers) throws Exception {
+ public static void stopCollectStatisticsAndRead(TestHandler... handlers) throws Exception {
List<Ignite> grids = G.allGrids();
assertFalse(grids.isEmpty());
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
index 1f7a59a..06ca562 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
@@ -144,6 +144,7 @@ import org.apache.ignite.internal.processors.query.h2.opt.GridH2Table;
import org.apache.ignite.internal.processors.query.h2.opt.H2Row;
import org.apache.ignite.internal.processors.query.h2.opt.QueryContext;
import org.apache.ignite.internal.processors.query.h2.opt.QueryContextRegistry;
+import org.apache.ignite.internal.processors.query.h2.sql.GridSqlConst;
import org.apache.ignite.internal.processors.query.h2.sql.GridSqlStatement;
import org.apache.ignite.internal.processors.query.h2.twostep.GridMapQueryExecutor;
import org.apache.ignite.internal.processors.query.h2.twostep.GridReduceQueryExecutor;
@@ -711,7 +712,13 @@ public class IgniteH2Indexing implements GridQueryIndexing {
@SuppressWarnings({"unchecked", "Anonymous2MethodRef"})
private long streamQuery0(String qry, String schemaName, IgniteDataStreamer streamer, QueryParserResultDml dml,
final Object[] args) throws IgniteCheckedException {
- Long qryId = runningQryMgr.register(qry, GridCacheQueryType.SQL_FIELDS, schemaName, true, null);
+ Long qryId = runningQryMgr.register(
+ QueryUtils.INCLUDE_SENSITIVE ? qry : sqlWithoutConst(dml.statement()),
+ GridCacheQueryType.SQL_FIELDS,
+ schemaName,
+ true,
+ null
+ );
Exception failReason = null;
@@ -1030,7 +1037,7 @@ public class IgniteH2Indexing implements GridQueryIndexing {
IgniteQueryErrorCode.UNSUPPORTED_OPERATION);
}
- Long qryId = registerRunningQuery(qryDesc, qryParams, null);
+ Long qryId = registerRunningQuery(qryDesc, qryParams, null, null);
CommandResult res = null;
@@ -1213,7 +1220,7 @@ public class IgniteH2Indexing implements GridQueryIndexing {
) {
IndexingQueryFilter filter = (qryDesc.local() ? backupFilter(null, qryParams.partitions()) : null);
- Long qryId = registerRunningQuery(qryDesc, qryParams, cancel);
+ Long qryId = registerRunningQuery(qryDesc, qryParams, cancel, dml.statement());
Exception failReason = null;
@@ -1298,7 +1305,7 @@ public class IgniteH2Indexing implements GridQueryIndexing {
assert cancel != null;
// Register query.
- Long qryId = registerRunningQuery(qryDesc, qryParams, cancel);
+ Long qryId = registerRunningQuery(qryDesc, qryParams, cancel, select.statement());
try (TraceSurroundings ignored = MTC.support(ctx.tracing().create(SQL_CURSOR_OPEN, MTC.span()))) {
GridNearTxLocal tx = null;
@@ -1559,11 +1566,19 @@ public class IgniteH2Indexing implements GridQueryIndexing {
* @param qryDesc Query descriptor.
* @param qryParams Query parameters.
* @param cancel Query cancel state holder.
+ * @param stmnt Parsed statement.
* @return Id of registered query or {@code null} if query wasn't registered.
*/
- private Long registerRunningQuery(QueryDescriptor qryDesc, QueryParameters qryParams, GridQueryCancel cancel) {
+ private Long registerRunningQuery(
+ QueryDescriptor qryDesc,
+ QueryParameters qryParams,
+ GridQueryCancel cancel,
+ @Nullable GridSqlStatement stmnt
+ ) {
+ String qry = QueryUtils.INCLUDE_SENSITIVE || stmnt == null ? qryDesc.sql() : sqlWithoutConst(stmnt);
+
Long res = runningQryMgr.register(
- qryDesc.sql(),
+ qry,
GridCacheQueryType.SQL_FIELDS,
qryDesc.schemaName(),
qryDesc.local(),
@@ -1574,7 +1589,7 @@ public class IgniteH2Indexing implements GridQueryIndexing {
ctx.event().record(new SqlQueryExecutionEvent(
ctx.discovery().localNode(),
GridCacheQueryType.SQL_FIELDS.name() + " query execution.",
- qryDesc.sql(),
+ qry,
qryParams.arguments(),
ctx.security().enabled() ? ctx.security().securityContext().subject().id() : null));
}
@@ -1583,6 +1598,23 @@ public class IgniteH2Indexing implements GridQueryIndexing {
}
/**
+ * @param stmnt Statement to print.
+ * @return SQL query where constant replaced with '?' char.
+ * @see GridSqlConst#getSQL()
+ * @see QueryUtils#includeSensitive()
+ */
+ private String sqlWithoutConst(GridSqlStatement stmnt) {
+ QueryUtils.INCLUDE_SENSITIVE_TL.set(false);
+
+ try {
+ return stmnt.getSQL();
+ }
+ finally {
+ QueryUtils.INCLUDE_SENSITIVE_TL.set(true);
+ }
+ }
+
+ /**
* Check security access for caches.
*
* @param cacheIds Cache IDs.
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlConst.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlConst.java
index 7837fc8..0f33cd8 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlConst.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlConst.java
@@ -23,6 +23,8 @@ import org.h2.value.Value;
import org.h2.value.ValueBoolean;
import org.h2.value.ValueNull;
+import static org.apache.ignite.internal.processors.query.QueryUtils.includeSensitive;
+
/**
* Constant value.
*/
@@ -56,6 +58,9 @@ public class GridSqlConst extends GridSqlElement {
/** {@inheritDoc} */
@Override public String getSQL() {
+ if (!includeSensitive())
+ return "?";
+
return val.getSQL();
}
}
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlDelete.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlDelete.java
index 225de58..898824e 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlDelete.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlDelete.java
@@ -20,6 +20,8 @@ package org.apache.ignite.internal.processors.query.h2.sql;
import org.h2.util.StatementBuilder;
import org.h2.util.StringUtils;
+import static org.apache.ignite.internal.processors.query.QueryUtils.delimeter;
+
/** */
public class GridSqlDelete extends GridSqlStatement {
/** */
@@ -52,16 +54,18 @@ public class GridSqlDelete extends GridSqlStatement {
/** {@inheritDoc} */
@Override public String getSQL() {
+ char delim = delimeter();
+
StatementBuilder buff = new StatementBuilder(explain() ? "EXPLAIN " : "");
buff.append("DELETE")
- .append("\nFROM ")
+ .append(delim).append("FROM ")
.append(from.getSQL());
if (where != null)
- buff.append("\nWHERE ").append(StringUtils.unEnclose(where.getSQL()));
+ buff.append(delim).append("WHERE ").append(StringUtils.unEnclose(where.getSQL()));
if (limit != null)
- buff.append("\nLIMIT (").append(StringUtils.unEnclose(limit.getSQL())).append(')');
+ buff.append(delim).append("LIMIT (").append(StringUtils.unEnclose(limit.getSQL())).append(')');
return buff.toString();
}
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlInsert.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlInsert.java
index 07b36e6..4570943 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlInsert.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlInsert.java
@@ -20,6 +20,8 @@ package org.apache.ignite.internal.processors.query.h2.sql;
import java.util.List;
import org.h2.util.StatementBuilder;
+import static org.apache.ignite.internal.processors.query.QueryUtils.delimeter;
+
/** */
public class GridSqlInsert extends GridSqlStatement {
/** */
@@ -48,18 +50,20 @@ public class GridSqlInsert extends GridSqlStatement {
/** {@inheritDoc} */
@Override public String getSQL() {
+ char delim = delimeter();
+
StatementBuilder buff = new StatementBuilder(explain() ? "EXPLAIN " : "");
buff.append("INSERT")
- .append("\nINTO ")
+ .append(delim).append("INTO ")
.append(into.getSQL())
.append('(');
for (GridSqlColumn col : cols) {
- buff.appendExceptFirst(", ");
- buff.append('\n')
+ buff.appendExceptFirst(",");
+ buff.append(delim)
.append(col.getSQL());
}
- buff.append("\n)\n");
+ buff.append(delim).append(')').append(delim);
if (direct)
buff.append("DIRECT ");
@@ -68,11 +72,11 @@ public class GridSqlInsert extends GridSqlStatement {
buff.append("SORTED ");
if (!rows.isEmpty()) {
- buff.append("VALUES\n");
+ buff.append("VALUES").append(delim);
StatementBuilder valuesBuff = new StatementBuilder();
for (GridSqlElement[] row : rows()) {
- valuesBuff.appendExceptFirst(",\n");
+ valuesBuff.appendExceptFirst("," + delim);
StatementBuilder rowBuff = new StatementBuilder("(");
for (GridSqlElement e : row) {
rowBuff.appendExceptFirst(", ");
@@ -84,7 +88,7 @@ public class GridSqlInsert extends GridSqlStatement {
buff.append(valuesBuff.toString());
}
else
- buff.append('\n')
+ buff.append(delim)
.append(qry.getSQL());
return buff.toString();
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlJoin.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlJoin.java
index a68e836..0df7c22 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlJoin.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlJoin.java
@@ -22,6 +22,8 @@ import org.h2.util.StatementBuilder;
import org.h2.util.StringUtils;
import org.jetbrains.annotations.Nullable;
+import static org.apache.ignite.internal.processors.query.QueryUtils.delimeter;
+
/**
* Join of two tables or subqueries.
*/
@@ -102,15 +104,17 @@ public class GridSqlJoin extends GridSqlElement {
/** {@inheritDoc} */
@Override public String getSQL() {
+ char delim = delimeter();
+
StatementBuilder buff = new StatementBuilder();
buff.append(leftTable().getSQL());
- buff.append(leftOuter ? " \n LEFT OUTER JOIN " : " \n INNER JOIN ");
+ buff.append(' ').append(delim).append(leftOuter ? " LEFT OUTER JOIN " : " INNER JOIN ");
buff.append(rightTable().getSQL());
- buff.append(" \n ON ").append(StringUtils.unEnclose(on().getSQL()));
+ buff.append(' ').append(delim).append(" ON ").append(StringUtils.unEnclose(on().getSQL()));
return buff.toString();
}
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlMerge.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlMerge.java
index 5b348bc..7c43efb 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlMerge.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlMerge.java
@@ -20,6 +20,8 @@ package org.apache.ignite.internal.processors.query.h2.sql;
import java.util.List;
import org.h2.util.StatementBuilder;
+import static org.apache.ignite.internal.processors.query.QueryUtils.delimeter;
+
/** */
public class GridSqlMerge extends GridSqlStatement {
/** */
@@ -36,24 +38,26 @@ public class GridSqlMerge extends GridSqlStatement {
/** {@inheritDoc} */
@Override public String getSQL() {
+ char delim = delimeter();
+
StatementBuilder buff = new StatementBuilder(explain() ? "EXPLAIN " : "");
buff.append("MERGE INTO ")
.append(into.getSQL())
- .append("(");
+ .append('(');
for (GridSqlColumn col : cols) {
buff.appendExceptFirst(", ");
- buff.append('\n')
+ buff.append(delim)
.append(col.getSQL());
}
- buff.append("\n)\n");
+ buff.append(delim).append(')').append(delim);
if (!rows.isEmpty()) {
- buff.append("VALUES\n");
+ buff.append("VALUES").append(delim);
StatementBuilder valuesBuff = new StatementBuilder();
for (GridSqlElement[] row : rows()) {
- valuesBuff.appendExceptFirst(",\n");
+ valuesBuff.appendExceptFirst("," + delim);
StatementBuilder rowBuff = new StatementBuilder("(");
for (GridSqlElement e : row) {
rowBuff.appendExceptFirst(", ");
@@ -65,7 +69,7 @@ public class GridSqlMerge extends GridSqlStatement {
buff.append(valuesBuff.toString());
}
else
- buff.append('\n')
+ buff.append(delim)
.append(qry.getSQL());
return buff.toString();
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuery.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuery.java
index 192808d..a82c83e 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuery.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuery.java
@@ -22,6 +22,8 @@ import java.util.List;
import org.h2.util.StatementBuilder;
import org.h2.util.StringUtils;
+import static org.apache.ignite.internal.processors.query.QueryUtils.delimeter;
+
/**
* SQL Query AST.
*/
@@ -145,7 +147,7 @@ public abstract class GridSqlQuery extends GridSqlStatement implements GridSqlAs
*/
protected void getSortLimitSQL(StatementBuilder buff) {
if (!sort.isEmpty()) {
- buff.append("\nORDER BY ");
+ buff.append(delimeter()).append("ORDER BY ");
int visibleCols = visibleColumns();
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlSelect.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlSelect.java
index 93c1b8e..b3f0645 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlSelect.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlSelect.java
@@ -24,6 +24,8 @@ import java.util.Set;
import org.h2.util.StatementBuilder;
import org.h2.util.StringUtils;
+import static org.apache.ignite.internal.processors.query.QueryUtils.delimeter;
+
/**
* Plain SELECT query.
*/
@@ -142,6 +144,8 @@ public class GridSqlSelect extends GridSqlQuery {
/** {@inheritDoc} */
@Override public String getSQL() {
+ char delim = delimeter();
+
StatementBuilder buff = new StatementBuilder(explain() ? "EXPLAIN SELECT" : "SELECT");
if (distinct)
@@ -149,18 +153,18 @@ public class GridSqlSelect extends GridSqlQuery {
for (GridSqlAst expression : columns(true)) {
buff.appendExceptFirst(",");
- buff.append('\n');
+ buff.append(delim);
buff.append(expression.getSQL());
}
if (from != null)
- buff.append("\nFROM ").append(from.getSQL());
+ buff.append(delim).append("FROM ").append(from.getSQL());
if (where != null)
- buff.append("\nWHERE ").append(StringUtils.unEnclose(where.getSQL()));
+ buff.append(delim).append("WHERE ").append(StringUtils.unEnclose(where.getSQL()));
if (grpCols != null) {
- buff.append("\nGROUP BY ");
+ buff.append(delim).append("GROUP BY ");
buff.resetCount();
@@ -172,7 +176,7 @@ public class GridSqlSelect extends GridSqlQuery {
}
if (havingCol >= 0) {
- buff.append("\nHAVING ");
+ buff.append(delim).append("HAVING ");
addAlias(buff, cols.get(havingCol));
}
@@ -180,7 +184,7 @@ public class GridSqlSelect extends GridSqlQuery {
getSortLimitSQL(buff);
if (isForUpdate)
- buff.append("\nFOR UPDATE");
+ buff.append(delim).append("FOR UPDATE");
return buff.toString();
}
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlUnion.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlUnion.java
index 71e361c..6095aa5 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlUnion.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlUnion.java
@@ -21,6 +21,8 @@ import javax.cache.CacheException;
import org.h2.command.dml.SelectUnion;
import org.h2.util.StatementBuilder;
+import static org.apache.ignite.internal.processors.query.QueryUtils.delimeter;
+
/**
* Select query with UNION.
*/
@@ -103,25 +105,27 @@ public class GridSqlUnion extends GridSqlQuery {
/** {@inheritDoc} */
@Override public String getSQL() {
- StatementBuilder buff = new StatementBuilder(explain() ? "EXPLAIN \n" : "");
+ char delim = delimeter();
+
+ StatementBuilder buff = new StatementBuilder(explain() ? "EXPLAIN " + delim : "");
buff.append('(').append(left.getSQL()).append(')');
switch (unionType()) {
case UNION_ALL:
- buff.append("\nUNION ALL\n");
+ buff.append(delim).append("UNION ALL").append(delim);
break;
case UNION:
- buff.append("\nUNION\n");
+ buff.append(delim).append("UNION").append(delim);
break;
case INTERSECT:
- buff.append("\nINTERSECT\n");
+ buff.append(delim).append("INTERSECT").append(delim);
break;
case EXCEPT:
- buff.append("\nEXCEPT\n");
+ buff.append(delim).append("EXCEPT").append(delim);
break;
default:
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlUpdate.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlUpdate.java
index 5393e4a..1c68958 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlUpdate.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlUpdate.java
@@ -22,6 +22,8 @@ import java.util.LinkedHashMap;
import org.h2.util.StatementBuilder;
import org.h2.util.StringUtils;
+import static org.apache.ignite.internal.processors.query.QueryUtils.delimeter;
+
/** */
public class GridSqlUpdate extends GridSqlStatement {
/** */
@@ -82,22 +84,24 @@ public class GridSqlUpdate extends GridSqlStatement {
/** {@inheritDoc} */
@Override public String getSQL() {
+ char delim = delimeter();
+
StatementBuilder buff = new StatementBuilder(explain() ? "EXPLAIN " : "");
buff.append("UPDATE ")
.append(target.getSQL())
- .append("\nSET\n");
+ .append(delim).append("SET").append(delim);
for (GridSqlColumn c : cols) {
GridSqlElement e = set.get(c.columnName());
- buff.appendExceptFirst(",\n ");
+ buff.appendExceptFirst("," + delim + " ");
buff.append(c.columnName()).append(" = ").append(e != null ? e.getSQL() : "DEFAULT");
}
if (where != null)
- buff.append("\nWHERE ").append(StringUtils.unEnclose(where.getSQL()));
+ buff.append(delim).append("WHERE ").append(StringUtils.unEnclose(where.getSQL()));
if (limit != null)
- buff.append("\nLIMIT ").append(StringUtils.unEnclose(limit.getSQL()));
+ buff.append(delim).append("LIMIT ").append(StringUtils.unEnclose(limit.getSQL()));
return buff.toString();
}
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/RemoveConstantsFromQueryTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/RemoveConstantsFromQueryTest.java
new file mode 100644
index 0000000..1f96727
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/RemoveConstantsFromQueryTest.java
@@ -0,0 +1,235 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.query;
+
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.regex.Pattern;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteSystemProperties;
+import org.apache.ignite.cache.query.SqlFieldsQuery;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.events.EventType;
+import org.apache.ignite.events.SqlQueryExecutionEvent;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.cache.index.AbstractIndexingCommonTest;
+import org.apache.ignite.internal.processors.cache.query.GridCacheQueryType;
+import org.apache.ignite.internal.processors.performancestatistics.AbstractPerformanceStatisticsTest;
+import org.apache.ignite.internal.util.lang.GridTuple3;
+import org.apache.ignite.internal.util.typedef.F;
+import org.junit.Test;
+
+import static org.apache.ignite.cluster.ClusterState.ACTIVE;
+import static org.apache.ignite.internal.processors.performancestatistics.AbstractPerformanceStatisticsTest.cleanPerformanceStatisticsDir;
+import static org.apache.ignite.internal.processors.performancestatistics.AbstractPerformanceStatisticsTest.startCollectStatistics;
+import static org.apache.ignite.internal.processors.performancestatistics.AbstractPerformanceStatisticsTest.stopCollectStatisticsAndRead;
+import static org.apache.ignite.internal.util.tostring.GridToStringBuilder.DFLT_TO_STRING_INCLUDE_SENSITIVE;
+
+/**
+ * Tests check that with {@link IgniteSystemProperties#IGNITE_TO_STRING_INCLUDE_SENSITIVE} == false literals from query
+ * will be deleted from query before logging it to history, events, profiling tool.
+ */
+public class RemoveConstantsFromQueryTest extends AbstractIndexingCommonTest {
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+ return super.getConfiguration(igniteInstanceName).setIncludeEventTypes(EventType.EVT_SQL_QUERY_EXECUTION);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ cleanPerformanceStatisticsDir();
+
+ QueryUtils.INCLUDE_SENSITIVE = false;
+ }
+
+ @Override protected void afterTestsStopped() throws Exception {
+ super.afterTestsStopped();
+
+ QueryUtils.INCLUDE_SENSITIVE = DFLT_TO_STRING_INCLUDE_SENSITIVE;
+ }
+
+ /** */
+ @Test
+ public void testConstantRemoved() throws Exception {
+ IgniteEx ignite = startGrid(0);
+
+ ignite.cluster().state(ACTIVE);
+
+ cleanPerformanceStatisticsDir();
+
+ startCollectStatistics();
+
+ String john = "John Connor";
+ String sarah = "Sarah Connor";
+
+ // SQL query, Regexp to find, Constant that should be removed.
+ List<GridTuple3<String, String, String>> qries = Arrays.asList(
+ F.t("CREATE TABLE TST(id INTEGER PRIMARY KEY, name VARCHAR, age integer)", null, null),
+
+ F.t("CREATE TABLE TST2(id INTEGER PRIMARY KEY, name VARCHAR, age integer)", null, null),
+
+ F.t("INSERT INTO TST(id, name, age) VALUES(1, '" + john + "', 16)",
+ "INSERT INTO .*TST.*VALUES.*",
+ john),
+
+ F.t("INSERT INTO TST SELECT id, name, age FROM TST2 WHERE name = 'John Connor'",
+ "INSERT INTO .*TST.*SELECT.*FROM .*TST2 WHERE.*",
+ john),
+
+ F.t("UPDATE TST SET name = '" + sarah + "' WHERE id = 1",
+ "UPDATE .*TST SET NAME.*WHERE ID.*",
+ sarah),
+
+ F.t("DELETE FROM TST WHERE name = '" + sarah + "'",
+ "DELETE FROM .*TST WHERE NAME = ?",
+ sarah),
+
+ F.t("SELECT * FROM TST WHERE name = '" + sarah + "'",
+ "SELECT .* FROM .*TST.*WHERE .*NAME = ?",
+ sarah),
+
+ F.t("SELECT * FROM TST WHERE name = SUBSTR('" + sarah + "', 0, 2)",
+ "SELECT .* FROM .*TST.*WHERE .*NAME = ?",
+ sarah.substring(0, 2)),
+
+ F.t("SELECT * FROM TST GROUP BY id HAVING name = '" + john + "'",
+ "SELECT .* FROM .*TST.*GROUP BY .*ID HAVING .*NAME = ?",
+ john),
+
+ F.t("SELECT * FROM TST GROUP BY id HAVING name = '" + sarah + "' UNION " +
+ "SELECT * FROM TST GROUP BY id HAVING name = '" + john + "'",
+ ".*SELECT .* FROM .*TST .* GROUP BY .*ID HAVING .*NAME = ?.* UNION " +
+ ".*SELECT .* FROM .*TST .* GROUP BY .*ID HAVING .*NAME = ?.*",
+ sarah),
+
+ F.t("SELECT CONCAT(name, '" + sarah + "') FROM TST",
+ "SELECT CONCAT(.*) FROM .*TST",
+ sarah),
+
+ F.t("ALTER TABLE TST ADD COLUMN department VARCHAR(200)", null, null),
+
+ F.t("DROP TABLE TST", null, null),
+
+ F.t("KILL SERVICE 'my_service'", null, null)
+ );
+
+ AtomicReference<String> lastQryFromEvt = new AtomicReference<>();
+
+ ignite.events().localListen(evt -> {
+ assertTrue(evt instanceof SqlQueryExecutionEvent);
+
+ lastQryFromEvt.set(((SqlQueryExecutionEvent)evt).text());
+
+ return true;
+ }, EventType.EVT_SQL_QUERY_EXECUTION);
+
+ for (GridTuple3<String, String, String> qry : qries) {
+ execSql(ignite, qry.get1());
+
+ String expHist = qry.get2() == null ? qry.get1() : qry.get2();
+ String qryFromEvt = lastQryFromEvt.get();
+
+ List<List<?>> hist = execSql(ignite, "SELECT SQL FROM SYS.SQL_QUERIES_HISTORY WHERE SQL = ?", qryFromEvt);
+
+ assertNotNull(hist);
+ assertEquals(1, hist.size());
+
+ String qryFromHist = hist.get(0).get(0).toString();
+
+ if (qry.get2() != null) {
+ Pattern ptrn = Pattern.compile(qry.get2());
+
+ assertTrue(qry.get2() + " should match " + qryFromHist, ptrn.matcher(qryFromHist).find());
+ assertTrue(qry.get2() + " should match " + qryFromEvt, ptrn.matcher(qryFromEvt).find());
+ }
+ else {
+ assertEquals(qryFromHist, expHist);
+ assertEquals(qryFromEvt, expHist);
+ }
+
+ if (qry.get3() != null) {
+ assertFalse(qryFromHist.contains(qry.get3()));
+ assertFalse(qryFromEvt.contains(qry.get3()));
+ }
+ }
+
+ Set<String> qriesFromStats = new HashSet<>();
+
+ stopCollectStatisticsAndRead(new AbstractPerformanceStatisticsTest.TestHandler() {
+ @Override public void query(
+ UUID nodeId,
+ GridCacheQueryType type,
+ String text,
+ long id,
+ long startTime,
+ long duration,
+ boolean success
+ ) {
+ qriesFromStats.add(text);
+ }
+ });
+
+ // `SELECT ... WHERE name = 'X'` and `SELECT ... WHERE name = SUBSTR(...)` produces the same query text
+ // and we have one extra query `SELECT sql FROM SYS.SQL_QUERIES_HISTORY WHERE sql = ?`
+ // so the sizes of two collection should be equal.
+ assertEquals(qries.size(), qriesFromStats.size());
+
+ assertTrue(qriesFromStats.contains("SELECT SQL FROM SYS.SQL_QUERIES_HISTORY WHERE SQL = ?1"));
+
+ for (GridTuple3<String, String, String> qry : qries) {
+ boolean found = false;
+
+ for (String qryFromStat : qriesFromStats) {
+ if (qry.get2() != null) {
+ if (!Pattern.compile(qry.get2()).matcher(qryFromStat).find())
+ continue;
+ }
+ else if (!qryFromStat.equals(qry.get1()))
+ continue;
+
+ found = qry.get3() == null || !qryFromStat.contains(qry.get3());
+
+ if (found)
+ break;
+ }
+
+ assertTrue(qry.get1() + " should be in statistics", found);
+ }
+ }
+
+ /**
+ * @param ignite Ignite.
+ * @param sql Sql.
+ * @param args Args.
+ */
+ private List<List<?>> execSql(Ignite ignite, String sql, Object... args) {
+ IgniteCache<?, ?> cache = ignite.getOrCreateCache(DEFAULT_CACHE_NAME);
+
+ SqlFieldsQuery qry = new SqlFieldsQuery(sql);
+
+ if (args != null && args.length > 0)
+ qry.setArgs(args);
+
+ return cache.query(qry).getAll();
+ }
+}
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteBinaryCacheQueryTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteBinaryCacheQueryTestSuite.java
index 3c2a5d3..95800b8 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteBinaryCacheQueryTestSuite.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteBinaryCacheQueryTestSuite.java
@@ -225,6 +225,7 @@ import org.apache.ignite.internal.processors.query.KillQueryFromNeighbourTest;
import org.apache.ignite.internal.processors.query.KillQueryOnClientDisconnectTest;
import org.apache.ignite.internal.processors.query.KillQueryTest;
import org.apache.ignite.internal.processors.query.MultipleStatementsSqlQuerySelfTest;
+import org.apache.ignite.internal.processors.query.RemoveConstantsFromQueryTest;
import org.apache.ignite.internal.processors.query.RunningQueriesTest;
import org.apache.ignite.internal.processors.query.SqlFieldTypeValidationOnKeyValueInsertTest;
import org.apache.ignite.internal.processors.query.SqlFieldTypeValidationTypesTest;
@@ -628,6 +629,7 @@ import org.junit.runners.Suite;
//Query history.
SqlQueryHistorySelfTest.class,
SqlQueryHistoryFromClientSelfTest.class,
+ RemoveConstantsFromQueryTest.class,
SqlIncompatibleDataTypeExceptionTest.class,