You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@asterixdb.apache.org by AsterixDB Code Review <do...@asterix-gerrit.ics.uci.edu> on 2023/07/01 11:51:54 UTC
Change in asterixdb[master]: [NO ISSUE][HYR] Added switch operator
From Sushrut Borkar <su...@gmail.com>:
Sushrut Borkar has uploaded this change for review. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17627 )
Change subject: [NO ISSUE][HYR] Added switch operator
......................................................................
[NO ISSUE][HYR] Added switch operator
- user model changes: no
- storage format changes: no
- interface changes: yes
details:
- The switch operator propagates each input tuple to one
or more output branches based on an output mapping given
at compile time.
Change-Id: If3a9c86fadb7c869bc97e4084d0ebaa43a7ec111
---
M hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/CCConfig.java
M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
M asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
A asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cache/QueryCacheKey.java
M hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/HashJoinExpressionAnnotation.java
A asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cache/QueryCacheValue.java
M asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/annotation/ExcludeFromSelectStarAnnotation.java
M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/LangExecutionUtil.java
M asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/RebalanceApiServlet.java
M hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/BroadcastExpressionAnnotation.java
M asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/aggregate-sql-sugar/avg_mixed/avg_mixed.5.query.sqlpp
A asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/SqlppExecutionWithCacheTest.java
M asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceRequestParameters.java
M asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/SqlppCompilationProvider.java
M asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/SessionConfig.java
M asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/upsert/upsert-case-returning/upsert-case-returning.3.query.sqlpp
M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
M hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java
M asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlCompatRewriteVisitor.java
A asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cache/QueryCache.java
M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/CcApplicationContext.java
M hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/PredicateCardinalityAnnotation.java
M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
M asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
M asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/SqlppAstPrintVisitor.java
M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/annotations/ExternalSubpathAnnotation.java
M asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/QueryPrintVisitor.java
M hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/JoinProductivityAnnotation.java
M asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/insert-return-records/insert-return-records.3.query.sqlpp
29 files changed, 559 insertions(+), 14 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/27/17627/1
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/SqlppCompilationProvider.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/SqlppCompilationProvider.java
index 72807a2..a50c36fa 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/SqlppCompilationProvider.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/SqlppCompilationProvider.java
@@ -87,7 +87,8 @@
CompilerProperties.COMPILER_CBO_KEY, CompilerProperties.COMPILER_CBO_TEST_KEY,
CompilerProperties.COMPILER_FORCE_JOIN_ORDER_KEY, CompilerProperties.COMPILER_QUERY_PLAN_SHAPE_KEY,
CompilerProperties.COMPILER_MIN_MEMORY_ALLOCATION_KEY, CompilerProperties.COMPILER_COLUMN_FILTER_KEY,
- CompilerProperties.COMPILER_BATCH_LOOKUP_KEY, FunctionUtil.IMPORT_PRIVATE_FUNCTIONS,
+ CompilerProperties.COMPILER_BATCH_LOOKUP_KEY, CompilerProperties.COMPILER_QUERYCACHE_BYPASS_KEY,
+ CompilerProperties.COMPILER_QUERYCACHE_CLEAR_KEY, FunctionUtil.IMPORT_PRIVATE_FUNCTIONS,
FuzzyUtils.SIM_FUNCTION_PROP_NAME, FuzzyUtils.SIM_THRESHOLD_PROP_NAME,
StartFeedStatement.WAIT_FOR_COMPLETION, FeedActivityDetails.FEED_POLICY_NAME,
FeedActivityDetails.COLLECT_LOCATIONS, SqlppQueryRewriter.INLINE_WITH_OPTION,
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/SessionConfig.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/SessionConfig.java
index 4e294ec..78f2ff9 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/SessionConfig.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/SessionConfig.java
@@ -130,6 +130,11 @@
public static final String FORMAT_INDENT_JSON = "indent-json";
/**
+ * Whether to bypass the query plan cache.
+ */
+ public static final String QUERYCACHE_BYPASS = "querycache-bypass";
+
+ /**
* Format flag: quote records in the results array.
*/
public static final String FORMAT_QUOTE_RECORD = "quote-record";
@@ -187,6 +192,17 @@
this.clientType = ClientType.ASTERIX;
}
+ public SessionConfig(SessionConfig config) {
+ this.fmt = config.fmt;
+ this.optimize = config.optimize;
+ this.executeQuery = config.executeQuery;
+ this.generateJobSpec = config.generateJobSpec;
+ this.flags = new HashMap<>(config.flags);
+ this.planFormat = config.planFormat;
+ this.clientType = config.clientType;
+ this.maxWarnings = config.maxWarnings;
+ }
+
/**
* Retrieve the client type for this execution.
*/
@@ -291,4 +307,25 @@
Boolean value = flags.get(flag);
return value == null ? false : value.booleanValue();
}
+
+ @Override
+ public boolean equals(Object o) {
+ if (!(o instanceof SessionConfig)) {
+ return false;
+ }
+ if (o == this) {
+ return true;
+ }
+ SessionConfig other = (SessionConfig) o;
+ return Objects.equals(other.fmt, fmt) && Objects.equals(other.optimize, optimize)
+ && Objects.equals(other.executeQuery, executeQuery)
+ && Objects.equals(other.generateJobSpec, generateJobSpec) && Objects.equals(other.flags, flags)
+ && Objects.equals(other.planFormat, planFormat) && Objects.equals(other.clientType, clientType)
+ && Objects.equals(other.maxWarnings, maxWarnings);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(fmt, optimize, executeQuery, generateJobSpec, flags, planFormat, clientType, maxWarnings);
+ }
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
index afdcee4..da6a00f 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
@@ -516,6 +516,14 @@
}
}
+ public String generateExpressionTreeString(IReturningStatement statement) throws CompilationException {
+ final StringWriter stringWriter = new StringWriter();
+ try (PrintWriter writer = new PrintWriter(stringWriter)) {
+ statement.accept(astPrintVisitorFactory.createLangVisitor(writer), 0);
+ return stringWriter.toString();
+ }
+ }
+
private void generateRewrittenExpressionTree(IReturningStatement statement) throws CompilationException {
final StringWriter stringWriter = new StringWriter();
try (PrintWriter writer = new PrintWriter(stringWriter)) {
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceRequestParameters.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceRequestParameters.java
index b6913e4..adcd374 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceRequestParameters.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceRequestParameters.java
@@ -64,6 +64,7 @@
CLIENT_TYPE("client-type"),
DATAVERSE("dataverse"),
PRETTY("pretty"),
+ BYPASS_CACHE("bypass_cache"),
MODE("mode"),
TIMEOUT("timeout"),
PLAN_FORMAT("plan-format"),
@@ -132,6 +133,7 @@
private Map<String, String> optionalParams = null;
private Map<String, JsonNode> statementParams = null;
private boolean pretty = false;
+ private boolean bypassCache = false;
private boolean expressionTree = false;
private boolean parseOnly = false; // don't execute; simply check for syntax correctness and named parameters.
private boolean compileOnly = false; // don't execute; compile only.
@@ -201,6 +203,14 @@
this.pretty = pretty;
}
+ public boolean isBypassCache() {
+ return bypassCache;
+ }
+
+ public void setBypassCache(boolean bypassCache) {
+ this.bypassCache = bypassCache;
+ }
+
public String getClientContextID() {
return clientContextID;
}
@@ -382,6 +392,7 @@
object.put("path", path);
object.put("statement", statement != null ? JSONUtil.escape(new StringBuilder(), statement).toString() : null);
object.put("pretty", pretty);
+ object.put("bypassCache", bypassCache);
object.put("mode", mode.getName());
object.put("clientContextID", clientContextID);
object.put("clientType", clientType.toString());
@@ -472,6 +483,7 @@
setMaxWarnings(parseLong(req, Parameter.MAX_WARNINGS.str(), valGetter, getMaxWarnings()));
setPretty(parseBoolean(req, Parameter.PRETTY.str(), valGetter, isPretty()));
+ setBypassCache(parseBoolean(req, Parameter.BYPASS_CACHE.str(), valGetter, isBypassCache()));
setExpressionTree(parseBoolean(req, Parameter.EXPRESSION_TREE.str(), valGetter, isExpressionTree()));
setRewrittenExpressionTree(
parseBoolean(req, Parameter.REWRITTEN_EXPRESSION_TREE.str(), valGetter, isRewrittenExpressionTree()));
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
index e14c185..f5868bb 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
@@ -495,6 +495,7 @@
sessionConfig.set(SessionConfig.OOB_OPTIMIZED_LOGICAL_PLAN, param.isOptimizedLogicalPlan());
sessionConfig.set(SessionConfig.OOB_HYRACKS_JOB, param.isJob());
sessionConfig.set(SessionConfig.FORMAT_INDENT_JSON, param.isPretty());
+ sessionConfig.set(SessionConfig.QUERYCACHE_BYPASS, param.isBypassCache());
sessionConfig.set(SessionConfig.FORMAT_QUOTE_RECORD,
format != SessionConfig.OutputFormat.CLEAN_JSON && format != SessionConfig.OutputFormat.LOSSLESS_JSON
&& format != SessionConfig.OutputFormat.LOSSLESS_ADM_JSON);
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/RebalanceApiServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/RebalanceApiServlet.java
index 58a37bd..c188e65 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/RebalanceApiServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/RebalanceApiServlet.java
@@ -37,6 +37,7 @@
import java.util.concurrent.Future;
import org.apache.asterix.app.active.ActiveNotificationHandler;
+import org.apache.asterix.app.cc.CcApplicationContext;
import org.apache.asterix.common.api.IMetadataLockManager;
import org.apache.asterix.common.dataflow.ICcApplicationContext;
import org.apache.asterix.common.metadata.DataverseName;
@@ -182,6 +183,9 @@
// Sets the content type.
HttpUtil.setContentType(response, HttpUtil.ContentType.APPLICATION_JSON, StandardCharsets.UTF_8);
+ // Clear the query plan cache
+ ((CcApplicationContext) appCtx).getQueryCache().clear();
+
if (datasetName == null) {
// Rebalances datasets in a given dataverse or all non-metadata datasets.
Iterable<Dataset> datasets = dataverseName == null ? getAllDatasetsForRebalance()
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cache/QueryCache.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cache/QueryCache.java
new file mode 100644
index 0000000..f42587e
--- /dev/null
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cache/QueryCache.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.app.cache;
+
+import java.util.Collections;
+import java.util.Map;
+
+import org.apache.commons.collections4.map.LRUMap;
+
+public class QueryCache {
+ private final Map<QueryCacheKey, QueryCacheValue> map;
+
+ // TODO: Add data structures for fine-grained cache invalidation
+
+ public QueryCache(int maxLength) {
+ map = Collections.synchronizedMap(new LRUMap<>(maxLength));
+ }
+
+ public QueryCacheValue get(QueryCacheKey key) {
+ return map.get(key);
+ }
+
+ public void put(QueryCacheKey key, QueryCacheValue value) {
+ map.put(key, value);
+ }
+
+ public void clear() {
+ map.clear();
+ }
+}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cache/QueryCacheKey.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cache/QueryCacheKey.java
new file mode 100644
index 0000000..3e03d53
--- /dev/null
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cache/QueryCacheKey.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.app.cache;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+import org.apache.asterix.common.metadata.DataverseName;
+import org.apache.asterix.metadata.declared.MetadataProvider;
+import org.apache.asterix.translator.SessionConfig;
+
+public class QueryCacheKey {
+ private final String queryString;
+ private final SessionConfig sessionConfig;
+ private final Map<String, Object> config;
+ private final long resultSetId;
+ private final DataverseName activeDataverse;
+
+ public QueryCacheKey(String queryString, SessionConfig sessionConfig, MetadataProvider metadataProvider) {
+ this.queryString = queryString;
+ this.sessionConfig = new SessionConfig(sessionConfig);
+ this.config = new HashMap<>(metadataProvider.getConfig());
+ this.resultSetId = metadataProvider.getResultSetId().getId();
+ this.activeDataverse = metadataProvider.getDefaultDataverseName();
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (!(o instanceof QueryCacheKey)) {
+ return false;
+ }
+ if (o == this) {
+ return true;
+ }
+ QueryCacheKey other = (QueryCacheKey) o;
+ return Objects.equals(other.queryString, queryString) && Objects.equals(other.sessionConfig, sessionConfig)
+ && Objects.equals(other.config, config) && Objects.equals(other.resultSetId, resultSetId)
+ && Objects.equals(other.activeDataverse, activeDataverse);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(queryString, sessionConfig, config, resultSetId, activeDataverse);
+ }
+}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cache/QueryCacheValue.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cache/QueryCacheValue.java
new file mode 100644
index 0000000..780c28b
--- /dev/null
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cache/QueryCacheValue.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.app.cache;
+
+import java.util.Set;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+
+import org.apache.hyracks.api.exceptions.Warning;
+import org.apache.hyracks.api.job.JobSpecification;
+
+public class QueryCacheValue {
+ private final JobSpecification spec;
+ private final Set<Warning> warnings;
+ private final Lock lock;
+
+ public QueryCacheValue(JobSpecification spec, Set<Warning> warnings) {
+ this.spec = spec;
+ this.warnings = warnings;
+ this.lock = new ReentrantLock();
+ }
+
+ public JobSpecification getSpec() {
+ return spec;
+ }
+
+ public Set<Warning> getWarnings() {
+ return warnings;
+ }
+
+ public Lock getLock() {
+ return lock;
+ }
+}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/CcApplicationContext.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/CcApplicationContext.java
index 0b8024b..605dcdf 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/CcApplicationContext.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/CcApplicationContext.java
@@ -24,6 +24,7 @@
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.function.Supplier;
+import org.apache.asterix.app.cache.QueryCache;
import org.apache.asterix.app.result.ResultReader;
import org.apache.asterix.common.api.IConfigValidator;
import org.apache.asterix.common.api.IConfigValidatorFactory;
@@ -74,6 +75,7 @@
import org.apache.hyracks.api.job.IJobLifecycleListener;
import org.apache.hyracks.api.result.IResultSet;
import org.apache.hyracks.client.result.ResultSet;
+import org.apache.hyracks.control.common.controllers.CCConfig;
import org.apache.hyracks.ipc.impl.HyracksConnection;
import org.apache.hyracks.storage.common.IStorageManager;
import org.apache.hyracks.util.NetworkUtil;
@@ -117,6 +119,7 @@
private final IRequestTracker requestTracker;
private final IConfigValidator configValidator;
private final IAdapterFactoryService adapterFactoryService;
+ private final QueryCache cache;
private final ReentrantReadWriteLock compilationLock = new ReentrantReadWriteLock(true);
private final IDataPartitioningProvider dataPartitioningProvider;
@@ -163,6 +166,7 @@
configValidator = configValidatorFactory.create();
this.adapterFactoryService = adapterFactoryService;
dataPartitioningProvider = DataPartitioningProvider.create(this);
+ this.cache = new QueryCache(ccServiceCtx.getAppConfig().getInt(CCConfig.Option.QUERY_CACHE_CAPACITY));
}
@Override
@@ -381,4 +385,8 @@
public CloudProperties getCloudProperties() {
return cloudProperties;
}
+
+ public QueryCache getQueryCache() {
+ return cache;
+ }
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index 28a6d2d..fdc500c 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -22,6 +22,7 @@
import static org.apache.asterix.common.utils.IdentifierUtil.dataset;
import static org.apache.asterix.common.utils.IdentifierUtil.dataverse;
import static org.apache.asterix.lang.common.statement.CreateFullTextFilterStatement.FIELD_TYPE_STOPWORDS;
+import static org.apache.commons.collections4.MapUtils.getBoolean;
import java.io.FileInputStream;
import java.io.InputStream;
@@ -41,6 +42,7 @@
import java.util.Properties;
import java.util.Set;
import java.util.concurrent.ExecutorService;
+import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.stream.Collectors;
@@ -55,6 +57,10 @@
import org.apache.asterix.app.active.ActiveEntityEventsListener;
import org.apache.asterix.app.active.ActiveNotificationHandler;
import org.apache.asterix.app.active.FeedEventsListener;
+import org.apache.asterix.app.cache.QueryCache;
+import org.apache.asterix.app.cache.QueryCacheKey;
+import org.apache.asterix.app.cache.QueryCacheValue;
+import org.apache.asterix.app.cc.CcApplicationContext;
import org.apache.asterix.app.external.ExternalLibraryJobUtils;
import org.apache.asterix.app.result.ExecutionError;
import org.apache.asterix.app.result.ResultHandle;
@@ -69,6 +75,7 @@
import org.apache.asterix.common.api.IRequestTracker;
import org.apache.asterix.common.api.IResponsePrinter;
import org.apache.asterix.common.cluster.IClusterStateManager;
+import org.apache.asterix.common.config.CompilerProperties;
import org.apache.asterix.common.config.DatasetConfig.DatasetType;
import org.apache.asterix.common.config.DatasetConfig.IndexType;
import org.apache.asterix.common.config.DatasetConfig.TransactionState;
@@ -291,6 +298,7 @@
protected final IResponsePrinter responsePrinter;
protected final WarningCollector warningCollector;
protected final ReentrantReadWriteLock compilationLock;
+ protected final QueryCache queryCache;
public QueryTranslator(ICcApplicationContext appCtx, List<Statement> statements, SessionOutput output,
ILangCompilationProvider compilationProvider, ExecutorService executorService,
@@ -310,6 +318,7 @@
this.responsePrinter = responsePrinter;
this.warningCollector = new WarningCollector();
this.compilationLock = appCtx.getCompilationLock();
+ this.queryCache = ((CcApplicationContext) appCtx).getQueryCache();
if (appCtx.getServiceContext().getAppConfig().getBoolean(CCConfig.Option.ENFORCE_FRAME_WRITER_PROTOCOL)) {
this.jobFlags.add(JobFlag.ENFORCE_CONTRACT);
}
@@ -369,6 +378,7 @@
break;
case CREATE_INDEX:
handleCreateIndexStatement(metadataProvider, stmt, hcc, requestParameters);
+ queryCache.clear();
break;
case CREATE_FULL_TEXT_FILTER:
handleCreateFullTextFilterStatement(metadataProvider, stmt);
@@ -384,24 +394,30 @@
break;
case DATAVERSE_DROP:
handleDataverseDropStatement(metadataProvider, stmt, hcc, requestParameters);
+ queryCache.clear();
break;
case DATASET_DROP:
handleDatasetDropStatement(metadataProvider, stmt, hcc, requestParameters);
+ queryCache.clear();
break;
case INDEX_DROP:
handleIndexDropStatement(metadataProvider, stmt, hcc, requestParameters);
+ queryCache.clear();
break;
case FULL_TEXT_FILTER_DROP:
handleFullTextFilterDrop(metadataProvider, stmt, hcc, requestParameters);
+ queryCache.clear();
break;
case FULL_TEXT_CONFIG_DROP:
handleFullTextConfigDrop(metadataProvider, stmt, hcc, requestParameters);
+ queryCache.clear();
break;
case TYPE_DROP:
handleTypeDropStatement(metadataProvider, stmt);
break;
case NODEGROUP_DROP:
handleNodegroupDropStatement(metadataProvider, stmt);
+ queryCache.clear();
break;
case CREATE_ADAPTER:
handleCreateAdapterStatement(metadataProvider, stmt);
@@ -414,24 +430,28 @@
break;
case FUNCTION_DROP:
handleFunctionDropStatement(metadataProvider, stmt, requestParameters);
+ queryCache.clear();
break;
case CREATE_LIBRARY:
handleCreateLibraryStatement(metadataProvider, stmt, hcc, requestParameters);
break;
case LIBRARY_DROP:
handleLibraryDropStatement(metadataProvider, stmt, hcc, requestParameters);
+ queryCache.clear();
break;
case CREATE_SYNONYM:
handleCreateSynonymStatement(metadataProvider, stmt);
break;
case SYNONYM_DROP:
handleDropSynonymStatement(metadataProvider, stmt);
+ queryCache.clear();
break;
case CREATE_VIEW:
handleCreateViewStatement(metadataProvider, stmt, stmtRewriter, requestParameters);
break;
case VIEW_DROP:
handleViewDropStatement(metadataProvider, stmt);
+ queryCache.clear();
break;
case LOAD:
if (stats.getProfileType() == Stats.ProfileType.FULL) {
@@ -467,6 +487,7 @@
break;
case DROP_FEED:
handleDropFeedStatement(metadataProvider, stmt, hcc);
+ queryCache.clear();
break;
case DROP_FEED_POLICY:
handleDropFeedPolicyStatement(metadataProvider, stmt);
@@ -476,6 +497,7 @@
break;
case DISCONNECT_FEED:
handleDisconnectFeedStatement(metadataProvider, stmt);
+ queryCache.clear();
break;
case START_FEED:
handleStartFeedStatement(metadataProvider, stmt, hcc);
@@ -499,9 +521,11 @@
break;
case ANALYZE:
handleAnalyzeStatement(metadataProvider, stmt, hcc, requestParameters);
+ queryCache.clear();
break;
case ANALYZE_DROP:
handleAnalyzeDropStatement(metadataProvider, stmt, hcc, requestParameters);
+ queryCache.clear();
break;
case COMPACT:
handleCompactStatement(metadataProvider, stmt, hcc);
@@ -559,6 +583,10 @@
throw new CompilationException(ErrorCode.ILLEGAL_SET_PARAMETER, pname);
}
config.put(pname, pvalue);
+ if (getBoolean(config, CompilerProperties.COMPILER_QUERYCACHE_CLEAR_KEY,
+ appCtx.getCompilerProperties().getClearQueryCache())) {
+ queryCache.clear();
+ }
}
protected LangRewritingContext createLangRewritingContext(MetadataProvider metadataProvider,
@@ -4533,14 +4561,55 @@
compilationLock.readLock().unlock();
}
};
+ final Pair<QueryCacheKey, QueryCacheValue> toCache = new Pair<>(null, null);
+ final Lock[] toUnlock = new Lock[1];
final IStatementCompiler compiler = () -> {
long compileStart = System.nanoTime();
MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
boolean bActiveTxn = true;
metadataProvider.setMetadataTxnContext(mdTxnCtx);
try {
- final JobSpecification jobSpec =
- rewriteCompileQuery(hcc, metadataProvider, query, null, stmtParams, requestParameters);
+ // check caching conditions
+ boolean bypassCache =
+ metadataProvider.getBooleanProperty(CompilerProperties.COMPILER_QUERYCACHE_BYPASS_KEY,
+ appCtx.getCompilerProperties().getBypassQueryCache());
+ boolean ignoreCache = bypassCache || sessionConfig.is(SessionConfig.QUERYCACHE_BYPASS)
+ || query.isExplain() || requestParameters.getStatementParameters() != null
+ || !sessionConfig.isGenerateJobSpec() || sessionConfig.is(SessionConfig.OOB_EXPR_TREE)
+ || sessionConfig.is(SessionConfig.OOB_REWRITTEN_EXPR_TREE)
+ || sessionConfig.is(SessionConfig.OOB_LOGICAL_PLAN)
+ || sessionConfig.is(SessionConfig.OOB_OPTIMIZED_LOGICAL_PLAN)
+ || sessionConfig.is(SessionConfig.OOB_HYRACKS_JOB);
+
+ final JobSpecification jobSpec;
+ if (ignoreCache) {
+ jobSpec = rewriteCompileQuery(hcc, metadataProvider, query, null, stmtParams, requestParameters);
+ } else {
+ final String queryString = apiFramework.generateExpressionTreeString(query);
+ final QueryCacheKey cacheKey = new QueryCacheKey(queryString, sessionConfig, metadataProvider);
+ final QueryCacheValue cacheValue = queryCache.get(cacheKey);
+ // try to acquire lock on query cache entry, since same job cannot be run from multiple threads
+ if (cacheValue != null && cacheValue.getLock().tryLock()) {
+ toUnlock[0] = cacheValue.getLock();
+ jobSpec = cacheValue.getSpec();
+ // add skipped warnings from cache
+ WarningUtil.mergeWarnings(cacheValue.getWarnings(), warningCollector);
+ } else {
+ Set<Warning> oldWarnings = new LinkedHashSet<>();
+ warningCollector.getWarnings(oldWarnings, Long.MAX_VALUE);
+ jobSpec =
+ rewriteCompileQuery(hcc, metadataProvider, query, null, stmtParams, requestParameters);
+ if (cacheValue == null) {
+ // find and cache warnings issued during rewriting and compilation
+ Set<Warning> newWarnings = new LinkedHashSet<>();
+ warningCollector.getWarnings(newWarnings, Long.MAX_VALUE);
+ newWarnings.removeAll(oldWarnings);
+ toCache.setFirst(cacheKey);
+ toCache.setSecond(new QueryCacheValue(jobSpec, newWarnings));
+ }
+ }
+ }
+
// update stats with count of compile-time warnings. needs to be adapted for multi-statement.
stats.updateTotalWarningsCount(warningCollector.getTotalWarningsCount());
afterCompile();
@@ -4556,8 +4625,19 @@
throw e;
}
};
- deliverResult(hcc, resultSet, compiler, metadataProvider, locker, resultDelivery, outMetadata, stats,
- requestParameters, true);
+ try {
+ deliverResult(hcc, resultSet, compiler, metadataProvider, locker, resultDelivery, outMetadata, stats,
+ requestParameters, true);
+ } finally {
+ // release lock on job spec from cache after job has finished running
+ if (toUnlock[0] != null) {
+ toUnlock[0].unlock();
+ }
+ }
+ // insert into query cache only after running job so that queries raising runtime errors are not cached
+ if (toCache.getFirst() != null) {
+ queryCache.put(toCache.getFirst(), toCache.getSecond());
+ }
}
private void deliverResult(IHyracksClientConnection hcc, IResultSet resultSet, IStatementCompiler compiler,
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
index f1d484e..70ea025 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
@@ -2136,11 +2136,17 @@
public void executeTest(String actualPath, TestCaseContext testCaseCtx, ProcessBuilder pb,
boolean isDmlRecoveryTest, TestGroup failedGroup) throws Exception {
- executeTest(actualPath, testCaseCtx, pb, isDmlRecoveryTest, failedGroup, null);
+ executeTest(actualPath, testCaseCtx, pb, isDmlRecoveryTest, failedGroup, null, false);
+ }
+
+ public void executeCacheTest(String actualPath, TestCaseContext testCaseCtx, ProcessBuilder pb,
+ boolean isDmlRecoveryTest, TestGroup failedGroup) throws Exception {
+ executeTest(actualPath, testCaseCtx, pb, isDmlRecoveryTest, failedGroup, null, true);
}
public void executeTest(String actualPath, TestCaseContext testCaseCtx, ProcessBuilder pb,
- boolean isDmlRecoveryTest, TestGroup failedGroup, TestGroup passedGroup) throws Exception {
+ boolean isDmlRecoveryTest, TestGroup failedGroup, TestGroup passedGroup, boolean testCache)
+ throws Exception {
MutableInt queryCount = new MutableInt(0);
testCaseCtx.numOfErrors = 0;
int numOfFiles = 0;
@@ -2171,8 +2177,20 @@
try {
boolean loopCmd = testFile.getName().endsWith(".loop.cmd");
if (!testFile.getName().startsWith(DIAGNOSE)) {
- executeTestFile(testCaseCtx, ctx, variableCtx, statement, isDmlRecoveryTest, pb, cUnit,
- queryCount, expectedResultFileCtxs, testFile, actualPath);
+ int loopIterations = 1;
+ // run query files 3 times to test the query plan cache
+ if (testCache && ctx.getType().equals("query") && !statement.contains("// loopIterations=1")) {
+ loopIterations = 3;
+ }
+ for (int i = 0; i < loopIterations; i++) {
+ BitSet warningsCopy = (BitSet) testCaseCtx.expectedWarnings.clone();
+ executeTestFile(testCaseCtx, ctx, variableCtx, statement, isDmlRecoveryTest, pb, cUnit,
+ queryCount, expectedResultFileCtxs, testFile, actualPath);
+ if (i < loopIterations - 1) {
+ queryCount.decrement();
+ testCaseCtx.expectedWarnings = warningsCopy;
+ }
+ }
}
if (loopCmd) {
// this was a loop file and we have exited the loop; reset the loop iteration
@@ -2601,12 +2619,20 @@
dropDataverse(dv);
}
}
+ clearCache();
} catch (Throwable th) {
th.printStackTrace();
throw th;
}
}
+ protected void clearCache() throws Exception {
+ String clearCacheStatement = "SET `compiler.querycache.clear` \"true\";\n";
+ InputStream resultStream = executeQueryService(clearCacheStatement, getEndpoint(Servlets.QUERY_SERVICE),
+ OutputFormat.CLEAN_JSON, UTF_8);
+ ResultExtractor.extract(resultStream, UTF_8, OutputFormat.CLEAN_JSON);
+ }
+
protected void listUserDefinedDataverses(List<DataverseName> outDataverses) throws Exception {
String query = "select dv.DataverseName from Metadata.`Dataverse` as dv order by dv.DataverseName";
InputStream resultStream =
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/LangExecutionUtil.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/LangExecutionUtil.java
index d704d8e..5fbc4f7 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/LangExecutionUtil.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/LangExecutionUtil.java
@@ -128,17 +128,26 @@
}
public static void test(TestCaseContext tcCtx) throws Exception {
- test(testExecutor, tcCtx);
+ test(testExecutor, tcCtx, false);
}
- public static void test(TestExecutor testExecutor, TestCaseContext tcCtx) throws Exception {
+ public static void testCache(TestCaseContext tcCtx) throws Exception {
+ test(testExecutor, tcCtx, true);
+ }
+
+ public static void test(TestExecutor testExecutor, TestCaseContext tcCtx, boolean testCache) throws Exception {
int repeat = LangExecutionUtil.repeat * tcCtx.getRepeat();
try {
for (int i = 1; i <= repeat; i++) {
if (repeat > 1) {
System.err.print("[" + i + "/" + repeat + "] ");
}
- testExecutor.executeTest(PATH_ACTUAL, tcCtx, null, false, ExecutionTestUtil.FailedGroup);
+
+ if (testCache) {
+ testExecutor.executeCacheTest(PATH_ACTUAL, tcCtx, null, false, ExecutionTestUtil.FailedGroup);
+ } else {
+ testExecutor.executeTest(PATH_ACTUAL, tcCtx, null, false, ExecutionTestUtil.FailedGroup);
+ }
try {
if (checkStorageDistribution) {
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/SqlppExecutionWithCacheTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/SqlppExecutionWithCacheTest.java
new file mode 100644
index 0000000..851dc2d
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/SqlppExecutionWithCacheTest.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.test.runtime;
+
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.asterix.common.api.INcApplicationContext;
+import org.apache.asterix.test.common.TestExecutor;
+import org.apache.asterix.testframework.context.TestCaseContext;
+import org.apache.hyracks.control.nc.NodeControllerService;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+/**
+ * Runs the SQL++ runtime tests with the storage parallelism, running each query 3 times to test the query plan cache.
+ */
+@RunWith(Parameterized.class)
+public class SqlppExecutionWithCacheTest {
+ protected static final String TEST_CONFIG_FILE_NAME = "src/main/resources/cc.conf";
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ final TestExecutor testExecutor = new TestExecutor();
+ LangExecutionUtil.setUp(TEST_CONFIG_FILE_NAME, testExecutor);
+ setNcEndpoints(testExecutor);
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ LangExecutionUtil.tearDown();
+ }
+
+ @Parameters(name = "SqlppExecutionTest {index}: {0}")
+ public static Collection<Object[]> tests() throws Exception {
+ return LangExecutionUtil.tests("only_sqlpp.xml", "testsuite_sqlpp.xml");
+ }
+
+ protected TestCaseContext tcCtx;
+
+ public SqlppExecutionWithCacheTest(TestCaseContext tcCtx) {
+ this.tcCtx = tcCtx;
+ }
+
+ @Test
+ public void test() throws Exception {
+ LangExecutionUtil.testCache(tcCtx);
+ }
+
+ private static void setNcEndpoints(TestExecutor testExecutor) {
+ final NodeControllerService[] ncs = ExecutionTestUtil.integrationUtil.ncs;
+ final Map<String, InetSocketAddress> ncEndPoints = new HashMap<>();
+ final String ip = InetAddress.getLoopbackAddress().getHostAddress();
+ for (NodeControllerService nc : ncs) {
+ final String nodeId = nc.getId();
+ final INcApplicationContext appCtx = (INcApplicationContext) nc.getApplicationContext();
+ int apiPort = appCtx.getExternalProperties().getNcApiPort();
+ ncEndPoints.put(nodeId, InetSocketAddress.createUnresolved(ip, apiPort));
+ }
+ testExecutor.setNcEndPoints(ncEndPoints);
+ }
+}
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/aggregate-sql-sugar/avg_mixed/avg_mixed.5.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/aggregate-sql-sugar/avg_mixed/avg_mixed.5.query.sqlpp
index 71b22e7..0de17b7 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/aggregate-sql-sugar/avg_mixed/avg_mixed.5.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/aggregate-sql-sugar/avg_mixed/avg_mixed.5.query.sqlpp
@@ -24,7 +24,7 @@
-- param max-warnings:json=100
use test;
-
+SET `compiler.querycache.bypass` "true";
select d1.id, avg(v) a
from d1, d1.v
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/insert-return-records/insert-return-records.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/insert-return-records/insert-return-records.3.query.sqlpp
index 09a3b59..d9ade67 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/insert-return-records/insert-return-records.3.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/insert-return-records/insert-return-records.3.query.sqlpp
@@ -23,6 +23,8 @@
* Date : Mar 2015
*/
+ // loopIterations=1
+
use test;
insert into TweetMessageuuids as message (
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/upsert/upsert-case-returning/upsert-case-returning.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/upsert/upsert-case-returning/upsert-case-returning.3.query.sqlpp
index 0677150..6cd6528 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/upsert/upsert-case-returning/upsert-case-returning.3.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/upsert/upsert-case-returning/upsert-case-returning.3.query.sqlpp
@@ -21,6 +21,9 @@
* Expected Res : Success
* Date : Jun 2018
*/
+
+ // loopIterations=1
+
use test;
upsert into subscriptions as record(
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/annotations/ExternalSubpathAnnotation.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/annotations/ExternalSubpathAnnotation.java
index e3e5ea3..364e432 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/annotations/ExternalSubpathAnnotation.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/annotations/ExternalSubpathAnnotation.java
@@ -22,6 +22,7 @@
public final class ExternalSubpathAnnotation implements IExpressionAnnotation {
+ public static final String HINT_STRING = "external-subpath";
private final String subPath;
public ExternalSubpathAnnotation(String subPath) {
@@ -31,4 +32,9 @@
public String getSubPath() {
return subPath;
}
+
+ @Override
+ public String toString() {
+ return HINT_STRING + ':' + subPath;
+ }
}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
index 01f9509..ca9a7c2 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
@@ -122,7 +122,9 @@
COMPILER_COLUMN_FILTER(
BOOLEAN,
AlgebricksConfig.COLUMN_FILTER_DEFAULT,
- "Enable/disable the use of column min/max filters");
+ "Enable/disable the use of column min/max filters"),
+ COMPILER_QUERYCACHE_BYPASS(BOOLEAN, AlgebricksConfig.QUERYCACHE_BYPASS_DEFAULT, "Bypass the query plan cache"),
+ COMPILER_QUERYCACHE_CLEAR(BOOLEAN, AlgebricksConfig.QUERYCACHE_CLEAR_DEFAULT, "Clear the query plan cache");
private final IOptionType type;
private final Object defaultValue;
@@ -204,6 +206,10 @@
public static final String COMPILER_COLUMN_FILTER_KEY = Option.COMPILER_COLUMN_FILTER.ini();
+ public static final String COMPILER_QUERYCACHE_BYPASS_KEY = Option.COMPILER_QUERYCACHE_BYPASS.ini();
+
+ public static final String COMPILER_QUERYCACHE_CLEAR_KEY = Option.COMPILER_QUERYCACHE_CLEAR.ini();
+
public static final int COMPILER_PARALLELISM_AS_STORAGE = 0;
public CompilerProperties(PropertiesAccessor accessor) {
@@ -306,4 +312,12 @@
public boolean isColumnFilter() {
return accessor.getBoolean(Option.COMPILER_COLUMN_FILTER);
}
+
+ public boolean getBypassQueryCache() {
+ return accessor.getBoolean(Option.COMPILER_QUERYCACHE_BYPASS);
+ }
+
+ public boolean getClearQueryCache() {
+ return accessor.getBoolean(Option.COMPILER_QUERYCACHE_CLEAR);
+ }
}
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/QueryPrintVisitor.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/QueryPrintVisitor.java
index aa875b4..fba031e 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/QueryPrintVisitor.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/QueryPrintVisitor.java
@@ -184,6 +184,11 @@
exprList.get(i).accept(this, step + 1);
}
out.println(skip(step) + "]");
+ if (ifbo.hasHints()) {
+ for (var hint : ifbo.getHints()) {
+ out.print(skip(step + 1) + hint.toString());
+ }
+ }
} else {
exprList.get(0).accept(this, step);
}
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/annotation/ExcludeFromSelectStarAnnotation.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/annotation/ExcludeFromSelectStarAnnotation.java
index a2998a7..d0e8f13 100644
--- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/annotation/ExcludeFromSelectStarAnnotation.java
+++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/annotation/ExcludeFromSelectStarAnnotation.java
@@ -23,8 +23,15 @@
public final class ExcludeFromSelectStarAnnotation implements IExpressionAnnotation {
+ public static final String HINT_STRING = "exclude-from-select-star";
+
public static final ExcludeFromSelectStarAnnotation INSTANCE = new ExcludeFromSelectStarAnnotation();
private ExcludeFromSelectStarAnnotation() {
}
+
+ @Override
+ public String toString() {
+ return HINT_STRING;
+ }
}
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlCompatRewriteVisitor.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlCompatRewriteVisitor.java
index b98151b..9cac5a0 100644
--- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlCompatRewriteVisitor.java
+++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlCompatRewriteVisitor.java
@@ -509,6 +509,8 @@
private static class SqlCompatSelectExpressionCoercionAnnotation implements IExpressionAnnotation {
+ static final String HINT_STRING = "sql-compat-select-coercion";
+
static final SqlCompatSelectExpressionCoercionAnnotation NONE_NONE =
new SqlCompatSelectExpressionCoercionAnnotation(SqlCompatSelectCoercionKind.NONE,
SqlCompatSelectCoercionKind.NONE, null);
@@ -525,6 +527,12 @@
this.typeCoercion = Objects.requireNonNull(typeCoercion);
this.typeCoercionFieldName = typeCoercionFieldName;
}
+
+ @Override
+ public String toString() {
+ String result = HINT_STRING + ':' + cardinalityCoercion + ':' + typeCoercion;
+ return typeCoercionFieldName == null ? result : result + ':' + typeCoercionFieldName;
+ }
}
private enum SqlCompatSelectCoercionKind {
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/SqlppAstPrintVisitor.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/SqlppAstPrintVisitor.java
index cab6e87..7a8580a 100644
--- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/SqlppAstPrintVisitor.java
+++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/SqlppAstPrintVisitor.java
@@ -401,6 +401,12 @@
printFieldList(step + 1, winExpr.getWindowFieldList());
}
}
+ if (winExpr.getFromLast() != null && winExpr.getFromLast()) {
+ out.println(skip(step + 1) + "FROM LAST");
+ }
+ if (winExpr.getIgnoreNulls() != null && winExpr.getIgnoreNulls()) {
+ out.println(skip(step + 1) + "IGNORE NULLS");
+ }
if (winExpr.hasAggregateFilterExpr()) {
out.println(skip(step + 1) + "FILTER (WHERE");
winExpr.getAggregateFilterExpr().accept(this, step + 2);
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/BroadcastExpressionAnnotation.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/BroadcastExpressionAnnotation.java
index f1b5093..2dcec9f 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/BroadcastExpressionAnnotation.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/BroadcastExpressionAnnotation.java
@@ -22,6 +22,8 @@
public final class BroadcastExpressionAnnotation implements IExpressionAnnotation {
+ public static final String HINT_STRING = "broadcast";
+
public enum BroadcastSide {
LEFT,
RIGHT;
@@ -71,4 +73,9 @@
}
return n;
}
+
+ @Override
+ public String toString() {
+ return name == null ? HINT_STRING : HINT_STRING + ':' + name;
+ }
}
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/HashJoinExpressionAnnotation.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/HashJoinExpressionAnnotation.java
index 88d9dd5..7a063f5 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/HashJoinExpressionAnnotation.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/HashJoinExpressionAnnotation.java
@@ -25,6 +25,8 @@
public class HashJoinExpressionAnnotation implements IExpressionAnnotation {
+ public static final String HINT_STRING = "hash-join";
+
public enum BuildOrProbe {
BUILD,
PROBE
@@ -76,4 +78,19 @@
}
return n;
}
+
+ @Override
+ public String toString() {
+ String result = HINT_STRING;
+ if (name != null) {
+ result += ':' + name;
+ }
+ if (buildOrProbe != null) {
+ result += ':' + buildOrProbe.toString();
+ }
+ if (side != null) {
+ result += ':' + side.toString();
+ }
+ return result;
+ }
}
\ No newline at end of file
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/JoinProductivityAnnotation.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/JoinProductivityAnnotation.java
index 0191b85..0cba019 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/JoinProductivityAnnotation.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/JoinProductivityAnnotation.java
@@ -21,6 +21,7 @@
public class JoinProductivityAnnotation implements IExpressionAnnotation {
+ public static final String HINT_STRING = "join-productivity";
private final double productivity;
private final String leftSideDataSet;
@@ -36,4 +37,10 @@
public String getLeftSideDataSet() {
return leftSideDataSet;
}
+
+ @Override
+ public String toString() {
+ String result = HINT_STRING + ':' + productivity;
+ return leftSideDataSet == null ? result : result + ':' + leftSideDataSet;
+ }
}
\ No newline at end of file
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/PredicateCardinalityAnnotation.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/PredicateCardinalityAnnotation.java
index e172449..155783b 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/PredicateCardinalityAnnotation.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/PredicateCardinalityAnnotation.java
@@ -21,6 +21,8 @@
public class PredicateCardinalityAnnotation implements IExpressionAnnotation {
+ public static final String HINT_STRING = "predicate-cardinality";
+
private final double selectivity;
public PredicateCardinalityAnnotation(double selectivity) {
@@ -30,4 +32,9 @@
public double getSelectivity() {
return selectivity;
}
+
+ @Override
+ public String toString() {
+ return HINT_STRING + ':' + selectivity;
+ }
}
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java
index 74f4447..d3ccf89 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java
@@ -46,4 +46,6 @@
StorageUtil.getIntSizeInBytes(8, StorageUtil.StorageUnit.KILOBYTE);
public static final boolean BATCH_LOOKUP_DEFAULT = true;
public static final boolean COLUMN_FILTER_DEFAULT = false;
+ public static final boolean QUERYCACHE_BYPASS_DEFAULT = false;
+ public static final boolean QUERYCACHE_CLEAR_DEFAULT = false;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/CCConfig.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/CCConfig.java
index d148145..4c2ff0f 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/CCConfig.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/CCConfig.java
@@ -73,6 +73,7 @@
CLUSTER_TOPOLOGY(STRING),
JOB_QUEUE_CLASS(STRING, "org.apache.hyracks.control.cc.scheduler.FIFOJobQueue"),
JOB_QUEUE_CAPACITY(POSITIVE_INTEGER, 4096),
+ QUERY_CACHE_CAPACITY(NONNEGATIVE_INTEGER, 1024),
JOB_MANAGER_CLASS(STRING, "org.apache.hyracks.control.cc.job.JobManager"),
ENFORCE_FRAME_WRITER_PROTOCOL(BOOLEAN, false),
CORES_MULTIPLIER(POSITIVE_INTEGER, 3),
@@ -188,6 +189,8 @@
return "Specify the implementation class name for the job queue";
case JOB_QUEUE_CAPACITY:
return "The maximum number of jobs to queue before rejecting new jobs";
+ case QUERY_CACHE_CAPACITY:
+ return "The maximum size of the query plan cache before replacing old entries";
case JOB_MANAGER_CLASS:
return "Specify the implementation class name for the job manager";
case ENFORCE_FRAME_WRITER_PROTOCOL:
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17627
To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Change-Id: If3a9c86fadb7c869bc97e4084d0ebaa43a7ec111
Gerrit-Change-Number: 17627
Gerrit-PatchSet: 1
Gerrit-Owner: Sushrut Borkar <su...@gmail.com>
Gerrit-MessageType: newchange