You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by dl...@apache.org on 2017/11/04 18:56:49 UTC
[3/3] asterixdb git commit: [ASTERIXDB-2154][COMP] Extensible
built-in function management
[ASTERIXDB-2154][COMP] Extensible built-in function management
- user model changes: no
- storage format changes: no
- interface changes: yes
Details:
- Support for augmenting built-in functions in ILangExtension
- Move function type inferers from NonTaggedDataFormat to
FunctionTypeInferers, exposed through IFunctionDescriptorFactory
and IFunctionManager
- Cleanup: obtain IFunctionManager/IDataFormat from MetadataProvider
Change-Id: Iafbf7e4e298fb7af6cb84b402cb00cfa2e90b5ce
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2118
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Contrib: Jenkins <je...@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Ian Maxon <im...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/76d24d49
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/76d24d49
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/76d24d49
Branch: refs/heads/master
Commit: 76d24d49055d764fc03b19bce441e97a563622b4
Parents: 52e5134
Author: Dmitry Lychagin <dm...@couchbase.com>
Authored: Fri Nov 3 13:28:21 2017 -0700
Committer: Dmitry Lychagin <dm...@couchbase.com>
Committed: Sat Nov 4 11:56:19 2017 -0700
----------------------------------------------------------------------
.../asterix/algebra/base/ILangExtension.java | 5 +
.../jobgen/QueryLogicalExpressionJobGen.java | 62 +-
.../optimizer/rules/ConstantFoldingRule.java | 9 +-
.../apache/asterix/api/common/APIFramework.java | 5 +-
.../asterix/app/cc/CCExtensionManager.java | 30 +-
.../org/apache/asterix/utils/ExtensionUtil.java | 36 +-
.../org/apache/asterix/runtime/ExceptionIT.java | 3 +-
.../apache/asterix/runtime/NullMissingTest.java | 3 +-
.../test/active/ActiveEventsListenerTest.java | 12 +
.../metadata/declared/DatasetDataSource.java | 4 +-
.../metadata/declared/FeedDataSource.java | 3 +-
.../metadata/declared/MetadataProvider.java | 17 +-
.../asterix/metadata/entities/Dataset.java | 2 +-
.../asterix/metadata/utils/DatasetUtil.java | 18 +-
.../utils/SecondaryBTreeOperationsHelper.java | 17 +-
...econdaryCorrelatedBTreeOperationsHelper.java | 17 +-
...CorrelatedInvertedIndexOperationsHelper.java | 16 +-
...econdaryCorrelatedRTreeOperationsHelper.java | 6 +-
...daryCorrelatedTreeIndexOperationsHelper.java | 7 +-
.../utils/SecondaryIndexOperationsHelper.java | 17 +-
.../SecondaryInvertedIndexOperationsHelper.java | 16 +-
.../utils/SecondaryRTreeOperationsHelper.java | 6 +-
.../asterix/formats/base/IDataFormat.java | 19 +-
.../functions/IFunctionDescriptorFactory.java | 10 +-
.../om/functions/IFunctionExtensionManager.java | 32 +
.../asterix/om/functions/IFunctionManager.java | 12 +-
.../om/functions/IFunctionTypeInferer.java | 29 +
.../collections/ListifyAggregateDescriptor.java | 10 +-
.../AbstractScalarAggregateDescriptor.java | 14 +-
.../scalar/ScalarAvgAggregateDescriptor.java | 10 +-
.../scalar/ScalarCountAggregateDescriptor.java | 10 +-
.../scalar/ScalarMaxAggregateDescriptor.java | 10 +-
.../scalar/ScalarMinAggregateDescriptor.java | 10 +-
.../scalar/ScalarSqlAvgAggregateDescriptor.java | 10 +-
.../ScalarSqlCountAggregateDescriptor.java | 11 +-
.../scalar/ScalarSqlMaxAggregateDescriptor.java | 10 +-
.../scalar/ScalarSqlMinAggregateDescriptor.java | 10 +-
.../scalar/ScalarSqlSumAggregateDescriptor.java | 10 +-
.../scalar/ScalarSumAggregateDescriptor.java | 10 +-
.../ClosedRecordConstructorDescriptor.java | 10 +-
.../OpenRecordConstructorDescriptor.java | 7 +
.../OrderedListConstructorDescriptor.java | 12 +-
.../UnorderedListConstructorDescriptor.java | 12 +-
.../functions/CastTypeDescriptor.java | 9 +-
.../functions/CastTypeLaxDescriptor.java | 7 +
.../functions/DeepEqualityDescriptor.java | 9 +-
.../records/FieldAccessByIndexDescriptor.java | 11 +-
.../records/FieldAccessNestedDescriptor.java | 11 +-
.../records/GetRecordFieldValueDescriptor.java | 11 +-
.../records/GetRecordFieldsDescriptor.java | 11 +-
.../records/RecordAddFieldsDescriptor.java | 8 +
.../records/RecordMergeDescriptor.java | 8 +
.../records/RecordPairsDescriptor.java | 7 +
.../records/RecordRemoveFieldsDescriptor.java | 18 +-
.../runtime/formats/NonTaggedDataFormat.java | 350 +---------
.../runtime/functions/FunctionCollection.java | 641 ++++++++++---------
.../runtime/functions/FunctionManager.java | 73 +++
.../functions/FunctionManagerHolder.java | 41 --
.../runtime/functions/FunctionManagerImpl.java | 68 --
.../runtime/functions/FunctionTypeInferers.java | 271 ++++++++
60 files changed, 1189 insertions(+), 944 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/base/ILangExtension.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/base/ILangExtension.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/base/ILangExtension.java
index 4803691..fb74475 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/base/ILangExtension.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/base/ILangExtension.java
@@ -21,6 +21,7 @@ package org.apache.asterix.algebra.base;
import org.apache.asterix.common.api.IExtension;
import org.apache.asterix.compiler.provider.ILangCompilationProvider;
+import org.apache.asterix.om.functions.IFunctionManager;
/**
* An interface for language extensions
@@ -38,4 +39,8 @@ public interface ILangExtension extends IExtension {
}
ILangCompilationProvider getLangCompilationProvider(Language lang);
+
+ default IFunctionManager getFunctionManager() {
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/jobgen/QueryLogicalExpressionJobGen.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/jobgen/QueryLogicalExpressionJobGen.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/jobgen/QueryLogicalExpressionJobGen.java
index 9f88bb6..812f3c9 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/jobgen/QueryLogicalExpressionJobGen.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/jobgen/QueryLogicalExpressionJobGen.java
@@ -23,11 +23,12 @@ import java.util.List;
import org.apache.asterix.common.dataflow.ICcApplicationContext;
import org.apache.asterix.common.functions.FunctionDescriptorTag;
import org.apache.asterix.external.library.ExternalFunctionDescriptorProvider;
-import org.apache.asterix.formats.base.IDataFormat;
+import org.apache.asterix.metadata.declared.MetadataProvider;
import org.apache.asterix.om.functions.BuiltinFunctions;
import org.apache.asterix.om.functions.IExternalFunctionInfo;
import org.apache.asterix.om.functions.IFunctionDescriptor;
-import org.apache.asterix.runtime.formats.FormatUtils;
+import org.apache.asterix.om.functions.IFunctionManager;
+import org.apache.asterix.om.functions.IFunctionTypeInferer;
import org.apache.commons.lang3.mutable.Mutable;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
@@ -40,6 +41,7 @@ import org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvir
import org.apache.hyracks.algebricks.core.algebra.expressions.StatefulFunctionCallExpression;
import org.apache.hyracks.algebricks.core.algebra.expressions.UnnestingFunctionCallExpression;
import org.apache.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
import org.apache.hyracks.algebricks.core.jobgen.impl.JobGenContext;
import org.apache.hyracks.algebricks.runtime.base.IAggregateEvaluatorFactory;
@@ -51,9 +53,10 @@ import org.apache.hyracks.algebricks.runtime.evaluators.ColumnAccessEvalFactory;
public class QueryLogicalExpressionJobGen implements ILogicalExpressionJobGen {
- public static final QueryLogicalExpressionJobGen INSTANCE = new QueryLogicalExpressionJobGen();
+ private final IFunctionManager functionManager;
- private QueryLogicalExpressionJobGen() {
+ public QueryLogicalExpressionJobGen(IFunctionManager functionManager) {
+ this.functionManager = functionManager;
}
@Override
@@ -61,7 +64,7 @@ public class QueryLogicalExpressionJobGen implements ILogicalExpressionJobGen {
IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas, JobGenContext context)
throws AlgebricksException {
IScalarEvaluatorFactory[] args = codegenArguments(expr, env, inputSchemas, context);
- IFunctionDescriptor fd = getFunctionDescriptor(expr, env, context);
+ IFunctionDescriptor fd = resolveFunction(expr, env);
switch (fd.getFunctionDescriptorTag()) {
case SERIALAGGREGATE:
return null;
@@ -79,7 +82,7 @@ public class QueryLogicalExpressionJobGen implements ILogicalExpressionJobGen {
IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas, JobGenContext context)
throws AlgebricksException {
IScalarEvaluatorFactory[] args = codegenArguments(expr, env, inputSchemas, context);
- return getFunctionDescriptor(expr, env, context).createRunningAggregateEvaluatorFactory(args);
+ return resolveFunction(expr, env).createRunningAggregateEvaluatorFactory(args);
}
@Override
@@ -87,22 +90,22 @@ public class QueryLogicalExpressionJobGen implements ILogicalExpressionJobGen {
IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas, JobGenContext context)
throws AlgebricksException {
IScalarEvaluatorFactory[] args = codegenArguments(expr, env, inputSchemas, context);
- return getFunctionDescriptor(expr, env, context).createUnnestingEvaluatorFactory(args);
+ return resolveFunction(expr, env).createUnnestingEvaluatorFactory(args);
}
@Override
public IScalarEvaluatorFactory createEvaluatorFactory(ILogicalExpression expr, IVariableTypeEnvironment env,
IOperatorSchema[] inputSchemas, JobGenContext context) throws AlgebricksException {
- IScalarEvaluatorFactory copyEvaluatorFactory = null;
+ IScalarEvaluatorFactory copyEvaluatorFactory;
switch (expr.getExpressionTag()) {
case VARIABLE: {
VariableReferenceExpression v = (VariableReferenceExpression) expr;
- copyEvaluatorFactory = createVariableEvaluatorFactory(v, inputSchemas, context);
+ copyEvaluatorFactory = createVariableEvaluatorFactory(v, inputSchemas);
return copyEvaluatorFactory;
}
case CONSTANT: {
ConstantExpression c = (ConstantExpression) expr;
- copyEvaluatorFactory = createConstantEvaluatorFactory(c, inputSchemas, context);
+ copyEvaluatorFactory = createConstantEvaluatorFactory(c, context);
return copyEvaluatorFactory;
}
case FUNCTION_CALL: {
@@ -117,7 +120,7 @@ public class QueryLogicalExpressionJobGen implements ILogicalExpressionJobGen {
}
private IScalarEvaluatorFactory createVariableEvaluatorFactory(VariableReferenceExpression expr,
- IOperatorSchema[] inputSchemas, JobGenContext context) throws AlgebricksException {
+ IOperatorSchema[] inputSchemas) throws AlgebricksException {
LogicalVariable variable = expr.getVariableReference();
for (IOperatorSchema scm : inputSchemas) {
int pos = scm.findVariable(variable);
@@ -132,22 +135,17 @@ public class QueryLogicalExpressionJobGen implements ILogicalExpressionJobGen {
IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas, JobGenContext context)
throws AlgebricksException {
IScalarEvaluatorFactory[] args = codegenArguments(expr, env, inputSchemas, context);
- IFunctionDescriptor fd = null;
- if (!(expr.getFunctionInfo() instanceof IExternalFunctionInfo)) {
- IDataFormat format = FormatUtils.getDefaultFormat();
- fd = format.resolveFunction(expr, env);
- } else {
- ICcApplicationContext appCtx = (ICcApplicationContext) context.getAppContext();
- fd = ExternalFunctionDescriptorProvider
- .getExternalFunctionDescriptor((IExternalFunctionInfo) expr.getFunctionInfo(), appCtx);
- }
+ IFunctionDescriptor fd = expr.getFunctionInfo() instanceof IExternalFunctionInfo
+ ? ExternalFunctionDescriptorProvider.getExternalFunctionDescriptor(
+ (IExternalFunctionInfo) expr.getFunctionInfo(), (ICcApplicationContext) context.getAppContext())
+ : resolveFunction(expr, env);
return fd.createEvaluatorFactory(args);
}
- private IScalarEvaluatorFactory createConstantEvaluatorFactory(ConstantExpression expr,
- IOperatorSchema[] inputSchemas, JobGenContext context) throws AlgebricksException {
- IDataFormat format = FormatUtils.getDefaultFormat();
- return format.getConstantEvalFactory(expr.getValue());
+ private IScalarEvaluatorFactory createConstantEvaluatorFactory(ConstantExpression expr, JobGenContext context)
+ throws AlgebricksException {
+ MetadataProvider metadataProvider = (MetadataProvider) context.getMetadataProvider();
+ return metadataProvider.getDataFormat().getConstantEvalFactory(expr.getValue());
}
private IScalarEvaluatorFactory[] codegenArguments(AbstractFunctionCallExpression expr,
@@ -168,14 +166,14 @@ public class QueryLogicalExpressionJobGen implements ILogicalExpressionJobGen {
AggregateFunctionCallExpression expr, IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas,
JobGenContext context) throws AlgebricksException {
IScalarEvaluatorFactory[] args = codegenArguments(expr, env, inputSchemas, context);
- IFunctionDescriptor fd = getFunctionDescriptor(expr, env, context);
+ IFunctionDescriptor fd = resolveFunction(expr, env);
switch (fd.getFunctionDescriptorTag()) {
case AGGREGATE: {
if (BuiltinFunctions.isAggregateFunctionSerializable(fd.getIdentifier())) {
AggregateFunctionCallExpression serialAggExpr = BuiltinFunctions
.makeSerializableAggregateFunctionExpression(fd.getIdentifier(), expr.getArguments());
- IFunctionDescriptor afdd = getFunctionDescriptor(serialAggExpr, env, context);
+ IFunctionDescriptor afdd = resolveFunction(serialAggExpr, env);
return afdd.createSerializableAggregateEvaluatorFactory(args);
} else {
throw new AlgebricksException(
@@ -194,10 +192,14 @@ public class QueryLogicalExpressionJobGen implements ILogicalExpressionJobGen {
}
}
- private IFunctionDescriptor getFunctionDescriptor(AbstractFunctionCallExpression expr, IVariableTypeEnvironment env,
- JobGenContext context) throws AlgebricksException {
- IFunctionDescriptor fd = FormatUtils.getDefaultFormat().resolveFunction(expr, env);
+ private IFunctionDescriptor resolveFunction(ILogicalExpression expr, IVariableTypeEnvironment env)
+ throws AlgebricksException {
+ FunctionIdentifier fnId = ((AbstractFunctionCallExpression) expr).getFunctionIdentifier();
+ IFunctionDescriptor fd = functionManager.lookupFunction(fnId);
+ IFunctionTypeInferer fnTypeInfer = functionManager.lookupFunctionTypeInferer(fnId);
+ if (fnTypeInfer != null) {
+ fnTypeInfer.infer(expr, fd, env);
+ }
return fd;
}
-
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java
index 5ef41c4..f7695ea 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java
@@ -36,9 +36,11 @@ import org.apache.asterix.formats.nontagged.BinaryIntegerInspector;
import org.apache.asterix.formats.nontagged.SerializerDeserializerProvider;
import org.apache.asterix.formats.nontagged.TypeTraitProvider;
import org.apache.asterix.jobgen.QueryLogicalExpressionJobGen;
+import org.apache.asterix.metadata.declared.MetadataProvider;
import org.apache.asterix.om.base.IAObject;
import org.apache.asterix.om.constants.AsterixConstantValue;
import org.apache.asterix.om.functions.BuiltinFunctions;
+import org.apache.asterix.om.functions.IFunctionExtensionManager;
import org.apache.asterix.om.typecomputer.base.TypeCastUtils;
import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.types.ATypeTag;
@@ -125,12 +127,13 @@ public class ConstantFoldingRule implements IAlgebraicRewriteRule {
private static final IOperatorSchema[] _emptySchemas = new IOperatorSchema[] {};
public ConstantFoldingRule(ICcApplicationContext appCtx) {
- jobGenCtx = new JobGenContext(null, null, appCtx, SerializerDeserializerProvider.INSTANCE,
+ MetadataProvider metadataProvider = new MetadataProvider(appCtx, null);
+ jobGenCtx = new JobGenContext(null, metadataProvider, appCtx, SerializerDeserializerProvider.INSTANCE,
BinaryHashFunctionFactoryProvider.INSTANCE, BinaryHashFunctionFamilyProvider.INSTANCE,
BinaryComparatorFactoryProvider.INSTANCE, TypeTraitProvider.INSTANCE, BinaryBooleanInspector.FACTORY,
BinaryIntegerInspector.FACTORY, ADMPrinterFactoryProvider.INSTANCE, MissingWriterFactory.INSTANCE, null,
- new ExpressionRuntimeProvider(QueryLogicalExpressionJobGen.INSTANCE), ExpressionTypeComputer.INSTANCE,
- null, null, null, null, GlobalConfig.DEFAULT_FRAME_SIZE, null);
+ new ExpressionRuntimeProvider(new QueryLogicalExpressionJobGen(metadataProvider.getFunctionManager())),
+ ExpressionTypeComputer.INSTANCE, null, null, null, null, GlobalConfig.DEFAULT_FRAME_SIZE, null);
}
@Override
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
index 8290446..2078288 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
@@ -263,7 +263,7 @@ public class APIFramework {
builder.setPhysicalOptimizationConfig(OptimizationConfUtil.getPhysicalOptimizationConfig());
builder.setLogicalRewrites(ruleSetFactory.getLogicalRewrites(metadataProvider.getApplicationContext()));
builder.setPhysicalRewrites(ruleSetFactory.getPhysicalRewrites(metadataProvider.getApplicationContext()));
- IDataFormat format = metadataProvider.getFormat();
+ IDataFormat format = metadataProvider.getDataFormat();
ICompilerFactory compilerFactory = builder.create();
builder.setExpressionEvalSizeComputer(format.getExpressionEvalSizeComputer());
builder.setIMergeAggregationExpressionFactory(new MergeAggregationExpressionFactory());
@@ -321,7 +321,8 @@ public class APIFramework {
builder.setBinaryBooleanInspectorFactory(format.getBinaryBooleanInspectorFactory());
builder.setBinaryIntegerInspectorFactory(format.getBinaryIntegerInspectorFactory());
builder.setComparatorFactoryProvider(format.getBinaryComparatorFactoryProvider());
- builder.setExpressionRuntimeProvider(new ExpressionRuntimeProvider(QueryLogicalExpressionJobGen.INSTANCE));
+ builder.setExpressionRuntimeProvider(
+ new ExpressionRuntimeProvider(new QueryLogicalExpressionJobGen(metadataProvider.getFunctionManager())));
builder.setHashFunctionFactoryProvider(format.getBinaryHashFunctionFactoryProvider());
builder.setHashFunctionFamilyProvider(format.getBinaryHashFunctionFamilyProvider());
builder.setMissingWriterFactory(format.getMissingWriterFactory());
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/CCExtensionManager.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/CCExtensionManager.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/CCExtensionManager.java
index 0c6b2cc..768416d 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/CCExtensionManager.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/CCExtensionManager.java
@@ -18,9 +18,9 @@
*/
package org.apache.asterix.app.cc;
-import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
-import java.util.Map;
+import java.util.Set;
import java.util.concurrent.ExecutorService;
import org.apache.asterix.algebra.base.ILangExtension;
@@ -35,22 +35,25 @@ import org.apache.asterix.common.exceptions.RuntimeDataException;
import org.apache.asterix.compiler.provider.AqlCompilationProvider;
import org.apache.asterix.compiler.provider.ILangCompilationProvider;
import org.apache.asterix.compiler.provider.SqlppCompilationProvider;
+import org.apache.asterix.om.functions.IFunctionExtensionManager;
+import org.apache.asterix.om.functions.IFunctionManager;
+import org.apache.asterix.runtime.functions.FunctionCollection;
+import org.apache.asterix.runtime.functions.FunctionManager;
import org.apache.asterix.translator.IStatementExecutorFactory;
import org.apache.asterix.utils.ExtensionUtil;
import org.apache.hyracks.algebricks.common.utils.Pair;
import org.apache.hyracks.api.exceptions.HyracksDataException;
/**
- * AsterixDB's implementation of {@code IAlgebraExtensionManager} which takes care of
- * initializing extensions for App and Compilation purposes
+ * AsterixDB's implementation of {@code IAlgebraExtensionManager} and {@code IFunctionExtensionManager}
+ * which takes care of initializing extensions for App and Compilation purposes
*/
-public class CCExtensionManager implements IAlgebraExtensionManager {
-
- private final Map<ExtensionId, IExtension> extensions = new HashMap<>();
+public class CCExtensionManager implements IAlgebraExtensionManager, IFunctionExtensionManager {
private final IStatementExecutorExtension statementExecutorExtension;
private final ILangCompilationProvider aqlCompilationProvider;
private final ILangCompilationProvider sqlppCompilationProvider;
+ private final IFunctionManager functionManager;
private transient IStatementExecutorFactory statementExecutorFactory;
/**
@@ -67,15 +70,16 @@ public class CCExtensionManager implements IAlgebraExtensionManager {
throws InstantiationException, IllegalAccessException, ClassNotFoundException, HyracksDataException {
Pair<ExtensionId, ILangCompilationProvider> aqlcp = null;
Pair<ExtensionId, ILangCompilationProvider> sqlppcp = null;
+ Pair<ExtensionId, IFunctionManager> fm = null;
IStatementExecutorExtension see = null;
if (list != null) {
+ Set<ExtensionId> extensionIds = new HashSet<>();
for (AsterixExtension extensionConf : list) {
IExtension extension = (IExtension) Class.forName(extensionConf.getClassName()).newInstance();
extension.configure(extensionConf.getArgs());
- if (extensions.containsKey(extension.getId())) {
+ if (!extensionIds.add(extension.getId())) {
throw new RuntimeDataException(ErrorCode.EXTENSION_ID_CONFLICT, extension.getId());
}
- extensions.put(extension.getId(), extension);
switch (extension.getExtensionKind()) {
case STATEMENT_EXECUTOR:
see = ExtensionUtil.extendStatementExecutor(see, (IStatementExecutorExtension) extension);
@@ -84,6 +88,7 @@ public class CCExtensionManager implements IAlgebraExtensionManager {
ILangExtension le = (ILangExtension) extension;
aqlcp = ExtensionUtil.extendLangCompilationProvider(Language.AQL, aqlcp, le);
sqlppcp = ExtensionUtil.extendLangCompilationProvider(Language.SQLPP, sqlppcp, le);
+ fm = ExtensionUtil.extendFunctionManager(fm, le);
break;
default:
break;
@@ -93,6 +98,8 @@ public class CCExtensionManager implements IAlgebraExtensionManager {
this.statementExecutorExtension = see;
this.aqlCompilationProvider = aqlcp == null ? new AqlCompilationProvider() : aqlcp.second;
this.sqlppCompilationProvider = sqlppcp == null ? new SqlppCompilationProvider() : sqlppcp.second;
+ this.functionManager =
+ fm == null ? new FunctionManager(FunctionCollection.createDefaultFunctionCollection()) : fm.second;
}
/** @deprecated use getStatementExecutorFactory instead */
@@ -117,4 +124,9 @@ public class CCExtensionManager implements IAlgebraExtensionManager {
default: throw new IllegalArgumentException(String.valueOf(lang));
}
}
+
+ @Override
+ public IFunctionManager getFunctionManager() {
+ return functionManager;
+ }
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/ExtensionUtil.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/ExtensionUtil.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/ExtensionUtil.java
index 07eed0d..380c5a9 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/ExtensionUtil.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/ExtensionUtil.java
@@ -26,6 +26,7 @@ import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.common.exceptions.RuntimeDataException;
import org.apache.asterix.compiler.provider.ILangCompilationProvider;
import org.apache.asterix.metadata.api.IMetadataExtension;
+import org.apache.asterix.om.functions.IFunctionManager;
import org.apache.asterix.translator.IStatementExecutorFactory;
import org.apache.hyracks.algebricks.common.utils.Pair;
@@ -52,18 +53,39 @@ public class ExtensionUtil {
*/
public static Pair<ExtensionId, ILangCompilationProvider> extendLangCompilationProvider(Language lang,
Pair<ExtensionId, ILangCompilationProvider> cp, ILangExtension le) throws RuntimeDataException {
- if (cp != null && le.getLangCompilationProvider(lang) != null) {
+ ILangCompilationProvider lecp = le.getLangCompilationProvider(lang);
+ if (cp != null && lecp != null) {
throw new RuntimeDataException(ErrorCode.EXTENSION_COMPONENT_CONFLICT, le.getId(), cp.first,
lang.toString());
}
- return (le.getLangCompilationProvider(lang) != null)
- ? new Pair<>(le.getId(), le.getLangCompilationProvider(lang)) : cp;
+ return lecp != null ? new Pair<>(le.getId(), lecp) : cp;
+ }
+
+ /**
+ * Validate no extension conflict and return function manager extension
+ *
+ * @param fm
+ * place holder for function manager extension
+ * @param le
+ * user defined extension
+ * @return the user defined extension
+ * @throws RuntimeDataException
+ * if extension conflict was detected
+ */
+ public static Pair<ExtensionId, IFunctionManager> extendFunctionManager(Pair<ExtensionId, IFunctionManager> fm,
+ ILangExtension le) throws RuntimeDataException {
+ IFunctionManager lefm = le.getFunctionManager();
+ if (fm != null && lefm != null) {
+ throw new RuntimeDataException(ErrorCode.EXTENSION_COMPONENT_CONFLICT, le.getId(), fm.first,
+ IFunctionManager.class.getSimpleName());
+ }
+ return lefm != null ? new Pair<>(le.getId(), lefm) : fm;
}
/**
* Validate no extension conflict and return statement executor extension
*
- * @param qte
+ * @param see
* place holder for statement executor extension
* @param extension
* user defined extension
@@ -71,10 +93,10 @@ public class ExtensionUtil {
* @throws RuntimeDataException
* if extension conflict was detected
*/
- public static IStatementExecutorExtension extendStatementExecutor(IStatementExecutorExtension qte,
+ public static IStatementExecutorExtension extendStatementExecutor(IStatementExecutorExtension see,
IStatementExecutorExtension extension) throws RuntimeDataException {
- if (qte != null) {
- throw new RuntimeDataException(ErrorCode.EXTENSION_COMPONENT_CONFLICT, qte.getId(), extension.getId(),
+ if (see != null) {
+ throw new RuntimeDataException(ErrorCode.EXTENSION_COMPONENT_CONFLICT, see.getId(), extension.getId(),
IStatementExecutorFactory.class.getSimpleName());
}
return extension;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/ExceptionIT.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/ExceptionIT.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/ExceptionIT.java
index 3c54c7c..4bcee04 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/ExceptionIT.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/ExceptionIT.java
@@ -43,7 +43,8 @@ public class ExceptionIT {
@Test
public void test() throws Exception {
- List<IFunctionDescriptorFactory> functions = FunctionCollection.getFunctionDescriptorFactories();
+ List<IFunctionDescriptorFactory> functions =
+ FunctionCollection.createDefaultFunctionCollection().getFunctionDescriptorFactories();
int testedFunctions = 0;
for (IFunctionDescriptorFactory func : functions) {
String className = func.getClass().getName();
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/NullMissingTest.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/NullMissingTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/NullMissingTest.java
index f5e7402..3b33868 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/NullMissingTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/NullMissingTest.java
@@ -43,7 +43,8 @@ public class NullMissingTest {
@Test
public void test() throws Exception {
- List<IFunctionDescriptorFactory> functions = FunctionCollection.getFunctionDescriptorFactories();
+ List<IFunctionDescriptorFactory> functions =
+ FunctionCollection.createDefaultFunctionCollection().getFunctionDescriptorFactories();
int testedFunctions = 0;
for (IFunctionDescriptorFactory func : functions) {
String className = func.getClass().getName();
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/active/ActiveEventsListenerTest.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/active/ActiveEventsListenerTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/active/ActiveEventsListenerTest.java
index b1191ec..40d4f6a 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/active/ActiveEventsListenerTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/active/ActiveEventsListenerTest.java
@@ -45,6 +45,9 @@ import org.apache.asterix.metadata.declared.MetadataProvider;
import org.apache.asterix.metadata.entities.Dataset;
import org.apache.asterix.metadata.entities.Feed;
import org.apache.asterix.metadata.lock.MetadataLockManager;
+import org.apache.asterix.om.functions.IFunctionExtensionManager;
+import org.apache.asterix.runtime.functions.FunctionCollection;
+import org.apache.asterix.runtime.functions.FunctionManager;
import org.apache.asterix.runtime.utils.CcApplicationContext;
import org.apache.asterix.test.active.TestEventsListener.Behavior;
import org.apache.asterix.test.base.TestMethodTracer;
@@ -82,6 +85,7 @@ public class ActiveEventsListenerTest {
static CcApplicationContext appCtx;
static IStatementExecutor statementExecutor;
static IHyracksClientConnection hcc;
+ static IFunctionExtensionManager functionExtensionManager;
static MetadataProvider metadataProvider;
static IStorageComponentProvider componentProvider;
static JobIdFactory jobIdFactory;
@@ -121,6 +125,10 @@ public class ActiveEventsListenerTest {
Mockito.when(ccServiceCtx.getControllerService()).thenReturn(ccService);
Mockito.when(ccService.getExecutor()).thenReturn(executor);
locations = new AlgebricksAbsolutePartitionConstraint(nodes);
+ functionExtensionManager = Mockito.mock(IFunctionExtensionManager.class);
+ Mockito.when(functionExtensionManager.getFunctionManager())
+ .thenReturn(new FunctionManager(FunctionCollection.createDefaultFunctionCollection()));
+ Mockito.when(appCtx.getExtensionManager()).thenReturn(functionExtensionManager);
metadataProvider = new MetadataProvider(appCtx, null);
clusterController = new TestClusterControllerActor("CC", handler, allDatasets);
nodeControllers = new TestNodeControllerActor[2];
@@ -1449,6 +1457,10 @@ public class ActiveEventsListenerTest {
CcApplicationContext ccAppCtx = Mockito.mock(CcApplicationContext.class);
IStatementExecutor statementExecutor = Mockito.mock(IStatementExecutor.class);
IHyracksClientConnection hcc = Mockito.mock(IHyracksClientConnection.class);
+ IFunctionExtensionManager functionExtensionManager = Mockito.mock(IFunctionExtensionManager.class);
+ Mockito.when(functionExtensionManager.getFunctionManager())
+ .thenReturn(new FunctionManager(FunctionCollection.createDefaultFunctionCollection()));
+ Mockito.when(ccAppCtx.getExtensionManager()).thenReturn(functionExtensionManager);
Mockito.when(ccAppCtx.getActiveNotificationHandler()).thenReturn(handler);
Mockito.when(ccAppCtx.getMetadataLockManager()).thenReturn(lockManager);
Mockito.when(ccAppCtx.getServiceContext()).thenReturn(ccServiceCtx);
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java
index e2b1761..3f3a27f 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java
@@ -31,7 +31,6 @@ import org.apache.asterix.metadata.entities.InternalDatasetDetails;
import org.apache.asterix.metadata.utils.KeyFieldTypeUtil;
import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.types.IAType;
-import org.apache.asterix.runtime.formats.NonTaggedDataFormat;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.common.utils.Pair;
@@ -107,8 +106,7 @@ public class DatasetDataSource extends DataSource {
ExternalDatasetDetails edd = (ExternalDatasetDetails) externalDataset.getDatasetDetails();
IAdapterFactory adapterFactory = metadataProvider.getConfiguredAdapterFactory(externalDataset,
edd.getAdapter(), edd.getProperties(), (ARecordType) itemType, null);
- return metadataProvider.buildExternalDatasetDataScannerRuntime(jobSpec, itemType, adapterFactory,
- NonTaggedDataFormat.INSTANCE);
+ return metadataProvider.buildExternalDatasetDataScannerRuntime(jobSpec, itemType, adapterFactory);
case INTERNAL:
DataSourceId id = getId();
String dataverseName = id.getDataverseName();
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FeedDataSource.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FeedDataSource.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FeedDataSource.java
index 0b6608c..325d23b 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FeedDataSource.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FeedDataSource.java
@@ -33,7 +33,6 @@ import org.apache.asterix.metadata.entities.FeedPolicyEntity;
import org.apache.asterix.metadata.feeds.BuiltinFeedPolicies;
import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.types.IAType;
-import org.apache.asterix.runtime.formats.NonTaggedDataFormat;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
@@ -170,7 +169,7 @@ public class FeedDataSource extends DataSource implements IMutationDataSource {
try {
ARecordType feedOutputType = (ARecordType) itemType;
ISerializerDeserializer payloadSerde =
- NonTaggedDataFormat.INSTANCE.getSerdeProvider().getSerializerDeserializer(feedOutputType);
+ metadataProvider.getDataFormat().getSerdeProvider().getSerializerDeserializer(feedOutputType);
ArrayList<ISerializerDeserializer> serdes = new ArrayList<>();
serdes.add(payloadSerde);
if (metaItemType != null) {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
index 9271f33..2f94ad7 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
@@ -57,6 +57,7 @@ import org.apache.asterix.external.provider.AdapterFactoryProvider;
import org.apache.asterix.external.util.ExternalDataConstants;
import org.apache.asterix.external.util.FeedConstants;
import org.apache.asterix.formats.base.IDataFormat;
+import org.apache.asterix.om.functions.IFunctionExtensionManager;
import org.apache.asterix.formats.nontagged.BinaryComparatorFactoryProvider;
import org.apache.asterix.formats.nontagged.LinearizeComparatorFactoryProvider;
import org.apache.asterix.formats.nontagged.TypeTraitProvider;
@@ -78,6 +79,7 @@ import org.apache.asterix.metadata.utils.DatasetUtil;
import org.apache.asterix.metadata.utils.MetadataConstants;
import org.apache.asterix.metadata.utils.SplitsAndConstraintsUtil;
import org.apache.asterix.om.functions.BuiltinFunctions;
+import org.apache.asterix.om.functions.IFunctionManager;
import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.types.ATypeTag;
import org.apache.asterix.om.types.IAType;
@@ -143,6 +145,7 @@ public class MetadataProvider implements IMetadataProvider<DataSourceId, String>
private final ICcApplicationContext appCtx;
private final IStorageComponentProvider storageComponentProvider;
private final StorageProperties storageProperties;
+ private final IFunctionManager functionManager;
private final Dataverse defaultDataverse;
private final LockList locks;
private final Map<String, String> config;
@@ -164,6 +167,7 @@ public class MetadataProvider implements IMetadataProvider<DataSourceId, String>
this.defaultDataverse = defaultDataverse == null ? MetadataBuiltinEntities.DEFAULT_DATAVERSE : defaultDataverse;
this.storageComponentProvider = appCtx.getStorageComponentProvider();
storageProperties = appCtx.getStorageProperties();
+ functionManager = ((IFunctionExtensionManager) appCtx.getExtensionManager()).getFunctionManager();
locks = new LockList();
config = new HashMap<>();
}
@@ -259,7 +263,11 @@ public class MetadataProvider implements IMetadataProvider<DataSourceId, String>
return isTemporaryDatasetWriteJob;
}
- public IDataFormat getFormat() {
+ public IFunctionManager getFunctionManager() {
+ return functionManager;
+ }
+
+ public IDataFormat getDataFormat() {
return FormatUtils.getDefaultFormat();
}
@@ -868,13 +876,14 @@ public class MetadataProvider implements IMetadataProvider<DataSourceId, String>
}
public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> buildExternalDatasetDataScannerRuntime(
- JobSpecification jobSpec, IAType itemType, IAdapterFactory adapterFactory, IDataFormat format)
+ JobSpecification jobSpec, IAType itemType, IAdapterFactory adapterFactory)
throws AlgebricksException {
if (itemType.getTypeTag() != ATypeTag.OBJECT) {
throw new AlgebricksException("Can only scan datasets of records.");
}
- ISerializerDeserializer<?> payloadSerde = format.getSerdeProvider().getSerializerDeserializer(itemType);
+ ISerializerDeserializer<?> payloadSerde =
+ getDataFormat().getSerdeProvider().getSerializerDeserializer(itemType);
RecordDescriptor scannerDesc = new RecordDescriptor(new ISerializerDeserializer[] { payloadSerde });
ExternalScanOperatorDescriptor dataScanner =
@@ -1501,7 +1510,7 @@ public class MetadataProvider implements IMetadataProvider<DataSourceId, String>
// Generate Output Record format
ISerializerDeserializer<?>[] tokenKeyPairFields = new ISerializerDeserializer[numTokenKeyPairFields];
ITypeTraits[] tokenKeyPairTypeTraits = new ITypeTraits[numTokenKeyPairFields];
- ISerializerDeserializerProvider serdeProvider = FormatUtils.getDefaultFormat().getSerdeProvider();
+ ISerializerDeserializerProvider serdeProvider = getDataFormat().getSerdeProvider();
// The order of the output record: propagated variables (including
// PK and SK), token, and number of token.
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java
index 9156b0f..8e1c34d 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java
@@ -730,7 +730,7 @@ public class Dataset implements IMetadataEntity<Dataset>, IDataset {
ISerializerDeserializer[] primaryRecFields =
new ISerializerDeserializer[numPrimaryKeys + 1 + (hasMetaPart() ? 1 : 0)];
ITypeTraits[] primaryTypeTraits = new ITypeTraits[numPrimaryKeys + 1 + (hasMetaPart() ? 1 : 0)];
- ISerializerDeserializerProvider serdeProvider = metadataProvider.getFormat().getSerdeProvider();
+ ISerializerDeserializerProvider serdeProvider = metadataProvider.getDataFormat().getSerdeProvider();
List<Integer> indicators = null;
if (hasMetaPart()) {
indicators = ((InternalDatasetDetails) getDatasetDetails()).getKeySourceIndicator();
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
index e4a6ca8..53c3dc0 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
@@ -41,6 +41,7 @@ import org.apache.asterix.common.exceptions.MetadataException;
import org.apache.asterix.common.transactions.IRecoveryManager;
import org.apache.asterix.common.transactions.JobId;
import org.apache.asterix.external.indexing.IndexingConstants;
+import org.apache.asterix.formats.base.IDataFormat;
import org.apache.asterix.formats.nontagged.SerializerDeserializerProvider;
import org.apache.asterix.formats.nontagged.TypeTraitProvider;
import org.apache.asterix.metadata.MetadataManager;
@@ -57,7 +58,6 @@ import org.apache.asterix.om.base.AString;
import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.types.BuiltinType;
import org.apache.asterix.om.types.IAType;
-import org.apache.asterix.runtime.formats.FormatUtils;
import org.apache.asterix.runtime.job.listener.JobEventListenerFactory;
import org.apache.asterix.runtime.operators.LSMPrimaryUpsertOperatorDescriptor;
import org.apache.asterix.runtime.utils.RuntimeUtils;
@@ -419,16 +419,16 @@ public class DatasetUtil {
+ (dataset.hasMetaPart() ? 2 : 1) + numFilterFields];
ISerializerDeserializer<?>[] outputSerDes = new ISerializerDeserializer[inputRecordDesc.getFieldCount()
+ (dataset.hasMetaPart() ? 2 : 1) + numFilterFields];
+ IDataFormat dataFormat = metadataProvider.getDataFormat();
// add the previous record first
int f = 0;
- outputSerDes[f] = FormatUtils.getDefaultFormat().getSerdeProvider().getSerializerDeserializer(itemType);
+ outputSerDes[f] = dataFormat.getSerdeProvider().getSerializerDeserializer(itemType);
f++;
// add the previous meta second
if (dataset.hasMetaPart()) {
- outputSerDes[f] =
- FormatUtils.getDefaultFormat().getSerdeProvider().getSerializerDeserializer(metaItemType);
- outputTypeTraits[f] = FormatUtils.getDefaultFormat().getTypeTraitProvider().getTypeTrait(metaItemType);
+ outputSerDes[f] = dataFormat.getSerdeProvider().getSerializerDeserializer(metaItemType);
+ outputTypeTraits[f] = dataFormat.getTypeTraitProvider().getTypeTrait(metaItemType);
f++;
}
// add the previous filter third
@@ -443,10 +443,10 @@ public class DatasetUtil {
}
}
fieldIdx = i;
- outputTypeTraits[f] = FormatUtils.getDefaultFormat().getTypeTraitProvider()
- .getTypeTrait(itemType.getFieldTypes()[fieldIdx]);
- outputSerDes[f] = FormatUtils.getDefaultFormat().getSerdeProvider()
- .getSerializerDeserializer(itemType.getFieldTypes()[fieldIdx]);
+ outputTypeTraits[f] =
+ dataFormat.getTypeTraitProvider().getTypeTrait(itemType.getFieldTypes()[fieldIdx]);
+ outputSerDes[f] =
+ dataFormat.getSerdeProvider().getSerializerDeserializer(itemType.getFieldTypes()[fieldIdx]);
f++;
}
for (int j = 0; j < inputRecordDesc.getFieldCount(); j++) {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java
index 445fbd8..78d9c19 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java
@@ -212,10 +212,10 @@ public class SecondaryBTreeOperationsHelper extends SecondaryTreeIndexOperations
ITypeTraits[] enforcedTypeTraits =
new ITypeTraits[1 + numPrimaryKeys + (dataset.hasMetaPart() ? 1 : 0) + numFilterFields];
secondaryTypeTraits = new ITypeTraits[numSecondaryKeys + numPrimaryKeys];
- ISerializerDeserializerProvider serdeProvider = metadataProvider.getFormat().getSerdeProvider();
- ITypeTraitProvider typeTraitProvider = metadataProvider.getFormat().getTypeTraitProvider();
+ ISerializerDeserializerProvider serdeProvider = metadataProvider.getDataFormat().getSerdeProvider();
+ ITypeTraitProvider typeTraitProvider = metadataProvider.getDataFormat().getTypeTraitProvider();
IBinaryComparatorFactoryProvider comparatorFactoryProvider =
- metadataProvider.getFormat().getBinaryComparatorFactoryProvider();
+ metadataProvider.getDataFormat().getBinaryComparatorFactoryProvider();
// Record column is 0 for external datasets, numPrimaryKeys for internal ones
int recordColumn = dataset.getDatasetType() == DatasetType.INTERNAL ? numPrimaryKeys : 0;
boolean isOverridingKeyFieldTypes = index.isOverridingKeyFieldTypes();
@@ -230,9 +230,9 @@ public class SecondaryBTreeOperationsHelper extends SecondaryTreeIndexOperations
sourceType = metaType;
sourceColumn = recordColumn + 1;
}
- secondaryFieldAccessEvalFactories[i] = metadataProvider.getFormat().getFieldAccessEvaluatorFactory(
- isOverridingKeyFieldTypes ? enforcedItemType : sourceType, index.getKeyFieldNames().get(i),
- sourceColumn);
+ secondaryFieldAccessEvalFactories[i] = metadataProvider.getDataFormat().getFieldAccessEvaluatorFactory(
+ metadataProvider.getFunctionManager(), isOverridingKeyFieldTypes ? enforcedItemType : sourceType,
+ index.getKeyFieldNames().get(i), sourceColumn);
Pair<IAType, Boolean> keyTypePair = Index.getNonNullableOpenFieldType(index.getKeyFieldTypes().get(i),
index.getKeyFieldNames().get(i), sourceType);
IAType keyType = keyTypePair.first;
@@ -270,8 +270,9 @@ public class SecondaryBTreeOperationsHelper extends SecondaryTreeIndexOperations
}
if (numFilterFields > 0) {
- secondaryFieldAccessEvalFactories[numSecondaryKeys] = metadataProvider.getFormat()
- .getFieldAccessEvaluatorFactory(itemType, filterFieldName, numPrimaryKeys);
+ secondaryFieldAccessEvalFactories[numSecondaryKeys] =
+ metadataProvider.getDataFormat().getFieldAccessEvaluatorFactory(
+ metadataProvider.getFunctionManager(), itemType, filterFieldName, numPrimaryKeys);
Pair<IAType, Boolean> keyTypePair = Index.getNonNullableKeyFieldType(filterFieldName, itemType);
IAType type = keyTypePair.first;
ISerializerDeserializer serde = serdeProvider.getSerializerDeserializer(type);
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedBTreeOperationsHelper.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedBTreeOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedBTreeOperationsHelper.java
index 96fdf27..8ef5f34 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedBTreeOperationsHelper.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedBTreeOperationsHelper.java
@@ -137,10 +137,10 @@ public class SecondaryCorrelatedBTreeOperationsHelper extends SecondaryCorrelate
ITypeTraits[] enforcedTypeTraits =
new ITypeTraits[1 + numPrimaryKeys + (dataset.hasMetaPart() ? 1 : 0) + numFilterFields];
secondaryTypeTraits = new ITypeTraits[numSecondaryKeys + numPrimaryKeys];
- ISerializerDeserializerProvider serdeProvider = metadataProvider.getFormat().getSerdeProvider();
- ITypeTraitProvider typeTraitProvider = metadataProvider.getFormat().getTypeTraitProvider();
+ ISerializerDeserializerProvider serdeProvider = metadataProvider.getDataFormat().getSerdeProvider();
+ ITypeTraitProvider typeTraitProvider = metadataProvider.getDataFormat().getTypeTraitProvider();
IBinaryComparatorFactoryProvider comparatorFactoryProvider =
- metadataProvider.getFormat().getBinaryComparatorFactoryProvider();
+ metadataProvider.getDataFormat().getBinaryComparatorFactoryProvider();
// Record column is 0 for external datasets, numPrimaryKeys for internal ones
int recordColumn = NUM_TAG_FIELDS + numPrimaryKeys;
boolean isOverridingKeyTypes = index.isOverridingKeyFieldTypes();
@@ -155,9 +155,9 @@ public class SecondaryCorrelatedBTreeOperationsHelper extends SecondaryCorrelate
sourceType = metaType;
sourceColumn = recordColumn + 1;
}
- secondaryFieldAccessEvalFactories[i] = metadataProvider.getFormat().getFieldAccessEvaluatorFactory(
- isOverridingKeyTypes ? enforcedItemType : sourceType, index.getKeyFieldNames().get(i),
- sourceColumn);
+ secondaryFieldAccessEvalFactories[i] = metadataProvider.getDataFormat().getFieldAccessEvaluatorFactory(
+ metadataProvider.getFunctionManager(), isOverridingKeyTypes ? enforcedItemType : sourceType,
+ index.getKeyFieldNames().get(i), sourceColumn);
Pair<IAType, Boolean> keyTypePair = Index.getNonNullableOpenFieldType(index.getKeyFieldTypes().get(i),
index.getKeyFieldNames().get(i), sourceType);
IAType keyType = keyTypePair.first;
@@ -185,8 +185,9 @@ public class SecondaryCorrelatedBTreeOperationsHelper extends SecondaryCorrelate
}
if (numFilterFields > 0) {
- secondaryFieldAccessEvalFactories[numSecondaryKeys] = metadataProvider.getFormat()
- .getFieldAccessEvaluatorFactory(itemType, filterFieldName, recordColumn);
+ secondaryFieldAccessEvalFactories[numSecondaryKeys] =
+ metadataProvider.getDataFormat().getFieldAccessEvaluatorFactory(
+ metadataProvider.getFunctionManager(), itemType, filterFieldName, recordColumn);
Pair<IAType, Boolean> keyTypePair = Index.getNonNullableKeyFieldType(filterFieldName, itemType);
IAType type = keyTypePair.first;
ISerializerDeserializer serde = serdeProvider.getSerializerDeserializer(type);
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedInvertedIndexOperationsHelper.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedInvertedIndexOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedInvertedIndexOperationsHelper.java
index 15f8a23..262b259 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedInvertedIndexOperationsHelper.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedInvertedIndexOperationsHelper.java
@@ -29,7 +29,6 @@ import org.apache.asterix.metadata.entities.Index;
import org.apache.asterix.om.types.IAType;
import org.apache.asterix.om.utils.NonTaggedFormatUtil;
import org.apache.asterix.om.utils.RecordUtil;
-import org.apache.asterix.runtime.formats.FormatUtils;
import org.apache.asterix.runtime.utils.RuntimeUtils;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraintHelper;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
@@ -108,13 +107,13 @@ public class SecondaryCorrelatedInvertedIndexOperationsHelper extends SecondaryC
ISerializerDeserializer[] enforcedRecFields = new ISerializerDeserializer[1 + numPrimaryKeys + numFilterFields];
secondaryTypeTraits = new ITypeTraits[numSecondaryKeys + numPrimaryKeys];
ITypeTraits[] enforcedTypeTraits = new ITypeTraits[1 + numPrimaryKeys];
- ISerializerDeserializerProvider serdeProvider = FormatUtils.getDefaultFormat().getSerdeProvider();
- ITypeTraitProvider typeTraitProvider = FormatUtils.getDefaultFormat().getTypeTraitProvider();
+ ISerializerDeserializerProvider serdeProvider = metadataProvider.getDataFormat().getSerdeProvider();
+ ITypeTraitProvider typeTraitProvider = metadataProvider.getDataFormat().getTypeTraitProvider();
int recordColumn = NUM_TAG_FIELDS + numPrimaryKeys;
if (numSecondaryKeys > 0) {
- secondaryFieldAccessEvalFactories[0] = FormatUtils.getDefaultFormat().getFieldAccessEvaluatorFactory(
- isOverridingKeyFieldTypes ? enforcedItemType : itemType, index.getKeyFieldNames().get(0),
- recordColumn);
+ secondaryFieldAccessEvalFactories[0] = metadataProvider.getDataFormat().getFieldAccessEvaluatorFactory(
+ metadataProvider.getFunctionManager(), isOverridingKeyFieldTypes ? enforcedItemType : itemType,
+ index.getKeyFieldNames().get(0), recordColumn);
Pair<IAType, Boolean> keyTypePair = Index.getNonNullableOpenFieldType(index.getKeyFieldTypes().get(0),
index.getKeyFieldNames().get(0), itemType);
secondaryKeyType = keyTypePair.first;
@@ -124,8 +123,9 @@ public class SecondaryCorrelatedInvertedIndexOperationsHelper extends SecondaryC
secondaryTypeTraits[0] = typeTraitProvider.getTypeTrait(secondaryKeyType);
}
if (numFilterFields > 0) {
- secondaryFieldAccessEvalFactories[numSecondaryKeys] = FormatUtils.getDefaultFormat()
- .getFieldAccessEvaluatorFactory(itemType, filterFieldName, recordColumn);
+ secondaryFieldAccessEvalFactories[numSecondaryKeys] =
+ metadataProvider.getDataFormat().getFieldAccessEvaluatorFactory(
+ metadataProvider.getFunctionManager(), itemType, filterFieldName, recordColumn);
Pair<IAType, Boolean> keyTypePair = Index.getNonNullableKeyFieldType(filterFieldName, itemType);
IAType type = keyTypePair.first;
ISerializerDeserializer serde = serdeProvider.getSerializerDeserializer(type);
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedRTreeOperationsHelper.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedRTreeOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedRTreeOperationsHelper.java
index 8fd8a7a..b663b42 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedRTreeOperationsHelper.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedRTreeOperationsHelper.java
@@ -89,9 +89,9 @@ public class SecondaryCorrelatedRTreeOperationsHelper extends SecondaryCorrelate
int numDimensions = NonTaggedFormatUtil.getNumDimensions(spatialType.getTypeTag());
numNestedSecondaryKeyFields = numDimensions * 2;
int recordColumn = NUM_TAG_FIELDS + numPrimaryKeys;
- secondaryFieldAccessEvalFactories =
- metadataProvider.getFormat().createMBRFactory(isOverridingKeyFieldTypes ? enforcedItemType : itemType,
- secondaryKeyFields.get(0), recordColumn, numDimensions, filterFieldName, isPointMBR);
+ secondaryFieldAccessEvalFactories = metadataProvider.getDataFormat().createMBRFactory(
+ metadataProvider.getFunctionManager(), isOverridingKeyFieldTypes ? enforcedItemType : itemType,
+ secondaryKeyFields.get(0), recordColumn, numDimensions, filterFieldName, isPointMBR);
secondaryComparatorFactories = new IBinaryComparatorFactory[numNestedSecondaryKeyFields];
valueProviderFactories = new IPrimitiveValueProviderFactory[numNestedSecondaryKeyFields];
ISerializerDeserializer[] secondaryRecFields =
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedTreeIndexOperationsHelper.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedTreeIndexOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedTreeIndexOperationsHelper.java
index 171d72e..b99ae2f 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedTreeIndexOperationsHelper.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedTreeIndexOperationsHelper.java
@@ -32,7 +32,6 @@ import org.apache.asterix.metadata.entities.Dataset;
import org.apache.asterix.metadata.entities.Index;
import org.apache.asterix.om.functions.BuiltinFunctions;
import org.apache.asterix.om.functions.IFunctionDescriptor;
-import org.apache.asterix.runtime.functions.FunctionManagerHolder;
import org.apache.asterix.runtime.operators.LSMSecondaryIndexBulkLoadOperatorDescriptor;
import org.apache.asterix.runtime.operators.LSMSecondaryIndexCreationTupleProcessorOperatorDescriptor;
import org.apache.asterix.transaction.management.opcallbacks.PrimaryIndexInstantSearchOperationCallbackFactory;
@@ -155,10 +154,6 @@ public abstract class SecondaryCorrelatedTreeIndexOperationsHelper extends Secon
@Override
protected AlgebricksMetaOperatorDescriptor createCastOp(JobSpecification spec, DatasetType dsType,
boolean strictCast) throws AlgebricksException {
- IFunctionDescriptor castFuncDesc = FunctionManagerHolder.getFunctionManager()
- .lookupFunction(strictCast ? BuiltinFunctions.CAST_TYPE : BuiltinFunctions.CAST_TYPE_LAX);
- castFuncDesc.setImmutableStates(enforcedItemType, itemType);
-
int[] outColumns = new int[1];
// tags(2) + primary keys + record + meta part(?)
@@ -185,7 +180,7 @@ public abstract class SecondaryCorrelatedTreeIndexOperationsHelper extends Secon
IScalarEvaluatorFactory[] castEvalFact =
new IScalarEvaluatorFactory[] { new ColumnAccessEvalFactory(recordIdx) };
IScalarEvaluatorFactory[] sefs = new IScalarEvaluatorFactory[1];
- sefs[0] = castFuncDesc.createEvaluatorFactory(castEvalFact);
+ sefs[0] = createCastFunction(strictCast).createEvaluatorFactory(castEvalFact);
AssignRuntimeFactory castAssign = new AssignRuntimeFactory(outColumns, sefs, projectionList);
return new AlgebricksMetaOperatorDescriptor(spec, 1, 1, new IPushRuntimeFactory[] { castAssign },
new RecordDescriptor[] { getTaggedRecordDescriptor(enforcedRecDesc) });
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
index 98c47a2..5dac407 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
@@ -22,7 +22,6 @@ package org.apache.asterix.metadata.utils;
import static org.apache.hyracks.storage.am.common.dataflow.IndexDropOperatorDescriptor.DropOption;
import java.util.Collections;
-import java.util.EnumSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -52,7 +51,6 @@ import org.apache.asterix.om.types.IAType;
import org.apache.asterix.runtime.evaluators.functions.AndDescriptor;
import org.apache.asterix.runtime.evaluators.functions.IsUnknownDescriptor;
import org.apache.asterix.runtime.evaluators.functions.NotDescriptor;
-import org.apache.asterix.runtime.functions.FunctionManagerHolder;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraintHelper;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
@@ -239,7 +237,7 @@ public abstract class SecondaryIndexOperationsHelper {
ITypeTraits[] primaryTypeTraits = new ITypeTraits[numPrimaryKeys + 1 + (dataset.hasMetaPart() ? 1 : 0)];
primaryComparatorFactories = new IBinaryComparatorFactory[numPrimaryKeys];
primaryBloomFilterKeyFields = new int[numPrimaryKeys];
- ISerializerDeserializerProvider serdeProvider = metadataProvider.getFormat().getSerdeProvider();
+ ISerializerDeserializerProvider serdeProvider = metadataProvider.getDataFormat().getSerdeProvider();
List<Integer> indicators = null;
if (dataset.hasMetaPart()) {
indicators = ((InternalDatasetDetails) dataset.getDatasetDetails()).getKeySourceIndicator();
@@ -298,10 +296,6 @@ public abstract class SecondaryIndexOperationsHelper {
protected AlgebricksMetaOperatorDescriptor createCastOp(JobSpecification spec, DatasetType dsType,
boolean strictCast) throws AlgebricksException {
- IFunctionDescriptor castFuncDesc = FunctionManagerHolder.getFunctionManager()
- .lookupFunction(strictCast ? BuiltinFunctions.CAST_TYPE : BuiltinFunctions.CAST_TYPE_LAX);
- castFuncDesc.setImmutableStates(enforcedItemType, itemType);
-
int[] outColumns = new int[1];
int[] projectionList = new int[(dataset.hasMetaPart() ? 2 : 1) + numPrimaryKeys];
int recordIdx;
@@ -322,12 +316,19 @@ public abstract class SecondaryIndexOperationsHelper {
IScalarEvaluatorFactory[] castEvalFact =
new IScalarEvaluatorFactory[] { new ColumnAccessEvalFactory(recordIdx) };
IScalarEvaluatorFactory[] sefs = new IScalarEvaluatorFactory[1];
- sefs[0] = castFuncDesc.createEvaluatorFactory(castEvalFact);
+ sefs[0] = createCastFunction(strictCast).createEvaluatorFactory(castEvalFact);
AssignRuntimeFactory castAssign = new AssignRuntimeFactory(outColumns, sefs, projectionList);
return new AlgebricksMetaOperatorDescriptor(spec, 1, 1, new IPushRuntimeFactory[] { castAssign },
new RecordDescriptor[] { enforcedRecDesc });
}
+ protected IFunctionDescriptor createCastFunction(boolean strictCast) throws AlgebricksException {
+ IFunctionDescriptor castFuncDesc = metadataProvider.getFunctionManager()
+ .lookupFunction(strictCast ? BuiltinFunctions.CAST_TYPE : BuiltinFunctions.CAST_TYPE_LAX);
+ castFuncDesc.setImmutableStates(enforcedItemType, itemType);
+ return castFuncDesc;
+ }
+
protected ExternalSortOperatorDescriptor createSortOp(JobSpecification spec,
IBinaryComparatorFactory[] secondaryComparatorFactories, RecordDescriptor secondaryRecDesc) {
int[] sortFields = new int[secondaryComparatorFactories.length];
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryInvertedIndexOperationsHelper.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryInvertedIndexOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryInvertedIndexOperationsHelper.java
index b4d8a22..077e431 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryInvertedIndexOperationsHelper.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryInvertedIndexOperationsHelper.java
@@ -30,7 +30,6 @@ import org.apache.asterix.metadata.entities.Index;
import org.apache.asterix.om.types.IAType;
import org.apache.asterix.om.utils.NonTaggedFormatUtil;
import org.apache.asterix.om.utils.RecordUtil;
-import org.apache.asterix.runtime.formats.FormatUtils;
import org.apache.asterix.runtime.utils.RuntimeUtils;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraintHelper;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
@@ -111,12 +110,12 @@ public class SecondaryInvertedIndexOperationsHelper extends SecondaryTreeIndexOp
ISerializerDeserializer[] enforcedRecFields = new ISerializerDeserializer[1 + numPrimaryKeys + numFilterFields];
secondaryTypeTraits = new ITypeTraits[numSecondaryKeys + numPrimaryKeys];
ITypeTraits[] enforcedTypeTraits = new ITypeTraits[1 + numPrimaryKeys];
- ISerializerDeserializerProvider serdeProvider = FormatUtils.getDefaultFormat().getSerdeProvider();
- ITypeTraitProvider typeTraitProvider = FormatUtils.getDefaultFormat().getTypeTraitProvider();
+ ISerializerDeserializerProvider serdeProvider = metadataProvider.getDataFormat().getSerdeProvider();
+ ITypeTraitProvider typeTraitProvider = metadataProvider.getDataFormat().getTypeTraitProvider();
if (numSecondaryKeys > 0) {
- secondaryFieldAccessEvalFactories[0] = FormatUtils.getDefaultFormat().getFieldAccessEvaluatorFactory(
- isOverridingKeyFieldTypes ? enforcedItemType : itemType, index.getKeyFieldNames().get(0),
- numPrimaryKeys);
+ secondaryFieldAccessEvalFactories[0] = metadataProvider.getDataFormat().getFieldAccessEvaluatorFactory(
+ metadataProvider.getFunctionManager(), isOverridingKeyFieldTypes ? enforcedItemType : itemType,
+ index.getKeyFieldNames().get(0), numPrimaryKeys);
Pair<IAType, Boolean> keyTypePair = Index.getNonNullableOpenFieldType(index.getKeyFieldTypes().get(0),
index.getKeyFieldNames().get(0), itemType);
secondaryKeyType = keyTypePair.first;
@@ -126,8 +125,9 @@ public class SecondaryInvertedIndexOperationsHelper extends SecondaryTreeIndexOp
secondaryTypeTraits[0] = typeTraitProvider.getTypeTrait(secondaryKeyType);
}
if (numFilterFields > 0) {
- secondaryFieldAccessEvalFactories[numSecondaryKeys] = FormatUtils.getDefaultFormat()
- .getFieldAccessEvaluatorFactory(itemType, filterFieldName, numPrimaryKeys);
+ secondaryFieldAccessEvalFactories[numSecondaryKeys] =
+ metadataProvider.getDataFormat().getFieldAccessEvaluatorFactory(
+ metadataProvider.getFunctionManager(), itemType, filterFieldName, numPrimaryKeys);
Pair<IAType, Boolean> keyTypePair = Index.getNonNullableKeyFieldType(filterFieldName, itemType);
IAType type = keyTypePair.first;
ISerializerDeserializer serde = serdeProvider.getSerializerDeserializer(type);
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryRTreeOperationsHelper.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryRTreeOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryRTreeOperationsHelper.java
index 08ee0aa..1726470 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryRTreeOperationsHelper.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryRTreeOperationsHelper.java
@@ -99,9 +99,9 @@ public class SecondaryRTreeOperationsHelper extends SecondaryTreeIndexOperations
int numDimensions = NonTaggedFormatUtil.getNumDimensions(spatialType.getTypeTag());
numNestedSecondaryKeyFields = numDimensions * 2;
int recordColumn = dataset.getDatasetType() == DatasetType.INTERNAL ? numPrimaryKeys : 0;
- secondaryFieldAccessEvalFactories =
- metadataProvider.getFormat().createMBRFactory(isOverridingKeyFieldTypes ? enforcedItemType : itemType,
- secondaryKeyFields.get(0), recordColumn, numDimensions, filterFieldName, isPointMBR);
+ secondaryFieldAccessEvalFactories = metadataProvider.getDataFormat().createMBRFactory(
+ metadataProvider.getFunctionManager(), isOverridingKeyFieldTypes ? enforcedItemType : itemType,
+ secondaryKeyFields.get(0), recordColumn, numDimensions, filterFieldName, isPointMBR);
secondaryComparatorFactories = new IBinaryComparatorFactory[numNestedSecondaryKeyFields];
valueProviderFactories = new IPrimitiveValueProviderFactory[numNestedSecondaryKeyFields];
ISerializerDeserializer[] secondaryRecFields =
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-om/src/main/java/org/apache/asterix/formats/base/IDataFormat.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/formats/base/IDataFormat.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/formats/base/IDataFormat.java
index 7624a8a..678a864 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/formats/base/IDataFormat.java
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/formats/base/IDataFormat.java
@@ -20,16 +20,13 @@ package org.apache.asterix.formats.base;
import java.util.List;
-import org.apache.asterix.om.functions.IFunctionDescriptor;
-import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
+import org.apache.asterix.om.functions.IFunctionManager;
import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.types.IAType;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.common.utils.Triple;
-import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import org.apache.hyracks.algebricks.core.algebra.expressions.IAlgebricksConstantValue;
import org.apache.hyracks.algebricks.core.algebra.expressions.IExpressionEvalSizeComputer;
-import org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
import org.apache.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
import org.apache.hyracks.algebricks.data.IBinaryBooleanInspectorFactory;
import org.apache.hyracks.algebricks.data.IBinaryComparatorFactoryProvider;
@@ -69,18 +66,16 @@ public interface IDataFormat {
public IMissingWriterFactory getMissingWriterFactory();
public Triple<IScalarEvaluatorFactory, ScalarFunctionCallExpression, IAType> partitioningEvaluatorFactory(
- ARecordType recType, List<String> fldName) throws AlgebricksException;
+ IFunctionManager functionManager, ARecordType recType, List<String> fldName) throws AlgebricksException;
- public IScalarEvaluatorFactory getFieldAccessEvaluatorFactory(ARecordType recType, List<String> fldName,
- int recordColumn) throws AlgebricksException;
-
- public IFunctionDescriptor resolveFunction(ILogicalExpression expr, IVariableTypeEnvironment typeEnvironment)
- throws AlgebricksException;
+ public IScalarEvaluatorFactory getFieldAccessEvaluatorFactory(IFunctionManager functionManager, ARecordType recType,
+ List<String> fldName, int recordColumn) throws AlgebricksException;
public IScalarEvaluatorFactory getConstantEvalFactory(IAlgebricksConstantValue value) throws AlgebricksException;
- public IScalarEvaluatorFactory[] createMBRFactory(ARecordType recType, List<String> fldName, int recordColumn,
- int dimension, List<String> filterFieldName, boolean isPointMBR) throws AlgebricksException;
+ public IScalarEvaluatorFactory[] createMBRFactory(IFunctionManager functionManager, ARecordType recType,
+ List<String> fldName, int recordColumn, int dimension, List<String> filterFieldName, boolean isPointMBR)
+ throws AlgebricksException;
public IExpressionEvalSizeComputer getExpressionEvalSizeComputer();
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IFunctionDescriptorFactory.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IFunctionDescriptorFactory.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IFunctionDescriptorFactory.java
index 526eab6..0e746f9 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IFunctionDescriptorFactory.java
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IFunctionDescriptorFactory.java
@@ -26,5 +26,13 @@ public interface IFunctionDescriptorFactory {
*
* @return a new IFunctionDescriptor instance
*/
- public IFunctionDescriptor createFunctionDescriptor();
+ IFunctionDescriptor createFunctionDescriptor();
+
+ /**
+ * Creates type inferer for this function
+ * @return an new instance of the type inferer or {@code null} if not provided
+ */
+ default IFunctionTypeInferer createFunctionTypeInferer() {
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IFunctionExtensionManager.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IFunctionExtensionManager.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IFunctionExtensionManager.java
new file mode 100644
index 0000000..fb8155a
--- /dev/null
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IFunctionExtensionManager.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.om.functions;
+
+/**
+ * An extension point for built-in functions
+ */
+public interface IFunctionExtensionManager {
+ /**
+ * Returns function manager
+ *
+ * @return a new {@link IFunctionManager} instance
+ */
+ IFunctionManager getFunctionManager();
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IFunctionManager.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IFunctionManager.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IFunctionManager.java
index f7c11ff..e1657ff 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IFunctionManager.java
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IFunctionManager.java
@@ -21,11 +21,13 @@ package org.apache.asterix.om.functions;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
-public interface IFunctionManager extends Iterable<IFunctionDescriptorFactory> {
-
- public void registerFunction(IFunctionDescriptorFactory descriptorFactory);
+/**
+ * A registry of {@link IFunctionDescriptor} and {@link IFunctionTypeInferer}
+ * for built-in functions
+ */
+public interface IFunctionManager {
- public void unregisterFunction(IFunctionDescriptorFactory descriptorFactory);
+ IFunctionDescriptor lookupFunction(FunctionIdentifier fid) throws AlgebricksException;
- public IFunctionDescriptor lookupFunction(FunctionIdentifier fid) throws AlgebricksException;
+ IFunctionTypeInferer lookupFunctionTypeInferer(FunctionIdentifier fid);
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IFunctionTypeInferer.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IFunctionTypeInferer.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IFunctionTypeInferer.java
new file mode 100644
index 0000000..b8e27e1
--- /dev/null
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IFunctionTypeInferer.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.om.functions;
+
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+
+public interface IFunctionTypeInferer {
+ void infer(ILogicalExpression expr, IFunctionDescriptor fd, IVariableTypeEnvironment context)
+ throws AlgebricksException;
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/aggregates/collections/ListifyAggregateDescriptor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/aggregates/collections/ListifyAggregateDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/aggregates/collections/ListifyAggregateDescriptor.java
index f117d14..4e35950 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/aggregates/collections/ListifyAggregateDescriptor.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/aggregates/collections/ListifyAggregateDescriptor.java
@@ -21,22 +21,29 @@ package org.apache.asterix.runtime.aggregates.collections;
import org.apache.asterix.om.functions.BuiltinFunctions;
import org.apache.asterix.om.functions.IFunctionDescriptor;
import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
+import org.apache.asterix.om.functions.IFunctionTypeInferer;
import org.apache.asterix.om.types.AOrderedListType;
import org.apache.asterix.runtime.aggregates.base.AbstractAggregateFunctionDynamicDescriptor;
+import org.apache.asterix.runtime.functions.FunctionTypeInferers;
import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
import org.apache.hyracks.algebricks.runtime.base.IAggregateEvaluatorFactory;
import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
public class ListifyAggregateDescriptor extends AbstractAggregateFunctionDynamicDescriptor {
- private static final long serialVersionUID = 1L;
public static final IFunctionDescriptorFactory FACTORY = new IFunctionDescriptorFactory() {
@Override
public IFunctionDescriptor createFunctionDescriptor() {
return new ListifyAggregateDescriptor();
}
+
+ @Override
+ public IFunctionTypeInferer createFunctionTypeInferer() {
+ return FunctionTypeInferers.SET_EXPRESSION_TYPE;
+ }
};
+ private static final long serialVersionUID = 1L;
private AOrderedListType oltype;
@Override
@@ -53,5 +60,4 @@ public class ListifyAggregateDescriptor extends AbstractAggregateFunctionDynamic
public IAggregateEvaluatorFactory createAggregateEvaluatorFactory(final IScalarEvaluatorFactory[] args) {
return new ListifyAggregateFunctionEvalFactory(args, oltype);
}
-
}