You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by mb...@apache.org on 2024/11/05 13:44:18 UTC
(asterixdb) 03/07: Merge branch gerrit/trinity into gerrit/goldfish
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit e956d8d6948577825747bad118c27dd379319132
Merge: 7e7960c090 3f20e39b24
Author: Hussain Towaileb <Hu...@Couchbase.com>
AuthorDate: Sat Oct 26 23:02:17 2024 +0300
Merge branch gerrit/trinity into gerrit/goldfish
Ext-ref: MB-63819
Change-Id: I14251ec5e66e4ee48505b10783e1a7bd6ebf49c2
.../app/function/ActiveRequestsDatasource.java | 7 ++++
.../app/function/CompletedRequestsDatasource.java | 7 ++++
.../app/function/DatasetResourcesDatasource.java | 15 +++++++++
.../asterix/app/function/DumpIndexDatasource.java | 21 +++++++++++-
.../asterix/app/function/DumpIndexRewriter.java | 2 +-
.../app/function/JobSummariesDatasource.java | 7 ++++
.../asterix/app/function/PingDatasource.java | 6 ++++
.../asterix/app/function/QueryIndexDatasource.java | 18 ++++++++++
.../app/function/QueryPartitionDatasource.java | 19 +++++++++++
.../app/function/StorageComponentsDatasource.java | 15 +++++++++
.../TPCDSAllTablesDataGeneratorDatasource.java | 15 +++++++++
.../TPCDSSingleTableDataGeneratorDatasource.java | 20 ++++++++++++
.../test.000.ddl.sqlpp | 34 +++++++++++++++++++
.../test.010.update.sqlpp | 21 ++++++++++++
.../test.020.query.sqlpp | 23 +++++++++++++
.../test.030.query.sqlpp | 24 ++++++++++++++
.../test.999.ddl.sqlpp | 20 ++++++++++++
.../test.020.unorderedtxt | 2 ++
.../test.030.plan | 38 ++++++++++++++++++++++
.../src/test/resources/runtimets/sqlpp_queries.xml | 5 +++
.../resources/runtimets/testsuite_it_python.xml | 2 ++
.../asterix/metadata/declared/DataSource.java | 24 ++++++++++++++
.../metadata/declared/DatasetDataSource.java | 5 ---
.../metadata/declared/FunctionDataSource.java | 16 +++++++++
.../core/algebra/metadata/IDataSource.java | 4 +--
.../visitors/IsomorphismOperatorVisitor.java | 17 +++++-----
26 files changed, 369 insertions(+), 18 deletions(-)
diff --cc asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexDatasource.java
index 1eac011a6a,e5d78a4721..23fcf8c1df
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexDatasource.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexDatasource.java
@@@ -39,18 -42,22 +42,25 @@@ public class DumpIndexDatasource extend
private final IndexDataflowHelperFactory indexDataflowHelperFactory;
private final RecordDescriptor recDesc;
private final IBinaryComparatorFactory[] comparatorFactories;
- private final AlgebricksAbsolutePartitionConstraint storageLocations;
+ private final AlgebricksAbsolutePartitionConstraint constraint;
+ private final int[][] partitionsMap;
+ private final Index index;
public DumpIndexDatasource(INodeDomain domain, IndexDataflowHelperFactory indexDataflowHelperFactory,
RecordDescriptor recDesc, IBinaryComparatorFactory[] comparatorFactories,
- AlgebricksAbsolutePartitionConstraint constraint, int[][] partitionsMap) throws AlgebricksException {
- AlgebricksAbsolutePartitionConstraint storageLocations, Index index) throws AlgebricksException {
++ AlgebricksAbsolutePartitionConstraint constraint, int[][] partitionsMap, Index index)
++ throws AlgebricksException {
super(DUMP_INDEX_DATASOURCE_ID, DumpIndexRewriter.DUMP_INDEX, domain);
this.indexDataflowHelperFactory = indexDataflowHelperFactory;
this.recDesc = recDesc;
this.comparatorFactories = comparatorFactories;
- this.storageLocations = storageLocations;
+ this.constraint = constraint;
+ this.partitionsMap = partitionsMap;
+ this.index = index;
+ }
+
+ public Index getIndex() {
+ return index;
}
@Override
diff --cc asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexRewriter.java
index d50315b3d4,b5ea73fac9..825d5718a0
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexRewriter.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexRewriter.java
@@@ -80,20 -66,13 +80,20 @@@ public class DumpIndexRewriter extends
}
ISecondaryIndexOperationsHelper secondaryIndexHelper =
SecondaryIndexOperationsHelper.createIndexOperationsHelper(dataset, index, metadataProvider, loc);
+ PartitioningProperties partitioningProperties =
+ metadataProvider.getPartitioningProperties(dataset, index.getIndexName());
IndexDataflowHelperFactory indexDataflowHelperFactory =
new IndexDataflowHelperFactory(metadataProvider.getStorageComponentProvider().getStorageManager(),
- secondaryIndexHelper.getSecondaryFileSplitProvider());
+ partitioningProperties.getSplitsProvider());
AlgebricksAbsolutePartitionConstraint secondaryPartitionConstraint =
- (AlgebricksAbsolutePartitionConstraint) secondaryIndexHelper.getSecondaryPartitionConstraint();
+ (AlgebricksAbsolutePartitionConstraint) partitioningProperties.getConstraints();
return new DumpIndexDatasource(context.getComputationNodeDomain(), indexDataflowHelperFactory,
secondaryIndexHelper.getSecondaryRecDesc(), secondaryIndexHelper.getSecondaryComparatorFactories(),
- secondaryPartitionConstraint, partitioningProperties.getComputeStorageMap());
- secondaryPartitionConstraint, index);
++ secondaryPartitionConstraint, partitioningProperties.getComputeStorageMap(), index);
+ }
+
+ @Override
+ protected boolean invalidArgs(List<Mutable<ILogicalExpression>> args) {
+ return args.size() < 3;
}
}
diff --cc asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/QueryIndexDatasource.java
index 52054d6d1d,1c036f87e4..d6a45af34e
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/QueryIndexDatasource.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/QueryIndexDatasource.java
@@@ -135,7 -134,16 +144,16 @@@ public class QueryIndexDatasource exten
}
private static DataSourceId createQueryIndexDataSourceId(Dataset dataset, String indexName) {
- return new DataSourceId(dataset.getDataverseName(), dataset.getDatasetName(),
+ return new DataSourceId(dataset.getDatabaseName(), dataset.getDataverseName(), dataset.getDatasetName(),
new String[] { indexName, QueryIndexRewriter.QUERY_INDEX.getName() });
}
+
+ @Override
+ public boolean sameFunctionDatasource(FunctionDataSource other) {
+ if (!Objects.equals(this.functionId, other.getFunctionId())) {
+ return false;
+ }
+ QueryIndexDatasource that = (QueryIndexDatasource) other;
+ return Objects.equals(this.ds, that.getDataset()) && Objects.equals(this.indexName, that.getIndexName());
+ }
}
diff --cc asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/QueryPartitionDatasource.java
index 4d40ba6913,0000000000..4917b33cd6
mode 100644,000000..100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/QueryPartitionDatasource.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/QueryPartitionDatasource.java
@@@ -1,131 -1,0 +1,150 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.app.function;
+
+import java.util.ArrayList;
+import java.util.List;
++import java.util.Objects;
+
+import org.apache.asterix.common.cluster.IClusterStateManager;
+import org.apache.asterix.metadata.api.IDatasourceFunction;
+import org.apache.asterix.metadata.declared.DataSourceId;
+import org.apache.asterix.metadata.declared.FunctionDataSource;
+import org.apache.asterix.metadata.declared.MetadataProvider;
+import org.apache.asterix.metadata.entities.Dataset;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.IAType;
+import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
+import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.common.utils.Pair;
+import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import org.apache.hyracks.algebricks.core.algebra.metadata.IDataSource;
+import org.apache.hyracks.algebricks.core.algebra.metadata.IDataSourcePropertiesProvider;
+import org.apache.hyracks.algebricks.core.algebra.metadata.IProjectionFiltrationInfo;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import org.apache.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
+import org.apache.hyracks.algebricks.core.algebra.properties.INodeDomain;
+import org.apache.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import org.apache.hyracks.algebricks.core.algebra.properties.RandomPartitioningProperty;
+import org.apache.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
+import org.apache.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import org.apache.hyracks.api.dataflow.IOperatorDescriptor;
+import org.apache.hyracks.api.job.JobSpecification;
+import org.apache.hyracks.storage.am.common.api.ITupleFilterFactory;
+
+public class QueryPartitionDatasource extends FunctionDataSource {
+
+ private final Dataset ds;
+ private final AlgebricksAbsolutePartitionConstraint storageLocations;
+ private final int partitionNum;
+
+ public QueryPartitionDatasource(Dataset ds, INodeDomain domain,
+ AlgebricksAbsolutePartitionConstraint storageLocations, ARecordType recType, int partitionNum)
+ throws AlgebricksException {
+ super(createQueryPartitionDataSourceId(ds), QueryPartitionRewriter.QUERY_PARTITION, domain, recType);
+ if (partitionNum < 0) {
+ throw new IllegalArgumentException("partition must be >= 0");
+ }
+ this.partitionNum = partitionNum;
+ this.ds = ds;
+ this.storageLocations = storageLocations;
+ }
+
++ public Dataset getDatasource() {
++ return ds;
++ }
++
++ public int getPartitionNumber() {
++ return partitionNum;
++ }
++
+ @Override
+ protected void initSchemaType(IAType iType) {
+ ARecordType type = (ARecordType) iType;
+ IAType[] fieldTypes = type.getFieldTypes();
+ schemaTypes = new IAType[fieldTypes.length];
+ System.arraycopy(fieldTypes, 0, schemaTypes, 0, schemaTypes.length);
+ }
+
+ @Override
+ protected AlgebricksAbsolutePartitionConstraint getLocations(IClusterStateManager csm, MetadataProvider md) {
+ return storageLocations;
+ }
+
+ @Override
+ public boolean isScanAccessPathALeaf() {
+ // the index scan op is not a leaf op. the ETS op will start the scan of the index. we need the ETS op below
+ // the index scan to be still generated
+ return false;
+ }
+
+ @Override
+ protected IDatasourceFunction createFunction(MetadataProvider metadataProvider,
+ AlgebricksAbsolutePartitionConstraint locations) {
+ throw new UnsupportedOperationException("query-partition() does not use record reader adapter");
+ }
+
+ @Override
+ public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> buildDatasourceScanRuntime(
+ MetadataProvider metadataProvider, IDataSource<DataSourceId> dataSource,
+ List<LogicalVariable> scanVariables, List<LogicalVariable> projectVariables, boolean projectPushed,
+ List<LogicalVariable> minFilterVars, List<LogicalVariable> maxFilterVars,
+ ITupleFilterFactory tupleFilterFactory, long outputLimit, IOperatorSchema opSchema,
+ IVariableTypeEnvironment typeEnv, JobGenContext context, JobSpecification jobSpec, Object implConfig,
+ IProjectionFiltrationInfo projectionInfo) throws AlgebricksException {
+ return metadataProvider.getBtreePartitionSearchRuntime(jobSpec, opSchema, typeEnv, context, ds,
+ tupleFilterFactory, outputLimit, partitionNum);
+ }
+
+ @Override
+ public IDataSourcePropertiesProvider getPropertiesProvider() {
+ return new IDataSourcePropertiesProvider() {
+ @Override
+ public IPhysicalPropertiesVector computeRequiredProperties(List<LogicalVariable> scanVariables,
+ IOptimizationContext ctx) {
+ return StructuralPropertiesVector.EMPTY_PROPERTIES_VECTOR;
+ }
+
+ @Override
+ public IPhysicalPropertiesVector computeDeliveredProperties(List<LogicalVariable> scanVariables,
+ IOptimizationContext ctx) {
+ List<ILocalStructuralProperty> propsLocal = new ArrayList<>(1);
+ return new StructuralPropertiesVector(new RandomPartitioningProperty(domain), propsLocal);
+ }
+ };
+ }
+
+ private static DataSourceId createQueryPartitionDataSourceId(Dataset dataset) {
+ return new DataSourceId(dataset.getDatabaseName(), dataset.getDataverseName(), dataset.getDatasetName(),
+ new String[] { dataset.getDatasetName(), QueryPartitionRewriter.QUERY_PARTITION.getName() });
+ }
++
++ @Override
++ public boolean sameFunctionDatasource(FunctionDataSource other) {
++ if (!Objects.equals(this.functionId, other.getFunctionId())) {
++ return false;
++ }
++ QueryPartitionDatasource that = (QueryPartitionDatasource) other;
++ return Objects.equals(this.ds, that.getDatasource())
++ && Objects.equals(this.partitionNum, that.getPartitionNumber());
++ }
+}
diff --cc asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSAllTablesDataGeneratorDatasource.java
index b4d1b03ab2,489b052795..ac19b0057e
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSAllTablesDataGeneratorDatasource.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSAllTablesDataGeneratorDatasource.java
@@@ -63,7 -69,16 +69,16 @@@ public class TPCDSAllTablesDataGenerato
}
@Override
- protected AlgebricksAbsolutePartitionConstraint getLocations(IClusterStateManager csm) {
- return csm.getSortedClusterLocations();
+ protected AlgebricksAbsolutePartitionConstraint getLocations(IClusterStateManager csm, MetadataProvider md) {
+ return md.getDataPartitioningProvider().getClusterLocations();
}
+
+ @Override
+ public boolean sameFunctionDatasource(FunctionDataSource other) {
+ if (!Objects.equals(this.functionId, other.getFunctionId())) {
+ return false;
+ }
+ TPCDSAllTablesDataGeneratorDatasource that = (TPCDSAllTablesDataGeneratorDatasource) other;
+ return Objects.equals(this.scalingFactor, that.getScalingFactor());
+ }
}
diff --cc asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSSingleTableDataGeneratorDatasource.java
index 3bbaee2079,c90dff173a..6551346ed5
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSSingleTableDataGeneratorDatasource.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSSingleTableDataGeneratorDatasource.java
@@@ -67,7 -77,17 +77,17 @@@ public class TPCDSSingleTableDataGenera
}
@Override
- protected AlgebricksAbsolutePartitionConstraint getLocations(IClusterStateManager csm) {
- return csm.getSortedClusterLocations();
+ protected AlgebricksAbsolutePartitionConstraint getLocations(IClusterStateManager csm, MetadataProvider md) {
+ return md.getDataPartitioningProvider().getClusterLocations();
}
+
+ @Override
+ public boolean sameFunctionDatasource(FunctionDataSource other) {
+ if (!Objects.equals(this.functionId, other.getFunctionId())) {
+ return false;
+ }
+ TPCDSSingleTableDataGeneratorDatasource that = (TPCDSSingleTableDataGeneratorDatasource) other;
+ return Objects.equals(this.tableName, that.getTableName())
+ && Objects.equals(this.scalingFactor, that.getScalingFactor());
+ }
}
diff --cc asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DataSource.java
index 1fa84f6aca,36273d2d3e..77620419ce
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DataSource.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DataSource.java
@@@ -166,5 -166,28 +167,28 @@@ public abstract class DataSource implem
List<LogicalVariable> minFilterVars, List<LogicalVariable> maxFilterVars,
ITupleFilterFactory tupleFilterFactory, long outputLimit, IOperatorSchema opSchema,
IVariableTypeEnvironment typeEnv, JobGenContext context, JobSpecification jobSpec, Object implConfig,
- IProjectionInfo<?> projectionInfo) throws AlgebricksException;
+ IProjectionFiltrationInfo projectionFiltrationInfo) throws AlgebricksException;
+
+ @Override
+ public boolean sameAs(IDataSource<?> other) {
+ if (this == other) {
+ return true;
+ }
+
+ if (!(other instanceof DataSource)) {
+ return false;
+ }
+
+ DataSource that = (DataSource) other;
+ if (!Objects.equals(this.id, that.getId()) || !Objects.equals(this.datasourceType, that.getDatasourceType())) {
+ return false;
+ }
+
+ if (this.datasourceType == Type.EXTERNAL_DATASET && that.getDatasourceType() == Type.EXTERNAL_DATASET
+ && !Objects.equals(this.getProperties(), other.getProperties())) {
+ return false;
+ }
+
+ return true;
+ }
}
diff --cc asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSource.java
index 91f7615cdc,bce22c1acd..2322e53ac9
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSource.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSource.java
@@@ -137,6 -124,22 +137,22 @@@ public abstract class FunctionDataSourc
}
protected static DataSourceId createDataSourceId(FunctionIdentifier fid, String... parameters) {
- return new DataSourceId(FunctionSignature.getDataverseName(fid), fid.getName(), parameters);
+ return new DataSourceId(fid.getDatabase(), FunctionSignature.getDataverseName(fid), fid.getName(), parameters);
}
+
+ protected abstract boolean sameFunctionDatasource(FunctionDataSource other);
+
+ @Override
+ public boolean sameAs(IDataSource<?> other) {
+ if (!super.sameAs(other)) {
+ return false;
+ }
+
+ if (!(other instanceof FunctionDataSource)) {
+ return false;
+ }
+
+ FunctionDataSource that = (FunctionDataSource) other;
+ return sameFunctionDatasource(that);
+ }
}
diff --cc hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismOperatorVisitor.java
index 05f41a0535,7ee3c53e9d..b985d09ce5
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismOperatorVisitor.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismOperatorVisitor.java
@@@ -491,12 -485,13 +491,13 @@@ public class IsomorphismOperatorVisito
if (!isomorphic) {
return Boolean.FALSE;
}
- IDataSource<?> dataSource = op.getDataSource();
- IDataSource<?> argDataSource = argScan.getDataSource();
- if (dataSource.compareProperties() && argDataSource.compareProperties()
- && !Objects.equals(dataSource.getProperties(), argDataSource.getProperties())) {
+
+ isomorphic = op.getOutputLimit() == argScan.getOutputLimit()
- && Objects.equals(op.getProjectionInfo(), argScan.getProjectionInfo());
++ && Objects.equals(op.getProjectionFiltrationInfo(), argScan.getProjectionFiltrationInfo());
+ if (!isomorphic) {
return Boolean.FALSE;
}
+
DataSourceScanOperator scanOpArg = (DataSourceScanOperator) copyAndSubstituteVar(op, arg);
ILogicalExpression opCondition = op.getSelectCondition() != null ? op.getSelectCondition().getValue() : null;
ILogicalExpression argCondition =