You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by am...@apache.org on 2025/07/03 21:16:35 UTC
(ignite-3) 01/01: wip.
This is an automated email from the ASF dual-hosted git repository.
amashenkov pushed a commit to branch ignite-22703
in repository https://gitbox.apache.org/repos/asf/ignite-3.git
commit 37c16f3e229a11b4d13e3f1b60aac40a65b45505
Author: amashenkov <am...@apache.org>
AuthorDate: Thu Jul 3 22:42:02 2025 +0300
wip.
---
.../internal/benchmark/SqlSelectAllBenchmark.java | 9 +++-
.../engine/exec/TableRowConverterFactoryImpl.java | 20 ++------
.../engine/rule/logical/ProjectScanMergeRule.java | 4 +-
.../sql/engine/schema/IgniteTableImpl.java | 56 ++++++----------------
.../sql/engine/schema/TableDescriptor.java | 10 ++++
.../sql/engine/schema/TableDescriptorImpl.java | 23 +++++++++
.../ignite/internal/sql/engine/util/Commons.java | 15 ++++++
.../ignite/internal/sql/engine/util/RexUtils.java | 7 ++-
.../exec/ExecutableTableRegistrySelfTest.java | 1 -
.../exec/ProjectedTableRowConverterSelfTest.java | 1 -
.../sql/engine/framework/TestBuilders.java | 8 ----
.../sql/engine/planner/AbstractPlannerTest.java | 10 ++++
.../sql/engine/planner/PlannerTimeoutTest.java | 1 -
13 files changed, 89 insertions(+), 76 deletions(-)
diff --git a/modules/runner/src/integrationTest/java/org/apache/ignite/internal/benchmark/SqlSelectAllBenchmark.java b/modules/runner/src/integrationTest/java/org/apache/ignite/internal/benchmark/SqlSelectAllBenchmark.java
index 1b62824dd32..5bd893251f1 100644
--- a/modules/runner/src/integrationTest/java/org/apache/ignite/internal/benchmark/SqlSelectAllBenchmark.java
+++ b/modules/runner/src/integrationTest/java/org/apache/ignite/internal/benchmark/SqlSelectAllBenchmark.java
@@ -85,7 +85,7 @@ public class SqlSelectAllBenchmark extends AbstractTpcBenchmark {
}
@Param({
- "IDENTITY", "RESHUFFLING"
+ "IDENTITY", "TRIMMING", "RESHUFFLING"
})
private Projection projection;
@@ -110,6 +110,13 @@ public class SqlSelectAllBenchmark extends AbstractTpcBenchmark {
new Runner(opt).run();
}
+ /**
+ * Projection types for the benchmark.
+ *
+ * <p>IDENTITY - selects all columns in the original order.
+ * TRIMMING - selects all columns except few ones (preserving related order).
+ * RESHUFFLING - selects all columns in a different order.
+ */
public enum Projection {
IDENTITY {
@Override
diff --git a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/exec/TableRowConverterFactoryImpl.java b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/exec/TableRowConverterFactoryImpl.java
index cf67297f027..824e2a64f96 100644
--- a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/exec/TableRowConverterFactoryImpl.java
+++ b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/exec/TableRowConverterFactoryImpl.java
@@ -97,13 +97,13 @@ public class TableRowConverterFactoryImpl implements TableRowConverterFactory {
? tableColumnSet
: projection.toIntArray();
- Int2ObjectMap<VirtualColumn> extraColumns = createVirtualColumns(mapping, partId);
-
- // TODO: IGNITE-22703 Ensure this check is correct.
- if (extraColumns.isEmpty() && isFullIdentityMapping(mapping)) {
+ // TODO: IGNITE-22703 Can we opmitimize this?
+ if (Commons.isIdentityMapping(mapping, schemaDescriptor.length())) {
return fullRowConverter;
}
+ Int2ObjectMap<VirtualColumn> extraColumns = createVirtualColumns(mapping, partId);
+
return new ProjectedTableRowConverterImpl(
schemaRegistry,
schemaDescriptor,
@@ -112,18 +112,6 @@ public class TableRowConverterFactoryImpl implements TableRowConverterFactory {
);
}
- private boolean isFullIdentityMapping(int[] mapping) {
- if (mapping.length != schemaDescriptor.length()) {
- return false;
- }
- for (int i = 0; i < mapping.length; i++) {
- if (mapping[i] != i) {
- return false;
- }
- }
- return true;
- }
-
private Int2ObjectMap<VirtualColumn> createVirtualColumns(int[] requiredColumns, int partId) {
if (virtualColumnsFactory.isEmpty()) {
return Int2ObjectMaps.emptyMap();
diff --git a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/rule/logical/ProjectScanMergeRule.java b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/rule/logical/ProjectScanMergeRule.java
index 528c92685fc..54d503718fe 100644
--- a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/rule/logical/ProjectScanMergeRule.java
+++ b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/rule/logical/ProjectScanMergeRule.java
@@ -17,9 +17,7 @@
package org.apache.ignite.internal.sql.engine.rule.logical;
-import it.unimi.dsi.fastutil.ints.IntArrayList;
import it.unimi.dsi.fastutil.ints.IntLinkedOpenHashSet;
-import it.unimi.dsi.fastutil.ints.IntList;
import it.unimi.dsi.fastutil.ints.IntSet;
import java.util.List;
import org.apache.calcite.plan.RelOptCluster;
@@ -148,7 +146,7 @@ public abstract class ProjectScanMergeRule<T extends ProjectableFilterableTableS
}
// TODO: IGNITE-22703 revisit required columns usage.
- if (RexUtils.isIdentity(projects, tbl.getRowType(typeFactory, requiredColumns), true)) {
+ if (RexUtils.isIdentity(projects, tbl.getRowType(typeFactory, requiredColumns))) {
projects = null;
}
diff --git a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/schema/IgniteTableImpl.java b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/schema/IgniteTableImpl.java
index 17e70aae638..af0b57c80be 100644
--- a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/schema/IgniteTableImpl.java
+++ b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/schema/IgniteTableImpl.java
@@ -29,7 +29,6 @@ import org.apache.calcite.plan.RelTraitSet;
import org.apache.calcite.rel.core.TableScan;
import org.apache.calcite.rel.hint.RelHint;
import org.apache.calcite.rel.type.RelDataType;
-import org.apache.calcite.rel.type.RelDataTypeFactory;
import org.apache.calcite.schema.Statistic;
import org.apache.calcite.util.ImmutableIntList;
import org.apache.ignite.internal.sql.engine.rel.logical.IgniteLogicalTableScan;
@@ -77,21 +76,6 @@ public class IgniteTableImpl extends AbstractIgniteDataSource implements IgniteT
colocationColumnTypes = new Lazy<>(this::evaluateTypes);
}
- private static RelDataType deriveDeleteRowType(
- IgniteTypeFactory typeFactory,
- TableDescriptor desc,
- ImmutableIntList keyColumns
- ) {
- var builder = new RelDataTypeFactory.Builder(typeFactory);
-
- RelDataType fullRow = desc.rowType(typeFactory, null);
- for (int i : keyColumns) {
- builder.add(fullRow.getFieldList().get(i));
- }
-
- return builder.build();
- }
-
private static @Nullable ImmutableIntList deriveColumnsToInsert(TableDescriptor desc) {
/*
Columns to insert are columns which will be expanded in case user omit
@@ -108,45 +92,35 @@ public class IgniteTableImpl extends AbstractIgniteDataSource implements IgniteT
See org.apache.ignite.internal.sql.engine.util.Commons.implicitPkEnabled, and
org.apache.ignite.internal.sql.engine.schema.SqlSchemaManagerImpl.injectDefault for details.
*/
+ if (!desc.hasHiddenColumns()) {
+ return null; // 'null' means that full projection will be used for insert.
+ }
+
IntList columnsToInsert = new IntArrayList(desc.columnsCount());
- boolean hiddenColumnFound = false;
for (ColumnDescriptor columnDescriptor : desc) {
- if (columnDescriptor.hidden()) {
- hiddenColumnFound = true;
-
- continue;
+ if (!columnDescriptor.hidden()) {
+ columnsToInsert.add(columnDescriptor.logicalIndex());
}
-
- columnsToInsert.add(columnDescriptor.logicalIndex());
- }
-
- if (hiddenColumnFound) {
- return ImmutableIntList.of(columnsToInsert.toIntArray());
}
- return null;
+ return ImmutableIntList.of(columnsToInsert.toIntArray());
}
private static @Nullable ImmutableIntList deriveColumnsToUpdate(TableDescriptor desc) {
+ if (!desc.hasVirtualColumns()) {
+ return null; // 'null' means that full projection will be used for update.
+ }
+
IntList columnsToUpdate = new IntArrayList(desc.columnsCount());
- boolean virtualColumnFound = false;
for (ColumnDescriptor columnDescriptor : desc) {
- if (columnDescriptor.virtual()) {
- virtualColumnFound = true;
-
- continue;
+ if (!columnDescriptor.virtual()) {
+ columnsToUpdate.add(columnDescriptor.logicalIndex());
}
-
- columnsToUpdate.add(columnDescriptor.logicalIndex());
- }
-
- if (virtualColumnFound) {
- return ImmutableIntList.of(columnsToUpdate.toIntArray());
}
- return null;
+ return ImmutableIntList.of(columnsToUpdate.toIntArray());
}
/** {@inheritDoc} */
@@ -220,6 +194,6 @@ public class IgniteTableImpl extends AbstractIgniteDataSource implements IgniteT
/** {@inheritDoc} */
@Override
public RelDataType rowTypeForDelete(IgniteTypeFactory factory) {
- return deriveDeleteRowType(factory, descriptor(), keyColumns);
+ return descriptor().rowType(factory, keyColumns);
}
}
diff --git a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/schema/TableDescriptor.java b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/schema/TableDescriptor.java
index 47f408060e8..6946c581757 100644
--- a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/schema/TableDescriptor.java
+++ b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/schema/TableDescriptor.java
@@ -64,4 +64,14 @@ public interface TableDescriptor extends InitializerExpressionFactory, Iterable<
* @return Actual count of columns.
*/
int columnsCount();
+
+ /**
+ * Returns {@code true} if table has hidden columns, {@code false} otherwise.
+ */
+ boolean hasHiddenColumns();
+
+ /**
+ * Returns {@code true} if table has virtual columns, {@code false} otherwise.
+ */
+ boolean hasVirtualColumns();
}
diff --git a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/schema/TableDescriptorImpl.java b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/schema/TableDescriptorImpl.java
index 3ae5b01b3c2..b0879684ba7 100644
--- a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/schema/TableDescriptorImpl.java
+++ b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/schema/TableDescriptorImpl.java
@@ -58,6 +58,9 @@ public class TableDescriptorImpl extends NullInitializerExpressionFactory implem
private final RelDataType rowType;
private final RelDataType rowTypeSansHidden;
+ private final boolean hasHiddenColumns;
+ private final boolean hasVirtualColumns;
+
/**
* Constructor.
*
@@ -73,6 +76,8 @@ public class TableDescriptorImpl extends NullInitializerExpressionFactory implem
RelDataTypeFactory.Builder typeBuilder = new RelDataTypeFactory.Builder(factory);
RelDataTypeFactory.Builder typeSansHiddenBuilder = new RelDataTypeFactory.Builder(factory);
+ boolean hasHiddenColumns = false;
+ boolean hasVirtualColumns = false;
for (ColumnDescriptor descriptor : columnDescriptors) {
RelDataType columnType = deriveLogicalType(factory, descriptor);
@@ -80,6 +85,12 @@ public class TableDescriptorImpl extends NullInitializerExpressionFactory implem
if (!descriptor.hidden()) {
typeSansHiddenBuilder.add(descriptor.name(), columnType);
+ } else {
+ hasHiddenColumns = true;
+ }
+
+ if (descriptor.virtual()) {
+ hasVirtualColumns = true;
}
descriptorsMap.put(descriptor.name(), descriptor);
@@ -89,6 +100,8 @@ public class TableDescriptorImpl extends NullInitializerExpressionFactory implem
this.descriptorsMap = descriptorsMap;
this.rowType = typeBuilder.build();
this.rowTypeSansHidden = typeSansHiddenBuilder.build();
+ this.hasHiddenColumns = hasHiddenColumns;
+ this.hasVirtualColumns = hasVirtualColumns;
}
@Override
@@ -190,6 +203,16 @@ public class TableDescriptorImpl extends NullInitializerExpressionFactory implem
return descriptors.length;
}
+ @Override
+ public boolean hasHiddenColumns() {
+ return hasHiddenColumns;
+ }
+
+ @Override
+ public boolean hasVirtualColumns() {
+ return hasVirtualColumns;
+ }
+
private RelDataType deriveLogicalType(RelDataTypeFactory factory, ColumnDescriptor desc) {
return native2relationalType(factory, desc.physicalType(), desc.nullable());
}
diff --git a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/util/Commons.java b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/util/Commons.java
index 32de6f50e26..b276a5432e5 100644
--- a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/util/Commons.java
+++ b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/util/Commons.java
@@ -829,4 +829,19 @@ public final class Commons {
// Create a non-strict equijoin condition that treats IS NOT DISTINCT_FROM as an equi join condition.
return JoinInfo.of(join.getLeft(), join.getRight(), join.getCondition());
}
+
+ /**
+ * Checks whether the given mapping is an identity mapping.
+ */
+ public static boolean isIdentityMapping(int[] mapping, int length) {
+ if (mapping.length != length) {
+ return false;
+ }
+ for (int i = 0; i < mapping.length; i++) {
+ if (mapping[i] != i) {
+ return false;
+ }
+ }
+ return true;
+ }
}
diff --git a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/util/RexUtils.java b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/util/RexUtils.java
index 55c3a138ec3..aa6e7ab99bc 100644
--- a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/util/RexUtils.java
+++ b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/util/RexUtils.java
@@ -162,16 +162,15 @@ public class RexUtils {
}
/** Returns whether a list of expressions projects the incoming fields. */
- public static boolean isIdentity(List<? extends RexNode> projects, RelDataType inputRowType, boolean local) {
+ public static boolean isIdentity(List<? extends RexNode> projects, RelDataType inputRowType) {
if (inputRowType.getFieldCount() != projects.size()) {
return false;
}
- final List<RelDataTypeField> fields = inputRowType.getFieldList();
- Class<? extends RexSlot> clazz = local ? RexLocalRef.class : RexInputRef.class;
+ List<RelDataTypeField> fields = inputRowType.getFieldList();
for (int i = 0; i < fields.size(); i++) {
- if (!clazz.isInstance(projects.get(i))) {
+ if (!(projects.get(i) instanceof RexLocalRef)) {
return false;
}
diff --git a/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/exec/ExecutableTableRegistrySelfTest.java b/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/exec/ExecutableTableRegistrySelfTest.java
index 4f3600f3043..ac7b579cedb 100644
--- a/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/exec/ExecutableTableRegistrySelfTest.java
+++ b/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/exec/ExecutableTableRegistrySelfTest.java
@@ -24,7 +24,6 @@ import static org.mockito.Mockito.when;
import java.util.Collections;
import java.util.Map;
-import java.util.Spliterators;
import org.apache.calcite.util.ImmutableIntList;
import org.apache.ignite.internal.TestHybridClock;
import org.apache.ignite.internal.components.SystemPropertiesNodeProperties;
diff --git a/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/exec/ProjectedTableRowConverterSelfTest.java b/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/exec/ProjectedTableRowConverterSelfTest.java
index 20c7f254a56..36e18de50ec 100644
--- a/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/exec/ProjectedTableRowConverterSelfTest.java
+++ b/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/exec/ProjectedTableRowConverterSelfTest.java
@@ -25,7 +25,6 @@ import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.stream.Collectors;
-import org.apache.calcite.util.BitSets;
import org.apache.ignite.internal.binarytuple.BinaryTupleBuilder;
import org.apache.ignite.internal.schema.BinaryRow;
import org.apache.ignite.internal.schema.BinaryRowImpl;
diff --git a/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/framework/TestBuilders.java b/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/framework/TestBuilders.java
index c5d930617ba..2140a1a3497 100644
--- a/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/framework/TestBuilders.java
+++ b/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/framework/TestBuilders.java
@@ -32,7 +32,6 @@ import java.time.Clock;
import java.time.ZoneId;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.BitSet;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
@@ -1712,13 +1711,6 @@ public class TestBuilders {
return builder.build();
}
- @Deprecated
- private static Object[] project(Object[] row, @Nullable BitSet requiredElements) {
- if (requiredElements == null) {
- return row;
- }
- return project(row, ImmutableIntList.copyOf(requiredElements.stream().iterator()));
- }
private static Object[] project(Object[] row, @Nullable ImmutableIntList requiredElements) {
if (requiredElements == null) {
return row;
diff --git a/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/planner/AbstractPlannerTest.java b/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/planner/AbstractPlannerTest.java
index f62124ade13..a473faa77df 100644
--- a/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/planner/AbstractPlannerTest.java
+++ b/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/planner/AbstractPlannerTest.java
@@ -891,6 +891,16 @@ public abstract class AbstractPlannerTest extends IgniteAbstractTest {
return rowType.getFieldCount();
}
+ @Override
+ public boolean hasHiddenColumns() {
+ return false;
+ }
+
+ @Override
+ public boolean hasVirtualColumns() {
+ return false;
+ }
+
/** {@inheritDoc} */
@Override
public boolean isGeneratedAlways(RelOptTable table, int idxColumn) {
diff --git a/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/planner/PlannerTimeoutTest.java b/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/planner/PlannerTimeoutTest.java
index c7cdc4b988f..48bbe7cb1e5 100644
--- a/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/planner/PlannerTimeoutTest.java
+++ b/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/planner/PlannerTimeoutTest.java
@@ -35,7 +35,6 @@ import org.apache.calcite.plan.volcano.VolcanoPlanner;
import org.apache.calcite.plan.volcano.VolcanoTimeoutException;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.RelVisitor;
-import org.apache.calcite.util.ImmutableBitSet;
import org.apache.calcite.util.ImmutableIntList;
import org.apache.ignite.internal.metrics.MetricManagerImpl;
import org.apache.ignite.internal.sql.engine.SqlOperationContext;