You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by am...@apache.org on 2025/05/12 11:13:08 UTC
(ignite-3) branch ignite-24936 created (now d86e55a50b4)
This is an automated email from the ASF dual-hosted git repository.
amashenkov pushed a change to branch ignite-24936
in repository https://gitbox.apache.org/repos/asf/ignite-3.git
at d86e55a50b4 Add __PART virtual column.
This branch includes the following new commits:
new d86e55a50b4 Add __PART virtual column.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(ignite-3) 01/01: Add __PART virtual column.
Posted by am...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
amashenkov pushed a commit to branch ignite-24936
in repository https://gitbox.apache.org/repos/asf/ignite-3.git
commit d86e55a50b447bb5c87ffe9610c91f2c81ab8753
Author: amashenkov <am...@apache.org>
AuthorDate: Mon May 12 13:49:47 2025 +0300
Add __PART virtual column.
---
.../sql/group1/syscolumns/system_columns.test | 26 ++++++++++++
.../engine/exec/TableRowConverterFactoryImpl.java | 48 +++++++++++++++-------
.../sql/engine/prepare/IgniteSqlValidator.java | 3 +-
.../sql/engine/schema/SqlSchemaManagerImpl.java | 10 ++---
.../ignite/internal/sql/engine/util/Commons.java | 4 +-
.../sql/engine/framework/TestBuilders.java | 13 ++++++
6 files changed, 83 insertions(+), 21 deletions(-)
diff --git a/modules/sql-engine/src/integrationTest/sql/group1/syscolumns/system_columns.test b/modules/sql-engine/src/integrationTest/sql/group1/syscolumns/system_columns.test
index a7bc0c296e2..a15d1ef01ec 100644
--- a/modules/sql-engine/src/integrationTest/sql/group1/syscolumns/system_columns.test
+++ b/modules/sql-engine/src/integrationTest/sql/group1/syscolumns/system_columns.test
@@ -12,6 +12,11 @@ insert into t0 values (101, 'val1'), (102, 'val2');
statement ok
insert into t0 values (103, 'val3');
+# statement error: Failed to validate query:
+statement error
+----
+insert into t0 (id, val, __part) values (104, 'val4', 1)
+
# statement error: Failed to validate query:
statement error
----
@@ -26,6 +31,21 @@ select * from t0
102 val2
103 val3
+query II rowsort
+select __part, id from t0
+----
+19 101
+17 102
+11 103
+
+query I rowsort
+select __part from t0
+----
+19
+17
+11
+
+# Select legacy partition system column
query II rowsort
select "__part", id from t0
----
@@ -40,6 +60,12 @@ select "__part" from t0
17
11
+query II rowsort
+select __part, "__part" id from t0
+----
+19 19
+17 17
+11 11
# Partition system column in WHERE
query IT
diff --git a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/exec/TableRowConverterFactoryImpl.java b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/exec/TableRowConverterFactoryImpl.java
index f187a47a049..bab9739dc7d 100644
--- a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/exec/TableRowConverterFactoryImpl.java
+++ b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/exec/TableRowConverterFactoryImpl.java
@@ -17,6 +17,7 @@
package org.apache.ignite.internal.sql.engine.exec;
+import it.unimi.dsi.fastutil.ints.Int2ObjectArrayMap;
import it.unimi.dsi.fastutil.ints.Int2ObjectMap;
import it.unimi.dsi.fastutil.ints.Int2ObjectMaps;
import java.util.BitSet;
@@ -37,16 +38,16 @@ public class TableRowConverterFactoryImpl implements TableRowConverterFactory {
private final SchemaDescriptor schemaDescriptor;
private final TableRowConverter fullRowConverter;
private final BitSet tableColumnSet;
- private IntFunction<VirtualColumn> virtualColumnFactory;
+ private final Int2ObjectArrayMap<IntFunction<VirtualColumn>> virtualColumnsFactory = new Int2ObjectArrayMap<>();
/**
* Creates a factory from given schema and indexes of primary key.
*
* @param tableDescriptor Table descriptor.
- * @param schemaRegistry Registry of all schemas known so far. Used in case table returned
- * a row in older version than required to make an upgrade.
- * @param schemaDescriptor Actual schema descriptor. Used as a target schema to convert
- * rows from sql format to one accepted by underlying table.
+ * @param schemaRegistry Registry of all schemas known so far. Used in case table returned a row in older version than required
+ * to make an upgrade.
+ * @param schemaDescriptor Actual schema descriptor. Used as a target schema to convert rows from sql format to one accepted by
+ * underlying table.
*/
public TableRowConverterFactoryImpl(
TableDescriptor tableDescriptor,
@@ -64,13 +65,20 @@ public class TableRowConverterFactoryImpl implements TableRowConverterFactory {
tableColumnSet = new BitSet();
tableColumnSet.set(0, tableDescriptor.columnsCount());
- ColumnDescriptor columnDescriptor = tableDescriptor.columnDescriptor(Commons.PART_COL_NAME);
-
- if (columnDescriptor != null) {
- assert columnDescriptor.virtual();
+ addVirtualColumn(tableDescriptor.columnDescriptor(Commons.PART_COL_NAME));
+ addVirtualColumn(tableDescriptor.columnDescriptor(Commons.PART_COL_NAME_LEGACY));
+ }
- virtualColumnFactory = (partId) -> new VirtualColumn(columnDescriptor.logicalIndex(), NativeTypes.INT32, false, partId);
+ private void addVirtualColumn(@Nullable ColumnDescriptor columnDescriptor) {
+ if (columnDescriptor == null) {
+ return;
}
+
+ assert columnDescriptor.virtual();
+
+ int columnIndex = columnDescriptor.logicalIndex();
+
+ virtualColumnsFactory.put(columnIndex, (partId) -> new VirtualColumn(columnIndex, NativeTypes.INT32, false, partId));
}
@Override
@@ -99,13 +107,25 @@ public class TableRowConverterFactoryImpl implements TableRowConverterFactory {
schemaRegistry,
schemaDescriptor,
requiredColumns,
- requireVirtualColumn ? createVirtualColumns(partId) : Int2ObjectMaps.emptyMap()
+ requireVirtualColumn ? createVirtualColumns(requiredColumns, partId) : Int2ObjectMaps.emptyMap()
);
}
- private Int2ObjectMap<VirtualColumn> createVirtualColumns(int partId) {
- VirtualColumn column = virtualColumnFactory.apply(partId);
+ private Int2ObjectMap<VirtualColumn> createVirtualColumns(BitSet requiredColumns, int partId) {
+ Int2ObjectMap<VirtualColumn> columnsMap = new Int2ObjectArrayMap<>(virtualColumnsFactory.size());
+
+ for (
+ int columnIndex = requiredColumns.nextSetBit(schemaDescriptor.length());
+ columnIndex >= 0;
+ columnIndex = requiredColumns.nextSetBit(columnIndex + 1)
+ ) {
+ columnsMap.put(columnIndex, virtualColumnsFactory.get(columnIndex).apply(partId));
+
+ if (columnIndex == Integer.MAX_VALUE) {
+ break; // Avoid overflow.
+ }
+ }
- return Int2ObjectMaps.singleton(column.columnIndex(), column);
+ return columnsMap;
}
}
diff --git a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/prepare/IgniteSqlValidator.java b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/prepare/IgniteSqlValidator.java
index cd7dc32fd07..a305f7a04ed 100644
--- a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/prepare/IgniteSqlValidator.java
+++ b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/prepare/IgniteSqlValidator.java
@@ -1291,7 +1291,8 @@ public class IgniteSqlValidator extends SqlValidatorImpl {
public static boolean isSystemFieldName(String alias) {
return (Commons.implicitPkEnabled() && Commons.IMPLICIT_PK_COL_NAME.equals(alias))
- || alias.equals(Commons.PART_COL_NAME);
+ || alias.equals(Commons.PART_COL_NAME)
+ || alias.equals(Commons.PART_COL_NAME_LEGACY);
}
// We use these scopes to filter out valid usages of a ROW operator.
diff --git a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/schema/SqlSchemaManagerImpl.java b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/schema/SqlSchemaManagerImpl.java
index 6fe85ed9155..e855a2baf2c 100644
--- a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/schema/SqlSchemaManagerImpl.java
+++ b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/schema/SqlSchemaManagerImpl.java
@@ -281,7 +281,7 @@ public class SqlSchemaManagerImpl implements SqlSchemaManager {
private static TableDescriptor createTableDescriptorForTable(CatalogTableDescriptor descriptor) {
List<CatalogTableColumnDescriptor> columns = descriptor.columns();
- List<ColumnDescriptor> colDescriptors = new ArrayList<>(columns.size() + 1);
+ List<ColumnDescriptor> colDescriptors = new ArrayList<>(columns.size() + 2);
Object2IntMap<String> columnToIndex = buildColumnToIndexMap(columns);
for (int i = 0; i < columns.size(); i++) {
@@ -301,8 +301,8 @@ public class SqlSchemaManagerImpl implements SqlSchemaManager {
}
// Add virtual column.
- ColumnDescriptorImpl partVirtualColumn = createPartitionVirtualColumn(columns.size());
- colDescriptors.add(partVirtualColumn);
+ colDescriptors.add(createPartitionVirtualColumn(columns.size(), Commons.PART_COL_NAME));
+ colDescriptors.add(createPartitionVirtualColumn(columns.size() + 1, Commons.PART_COL_NAME_LEGACY));
IgniteDistribution distribution = createDistribution(descriptor, columnToIndex);
@@ -331,9 +331,9 @@ public class SqlSchemaManagerImpl implements SqlSchemaManager {
return columnToIndex;
}
- private static ColumnDescriptorImpl createPartitionVirtualColumn(int logicalIndex) {
+ private static ColumnDescriptorImpl createPartitionVirtualColumn(int logicalIndex, String partColName) {
return new ColumnDescriptorImpl(
- Commons.PART_COL_NAME,
+ partColName,
false,
true,
true,
diff --git a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/util/Commons.java b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/util/Commons.java
index 1f8d2b49cef..6bee87dc25a 100644
--- a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/util/Commons.java
+++ b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/util/Commons.java
@@ -128,7 +128,9 @@ import org.jetbrains.annotations.TestOnly;
*/
public final class Commons {
public static final String IMPLICIT_PK_COL_NAME = "__p_key";
- public static final String PART_COL_NAME = "__part";
+ public static final String PART_COL_NAME = "__PART";
+ // Old name for partition column. Kept for backward compatibility.
+ public static final String PART_COL_NAME_LEGACY = "__part";
public static final int IN_BUFFER_SIZE = 512;
diff --git a/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/framework/TestBuilders.java b/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/framework/TestBuilders.java
index fb01f62fabf..43bbee14809 100644
--- a/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/framework/TestBuilders.java
+++ b/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/framework/TestBuilders.java
@@ -1086,6 +1086,19 @@ public class TestBuilders {
() -> {
throw new AssertionError("Partition virtual column is generated by a function");
}
+ ),
+ new ColumnDescriptorImpl(
+ Commons.PART_COL_NAME_LEGACY,
+ false,
+ true,
+ true,
+ false,
+ columns.size(),
+ NativeTypes.INT32,
+ DefaultValueStrategy.DEFAULT_COMPUTED,
+ () -> {
+ throw new AssertionError("Partition virtual column is generated by a function");
+ }
))), distribution);
Map<String, IgniteIndex> indexes = indexBuilders.stream()